From 6abbf79e7bbce219314defb539d3e62654815d3c Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 04:51:13 -0700 Subject: [PATCH 01/56] feat(cluster): record the PCM-X fail-closed arm per node The runtime fail-closed transition was silent: pcm_x_runtime_fail_closed() only flipped the gate and bumped a counter, so a fused node in a multi-node wedge (t/400 L3) could not be attributed to any of its ~77 call sites, and the t/400 probe only summed pg_cluster_state values across all four nodes. - Record the winning call site ("file:line") in PcmXShmemHeader and expose it as the text-valued pg_cluster_state key pcm_x_runtime_fail_closed_site; emit one LOG line per fuse outside critical sections. - Route every existing call site through a __FILE__/__LINE__ macro; call sites are textually unchanged (source-scan asserts unaffected). - t/400: per-node runtime probe (state / generation / recovery_blocked deltas / fail-closed site) so the L3 diag names the fused node and arm. - Unit: PcmXShmemHeader ABI pin 36400 -> 36480; new winning-site test; CritSectionCount + accessor stubs for the standalone links. --- src/backend/cluster/cluster_debug.c | 8 +++- src/backend/cluster/cluster_pcm_x_convert.c | 43 +++++++++++++++++-- src/include/cluster/cluster_pcm_x_convert.h | 21 +++++++-- .../t/400_pcm_x_queue_4node_liveness.pl | 19 ++++++++ src/test/cluster_unit/test_cluster_debug.c | 8 ++++ .../cluster_unit/test_cluster_pcm_x_convert.c | 28 +++++++++++- 6 files changed, 119 insertions(+), 8 deletions(-) diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 85134ca511..6d03fd698a 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -1995,15 +1995,21 @@ dump_pcm(ReturnSetInfo *rsinfo) emit_row(rsinfo, "pcm", "wm_prov_insert_fail_count", fmt_int64((int64)cluster_pcm_get_wm_prov_insert_fail_count())); - /* PCM-X queue core: 30 stable keys in the existing pcm category. */ + /* PCM-X queue core: 30 stable integer keys in the existing pcm category, + * plus one text-valued fail-closed provenance key (excluded from the + * integer-only snapshot tooling in the TAP suite). */ { PcmXStatsSnapshot stats = { 0 }; PcmXRuntimeSnapshot runtime = cluster_pcm_x_runtime_snapshot(); + char fail_closed_site[PCM_X_FAIL_CLOSED_SITE_LEN]; (void)cluster_pcm_x_stats_snapshot(&stats); emit_row(rsinfo, "pcm", "pcm_x_runtime_state", fmt_int32((int32)runtime.state)); emit_row(rsinfo, "pcm", "pcm_x_runtime_generation", fmt_int64((int64)runtime.gate_generation)); + if (!cluster_pcm_x_runtime_fail_closed_site(fail_closed_site, sizeof(fail_closed_site))) + strlcpy(fail_closed_site, "(none)", sizeof(fail_closed_site)); + emit_row(rsinfo, "pcm", "pcm_x_runtime_fail_closed_site", fail_closed_site); emit_row(rsinfo, "pcm", "pcm_x_queue_enqueue_count", fmt_int64((int64)stats.enqueue_count)); emit_row(rsinfo, "pcm", "pcm_x_queue_admit_count", fmt_int64((int64)stats.admit_count)); emit_row(rsinfo, "pcm", "pcm_x_queue_confirm_count", fmt_int64((int64)stats.confirm_count)); diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index 192c71249e..494b5d814e 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -1046,7 +1046,7 @@ cluster_pcm_x_stats_note_own_corrupt(void) static void -pcm_x_runtime_fail_closed(void) +pcm_x_runtime_fail_closed(const char *site_file, int site_line) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; bool transitioned; @@ -1057,14 +1057,51 @@ pcm_x_runtime_fail_closed(void) transitioned = cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_SHUTTING_DOWN, PCM_X_RUNTIME_RECOVERY_BLOCKED); if (transitioned && header != NULL) + { + const char *base; + + /* Record the fusing arm. Single writer: only the CAS winner above + * reaches this block, and a second fuse requires a full reactivation + * in between, so writes are serialized. */ + base = strrchr(site_file, '/'); + base = (base != NULL) ? base + 1 : site_file; + snprintf(header->fail_closed_site, sizeof(header->fail_closed_site), "%s:%d", base, + site_line); pcm_x_stats_increment(&header->stats.recovery_blocked_count); + + /* At most one log line per ACTIVE generation (single CAS winner), so + * this cannot flood. Skip inside critical sections; the shmem site + * above stays observable through pg_cluster_state either way. */ + if (CritSectionCount == 0) + ereport(LOG, + (errmsg("cluster PCM-X runtime fail-closed (recovery blocked) at %s:%d", base, + site_line), + errhint("Writer conversions on this node stay fail-closed until the PCM-X " + "runtime is reformed."))); + } } void -cluster_pcm_x_runtime_fail_closed(void) +cluster_pcm_x_runtime_fail_closed_at(const char *site_file, int site_line) { - pcm_x_runtime_fail_closed(); + pcm_x_runtime_fail_closed(site_file, site_line); +} + + +bool +cluster_pcm_x_runtime_fail_closed_site(char *buf, Size buflen) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (buf == NULL || buflen == 0) + return false; + buf[0] = '\0'; + if (header == NULL || header->fail_closed_site[0] == '\0') + return false; + strlcpy(buf, header->fail_closed_site, + Min(buflen, sizeof(header->fail_closed_site))); + return true; } diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h index 068ddee7cb..43d01b8f1e 100644 --- a/src/include/cluster/cluster_pcm_x_convert.h +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -1177,6 +1177,9 @@ typedef struct PcmXOutboundTargetFrontier { uint64 next_prehandle_sequence; } PcmXOutboundTargetFrontier; +/* Bytes reserved for the "file:line" fail-closed provenance string. */ +#define PCM_X_FAIL_CLOSED_SITE_LEN 80 + typedef struct PcmXShmemHeader { PcmXShmemLayout layout; PcmXAllocatorState allocator[PCM_X_ALLOC_COUNT]; @@ -1197,6 +1200,13 @@ typedef struct PcmXShmemHeader { PcmXPeerFrontier peer_frontiers[PCM_X_PROTOCOL_NODE_LIMIT]; PcmXStats stats; PcmXOutboundTargetFrontier outbound_targets[PCM_X_PROTOCOL_NODE_LIMIT]; + /* "file:line" of the call site that most recently won the transition into + * PCM_X_RUNTIME_RECOVERY_BLOCKED. Single-writer by construction: only the + * CAS winner inside pcm_x_runtime_fail_closed_at() writes it, and two + * successful transitions are always separated by a full reactivation. + * Readers are diagnostic-only (pg_cluster_state dump) and tolerate an + * in-progress overwrite after such a reactivation. */ + char fail_closed_site[PCM_X_FAIL_CLOSED_SITE_LEN]; } PcmXShmemHeader; StaticAssertDecl(sizeof(PcmXShmemLayout) == 440, "PCM-X shmem layout ABI"); @@ -1223,7 +1233,7 @@ StaticAssertDecl(offsetof(PcmXShmemHeader, peer_frontiers) == 33664, StaticAssertDecl(offsetof(PcmXShmemHeader, stats) == 35200, "PCM-X stats offset"); StaticAssertDecl(offsetof(PcmXShmemHeader, outbound_targets) == 35376, "PCM-X outbound target frontier array offset"); -StaticAssertDecl(sizeof(PcmXShmemHeader) == 36400, "PCM-X shmem header ABI"); +StaticAssertDecl(sizeof(PcmXShmemHeader) == 36480, "PCM-X shmem header ABI"); typedef enum PcmXAttachResult { PCM_X_ATTACH_OK = 0, @@ -1284,8 +1294,13 @@ extern bool cluster_pcm_x_runtime_reset_activating(uint32 expected_gate_generati extern bool cluster_pcm_x_runtime_transition(PcmXRuntimeState expected, PcmXRuntimeState desired); /* Cross-layer fail-closed seam for adapters that have already consumed an * irreversible external side effect and cannot safely retry it as ordinary - * queue work. */ -extern void cluster_pcm_x_runtime_fail_closed(void); + * queue work. The macro records the fusing arm (file:line) so a fused node + * can be diagnosed post-mortem without per-site plumbing. */ +extern void cluster_pcm_x_runtime_fail_closed_at(const char *site_file, int site_line); +#define cluster_pcm_x_runtime_fail_closed() \ + cluster_pcm_x_runtime_fail_closed_at(__FILE__, __LINE__) +/* Copies the recorded fail-closed site into buf; false when never fused. */ +extern bool cluster_pcm_x_runtime_fail_closed_site(char *buf, Size buflen); extern PcmXStepResult cluster_pcm_x_master_step(PcmXMasterTicketState current, PcmXMasterEvent event, uint32 guards, PcmXMasterTicketState *next); diff --git a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl index 1754c03a17..2c78972ca3 100644 --- a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl +++ b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl @@ -466,6 +466,25 @@ sub write_file . ($denied_after - $denied_before) . ' passive_s_release_delta=' . ($passive_s_after - $passive_s_before)); + +# Per-node runtime probe: the aggregate sums above cannot distinguish which +# node fused. Name the fused node and its fail-closed arm (file:line). +for my $i (0 .. 3) +{ + my $site = $quad->node($i)->safe_psql('postgres', + q{SELECT value FROM pg_cluster_state + WHERE category = 'pcm' AND key = 'pcm_x_runtime_fail_closed_site'}); + diag("L3 node$i runtime probe:" + . " state=$pcm_after_by_node[$i]{pcm_x_runtime_state}" + . " generation=$pcm_after_by_node[$i]{pcm_x_runtime_generation}" + . ' generation_delta=' + . ($pcm_after_by_node[$i]{pcm_x_runtime_generation} + - $pcm_before_by_node[$i]{pcm_x_runtime_generation}) + . ' recovery_blocked_delta=' + . ($pcm_after_by_node[$i]{pcm_x_queue_recovery_blocked_count} + - $pcm_before_by_node[$i]{pcm_x_queue_recovery_blocked_count}) + . " fail_closed_site=[$site]"); +} for my $key (@pcm_x_pcm_keys) { diag("L3 PCM-X state $key=$pcm_after{$key} delta=" diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index be7c737f80..b14241ae58 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -933,6 +933,14 @@ cluster_pcm_x_runtime_snapshot(void) return snapshot; } +bool +cluster_pcm_x_runtime_fail_closed_site(char *buf, Size buflen) +{ + if (buf != NULL && buflen > 0) + buf[0] = '\0'; + return false; +} + /* PGRAC spec-2.30 D9 R10 stub audit — 9 transition counter accessors. */ uint64 cluster_pcm_get_trans_n_to_s_count(void) diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 0b2e15d2dd..2ae5476421 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -250,6 +250,10 @@ cluster_shmem_register_region(const ClusterShmemRegion *region) registered_region_count++; } +/* Referenced by the fail-closed logging guard in cluster_pcm_x_convert.c; + * unit tests never run inside a critical section. */ +volatile uint32 CritSectionCount = 0; + bool errstart(int elevel, const char *domain pg_attribute_unused()) { @@ -2211,7 +2215,7 @@ UT_TEST(test_runtime_layout_abi_and_offsets_are_exact) UT_ASSERT_EQ(offsetof(PcmXShmemHeader, peer_frontiers), 33664); UT_ASSERT_EQ(offsetof(PcmXShmemHeader, stats), 35200); UT_ASSERT_EQ(offsetof(PcmXShmemHeader, outbound_targets), 35376); - UT_ASSERT_EQ(sizeof(PcmXShmemHeader), 36400); + UT_ASSERT_EQ(sizeof(PcmXShmemHeader), 36480); } UT_TEST(test_lwlock_held_limit_is_shared_200) @@ -7318,6 +7322,27 @@ UT_TEST(test_recovery_blocked_runtime_refuses_ack_and_local_mutators) UT_ASSERT(memcmp(member, &member_before, sizeof(*member)) == 0); } +UT_TEST(test_runtime_fail_closed_records_winning_site_only) +{ + char site[PCM_X_FAIL_CLOSED_SITE_LEN]; + char first[PCM_X_FAIL_CLOSED_SITE_LEN]; + + init_active_pcm_x(UINT64_C(77)); + /* Before any fuse the site is absent and the buffer is cleared. */ + site[0] = 'x'; + UT_ASSERT(!cluster_pcm_x_runtime_fail_closed_site(site, sizeof(site))); + UT_ASSERT_EQ(site[0], '\0'); + /* The macro records this file:line as the fusing arm. */ + cluster_pcm_x_runtime_fail_closed(); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT(cluster_pcm_x_runtime_fail_closed_site(first, sizeof(first))); + UT_ASSERT(strncmp(first, "test_cluster_pcm_x_convert.c:", 29) == 0); + /* A losing caller (runtime already blocked) must not clobber the site. */ + cluster_pcm_x_runtime_fail_closed(); + UT_ASSERT(cluster_pcm_x_runtime_fail_closed_site(site, sizeof(site))); + UT_ASSERT(strcmp(site, first) == 0); +} + UT_TEST(test_master_cancel_is_exact_and_unlinks_middle_without_fifo_damage) { PcmXShmemHeader *header; @@ -14273,6 +14298,7 @@ main(void) UT_RUN(test_delayed_pretransfer_cancel_fences_active_transfer); UT_RUN(test_recovery_blocked_runtime_cannot_promote_or_begin_transfer); UT_RUN(test_recovery_blocked_runtime_refuses_ack_and_local_mutators); + UT_RUN(test_runtime_fail_closed_records_winning_site_only); UT_RUN(test_master_cancel_is_exact_and_unlinks_middle_without_fifo_damage); UT_RUN(test_master_prehandle_cancel_replays_exactly_and_never_hits_reused_slot); UT_RUN(test_master_prehandle_identity_alias_is_corruption_not_stale_cancel); From 9e6d8b616080fbbe5dd9376366b21eba8453b6df Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 04:54:47 -0700 Subject: [PATCH 02/56] test(cluster): sync pinned pcm dump counts for the fail-closed site key t/024 L1, t/108 L1, and the unit exact-key-set dump test pin the pcm category row count (58 -> 59) and the exhaustive key list; the new pcm_x_runtime_fail_closed_site row joins both. --- src/test/cluster_tap/t/024_pcm_lock.pl | 6 +++--- src/test/cluster_tap/t/108_pcm_state_machine.pl | 4 ++-- src/test/cluster_unit/test_cluster_debug.c | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/test/cluster_tap/t/024_pcm_lock.pl b/src/test/cluster_tap/t/024_pcm_lock.pl index 4d8520341e..01adc6236c 100644 --- a/src/test/cluster_tap/t/024_pcm_lock.pl +++ b/src/test/cluster_tap/t/024_pcm_lock.pl @@ -55,14 +55,14 @@ # ---------- -# L1: pg_cluster_state.pcm category has 58 keys, including the 30-key PCM-X FIFO surface. +# L1: pg_cluster_state.pcm category has 59 keys, including the 31-key PCM-X FIFO surface. # activates the state-machine diagnostics. # ---------- is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='pcm'}), - '58', - 'L1 pg_cluster_state.pcm category has 58 keys (existing 28 + PCM-X FIFO/ownership/runtime 30)'); + '59', + 'L1 pg_cluster_state.pcm category has 59 keys (existing 28 + PCM-X FIFO/ownership/runtime 30 + fail-closed site)'); # ---------- diff --git a/src/test/cluster_tap/t/108_pcm_state_machine.pl b/src/test/cluster_tap/t/108_pcm_state_machine.pl index 0081dafc7b..9f7241696c 100644 --- a/src/test/cluster_tap/t/108_pcm_state_machine.pl +++ b/src/test/cluster_tap/t/108_pcm_state_machine.pl @@ -48,8 +48,8 @@ my $pcm_category_rows = $node_default->safe_psql( 'postgres', "SELECT count(*) FROM pg_cluster_state WHERE category = 'pcm'"); -is($pcm_category_rows, '58', - 'L1 pg_cluster_state pcm category has 58 rows (existing 28 + PCM-X FIFO/ownership/runtime 30)'); +is($pcm_category_rows, '59', + 'L1 pg_cluster_state pcm category has 59 rows (existing 28 + PCM-X FIFO/ownership/runtime 30 + fail-closed site)'); # L3 — api_state shows "active" when GUC=-1 default my $api_state_default = $node_default->safe_psql( diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index b14241ae58..94cb8d42f8 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -4246,6 +4246,7 @@ UT_TEST(test_debug_dump_exposes_exact_pcm_x_lmd_and_gcs_key_sets) static const char *const pcm_keys[] = { "pcm_x_runtime_state", "pcm_x_runtime_generation", + "pcm_x_runtime_fail_closed_site", "pcm_x_queue_enqueue_count", "pcm_x_queue_admit_count", "pcm_x_queue_confirm_count", @@ -4302,7 +4303,7 @@ UT_TEST(test_debug_dump_exposes_exact_pcm_x_lmd_and_gcs_key_sets) fcinfo->resultinfo = (fmNodePtr)&rsinfo; (void)cluster_dump_state(fcinfo); - UT_ASSERT_EQ(captured_dump_count("pcm", NULL), 58); + UT_ASSERT_EQ(captured_dump_count("pcm", NULL), 59); UT_ASSERT_EQ(captured_dump_count("lmd", NULL), 51); UT_ASSERT_EQ(captured_dump_count("gcs", NULL), 119); for (i = 0; i < (int)lengthof(pcm_keys); i++) From c2b2b4109ea4c726921d4975f87151700b585251 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 04:56:52 -0700 Subject: [PATCH 03/56] fix(cluster): capture the internal convert.c fail-closed arms too The static pcm_x_runtime_fail_closed() has ~170 zero-argument internal call sites in cluster_pcm_x_convert.c (the majority of all fail-closed arms; the previous commit only routed the extern seam). Rename the impl and mirror the __FILE__/__LINE__ macro file-locally so every internal arm records its site with the call sites textually unchanged; also fixes the stale forward declaration that broke the build. --- src/backend/cluster/cluster_pcm_x_convert.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index 494b5d814e..e1e71f51f6 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -561,7 +561,11 @@ typedef enum PcmXDirectoryMatch { } PcmXDirectoryMatch; -static void pcm_x_runtime_fail_closed(void); +static void pcm_x_runtime_fail_closed_impl(const char *site_file, int site_line); + +/* Capture each internal fail-closed arm (file:line) while keeping the many + * zero-argument call sites in this file textually unchanged. */ +#define pcm_x_runtime_fail_closed() pcm_x_runtime_fail_closed_impl(__FILE__, __LINE__) /* Required runtime precondition: never nest allocator under a tag domain. */ @@ -1046,7 +1050,7 @@ cluster_pcm_x_stats_note_own_corrupt(void) static void -pcm_x_runtime_fail_closed(const char *site_file, int site_line) +pcm_x_runtime_fail_closed_impl(const char *site_file, int site_line) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; bool transitioned; @@ -1085,7 +1089,7 @@ pcm_x_runtime_fail_closed(const char *site_file, int site_line) void cluster_pcm_x_runtime_fail_closed_at(const char *site_file, int site_line) { - pcm_x_runtime_fail_closed(site_file, site_line); + pcm_x_runtime_fail_closed_impl(site_file, site_line); } From ad47d84a08aa6a38ab481df35134e3fcaf69eab7 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 05:00:04 -0700 Subject: [PATCH 04/56] style(cluster): clang-format the fail-closed observability files --- src/backend/cluster/cluster_debug.c | 2 +- src/backend/cluster/cluster_pcm_x_convert.c | 6 ++---- src/include/cluster/cluster_pcm_x_convert.h | 3 +-- src/test/cluster_unit/test_cluster_pcm_x_convert.c | 4 ++-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 6d03fd698a..3e3e832542 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -2001,7 +2001,7 @@ dump_pcm(ReturnSetInfo *rsinfo) { PcmXStatsSnapshot stats = { 0 }; PcmXRuntimeSnapshot runtime = cluster_pcm_x_runtime_snapshot(); - char fail_closed_site[PCM_X_FAIL_CLOSED_SITE_LEN]; + char fail_closed_site[PCM_X_FAIL_CLOSED_SITE_LEN]; (void)cluster_pcm_x_stats_snapshot(&stats); emit_row(rsinfo, "pcm", "pcm_x_runtime_state", fmt_int32((int32)runtime.state)); diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index e1e71f51f6..df5ff39f8c 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -1060,8 +1060,7 @@ pcm_x_runtime_fail_closed_impl(const char *site_file, int site_line) if (!transitioned) transitioned = cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_SHUTTING_DOWN, PCM_X_RUNTIME_RECOVERY_BLOCKED); - if (transitioned && header != NULL) - { + if (transitioned && header != NULL) { const char *base; /* Record the fusing arm. Single writer: only the CAS winner above @@ -1103,8 +1102,7 @@ cluster_pcm_x_runtime_fail_closed_site(char *buf, Size buflen) buf[0] = '\0'; if (header == NULL || header->fail_closed_site[0] == '\0') return false; - strlcpy(buf, header->fail_closed_site, - Min(buflen, sizeof(header->fail_closed_site))); + strlcpy(buf, header->fail_closed_site, Min(buflen, sizeof(header->fail_closed_site))); return true; } diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h index 43d01b8f1e..2e8d55e9ad 100644 --- a/src/include/cluster/cluster_pcm_x_convert.h +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -1297,8 +1297,7 @@ extern bool cluster_pcm_x_runtime_transition(PcmXRuntimeState expected, PcmXRunt * queue work. The macro records the fusing arm (file:line) so a fused node * can be diagnosed post-mortem without per-site plumbing. */ extern void cluster_pcm_x_runtime_fail_closed_at(const char *site_file, int site_line); -#define cluster_pcm_x_runtime_fail_closed() \ - cluster_pcm_x_runtime_fail_closed_at(__FILE__, __LINE__) +#define cluster_pcm_x_runtime_fail_closed() cluster_pcm_x_runtime_fail_closed_at(__FILE__, __LINE__) /* Copies the recorded fail-closed site into buf; false when never fused. */ extern bool cluster_pcm_x_runtime_fail_closed_site(char *buf, Size buflen); extern PcmXStepResult cluster_pcm_x_master_step(PcmXMasterTicketState current, diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 2ae5476421..47c711ce28 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -7324,8 +7324,8 @@ UT_TEST(test_recovery_blocked_runtime_refuses_ack_and_local_mutators) UT_TEST(test_runtime_fail_closed_records_winning_site_only) { - char site[PCM_X_FAIL_CLOSED_SITE_LEN]; - char first[PCM_X_FAIL_CLOSED_SITE_LEN]; + char site[PCM_X_FAIL_CLOSED_SITE_LEN]; + char first[PCM_X_FAIL_CLOSED_SITE_LEN]; init_active_pcm_x(UINT64_C(77)); /* Before any fuse the site is absent and the buffer is cleared. */ From c0955a9c6a163c5b17889dfb36f060b292e880d6 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 05:06:18 -0700 Subject: [PATCH 05/56] test(cluster): t/400 mid-leg probe of writer wait events and queue gauges A wedged writer holds no observable state post-mortem (kill_kill), so the existing after-leg probe cannot distinguish where the stalled writers wait. Snapshot pg_stat_activity wait events and the live queue gauges per node at start_at+8s while the leg is still running; bounded, failure-tolerant, diag only. --- .../t/400_pcm_x_queue_4node_liveness.pl | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl index 2c78972ca3..adfa9736a6 100644 --- a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl +++ b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl @@ -422,6 +422,41 @@ sub write_file push @runs, \%run; } +# Mid-leg probe: while the four writers are (potentially) stalled, capture +# each node's writer wait state and the live queue gauges. A post-mortem +# probe cannot see this — a wedged writer holds no cluster state after +# kill_kill. Diagnostic only; every query is bounded and failure-tolerant. +{ + my $probe_at = $quad->node0->safe_psql('postgres', + "SELECT GREATEST(0.0, EXTRACT(EPOCH FROM " + . "(TIMESTAMPTZ '$start_at' + interval '8 seconds' - clock_timestamp())))"); + sleep($probe_at) if $probe_at > 0; + for my $i (0 .. 3) + { + my $waits = eval { + $quad->node($i)->safe_psql('postgres', + q{SELECT pid || ':' || state || ':' || coalesce(wait_event_type, '-') + || '/' || coalesce(wait_event, '-') + FROM pg_stat_activity + WHERE query LIKE 'UPDATE pcm_xq_hot%'}, + timeout => 10); + } // 'probe-failed'; + $waits =~ s/\n/ | /g; + my $gauges = eval { + $quad->node($i)->safe_psql('postgres', + q{SELECT string_agg(key || '=' || value, ' ' ORDER BY key) + FROM pg_cluster_state + WHERE category = 'pcm' + AND key IN ('pcm_x_runtime_state', 'pcm_x_queue_depth', + 'pcm_x_queue_live_tickets', 'pcm_x_queue_active_tags', + 'pcm_x_queue_promotion_count', 'pcm_x_queue_stale_count', + 'pcm_x_queue_recovery_blocked_count')}, + timeout => 10); + } // 'probe-failed'; + diag("L3 mid-leg node$i waits=[$waits] gauges=[$gauges]"); + } +} + for my $i (0 .. 3) { my $run = $runs[$i]; From 69f63a98fc3b4b03367a6d044d3afa0817f1af68 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 05:12:53 -0700 Subject: [PATCH 06/56] test(cluster): two-sample mid-leg probe to fingerprint the stall loop Sampling every pcm key per node at start_at+8s and +12s and printing the per-node movement between the samples separates the live retry loop (still ticking) from the wedged pipeline stages (frozen) during a pure L3 stall. --- .../t/400_pcm_x_queue_4node_liveness.pl | 64 +++++++++++-------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl index adfa9736a6..96c6e741fd 100644 --- a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl +++ b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl @@ -427,33 +427,47 @@ sub write_file # probe cannot see this — a wedged writer holds no cluster state after # kill_kill. Diagnostic only; every query is bounded and failure-tolerant. { - my $probe_at = $quad->node0->safe_psql('postgres', - "SELECT GREATEST(0.0, EXTRACT(EPOCH FROM " - . "(TIMESTAMPTZ '$start_at' + interval '8 seconds' - clock_timestamp())))"); - sleep($probe_at) if $probe_at > 0; + my @samples; + + for my $offset (8, 12) + { + my $probe_at = $quad->node0->safe_psql('postgres', + "SELECT GREATEST(0.0, EXTRACT(EPOCH FROM " + . "(TIMESTAMPTZ '$start_at' + interval '$offset seconds' - clock_timestamp())))"); + sleep($probe_at) if $probe_at > 0; + for my $i (0 .. 3) + { + my $waits = eval { + $quad->node($i)->safe_psql('postgres', + q{SELECT pid || ':' || state || ':' || coalesce(wait_event_type, '-') + || '/' || coalesce(wait_event, '-') + FROM pg_stat_activity + WHERE query LIKE 'UPDATE pcm_xq_hot%'}, + timeout => 10); + } // 'probe-failed'; + $waits =~ s/\n/ | /g; + $samples[$offset][$i] = eval { + state_snapshot($quad->node($i), 'pcm', \@pcm_x_pcm_keys); + }; + diag("L3 mid-leg t+$offset node$i waits=[$waits]" + . ($samples[$offset][$i] ? '' : ' snapshot-failed')); + } + } + + # During a pure stall the t+8 -> t+12 per-node counter movement names the + # spinning arm: whatever still ticks is the live retry loop, whatever is + # frozen is wedged. for my $i (0 .. 3) { - my $waits = eval { - $quad->node($i)->safe_psql('postgres', - q{SELECT pid || ':' || state || ':' || coalesce(wait_event_type, '-') - || '/' || coalesce(wait_event, '-') - FROM pg_stat_activity - WHERE query LIKE 'UPDATE pcm_xq_hot%'}, - timeout => 10); - } // 'probe-failed'; - $waits =~ s/\n/ | /g; - my $gauges = eval { - $quad->node($i)->safe_psql('postgres', - q{SELECT string_agg(key || '=' || value, ' ' ORDER BY key) - FROM pg_cluster_state - WHERE category = 'pcm' - AND key IN ('pcm_x_runtime_state', 'pcm_x_queue_depth', - 'pcm_x_queue_live_tickets', 'pcm_x_queue_active_tags', - 'pcm_x_queue_promotion_count', 'pcm_x_queue_stale_count', - 'pcm_x_queue_recovery_blocked_count')}, - timeout => 10); - } // 'probe-failed'; - diag("L3 mid-leg node$i waits=[$waits] gauges=[$gauges]"); + next unless $samples[8][$i] && $samples[12][$i]; + my @moved; + for my $key (@pcm_x_pcm_keys) + { + my $d = $samples[12][$i]{$key} - $samples[8][$i]{$key}; + push @moved, "$key:+$d" if $d != 0; + } + diag("L3 mid-leg node$i t+8..t+12 moved: " + . (@moved ? join(' ', @moved) : '(all frozen)')); } } From 776af9d5dd87784c1510a19e33f54619c760fdd0 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 05:23:09 -0700 Subject: [PATCH 07/56] test(cluster): mid-leg probe adds aux-process waits and per-peer wire deltas The terminal-leg retry path stages IC frames without touching any pcm counter, so a frozen pcm fingerprint cannot distinguish a silent resend loop from a dead pump. Sample cluster aux process wait events and pg_cluster_ic_peers msg send/recv counts at both probe instants. --- .../t/400_pcm_x_queue_4node_liveness.pl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl index 96c6e741fd..70ecda49fc 100644 --- a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl +++ b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl @@ -446,11 +446,30 @@ sub write_file timeout => 10); } // 'probe-failed'; $waits =~ s/\n/ | /g; + my $aux = eval { + $quad->node($i)->safe_psql('postgres', + q{SELECT backend_type || ':' || coalesce(wait_event, '-') + FROM pg_stat_activity + WHERE backend_type IN ('lmon', 'lms', 'lms worker', 'lck', + 'cssd', 'diag', 'cluster stats', 'qvotec') + ORDER BY backend_type}, + timeout => 10); + } // 'probe-failed'; + $aux =~ s/\n/ | /g; + my $wire = eval { + $quad->node($i)->safe_psql('postgres', + q{SELECT string_agg('peer' || node_id || ':s' || msg_send_count + || ':r' || msg_recv_count, ' ' ORDER BY node_id) + FROM pg_cluster_ic_peers}, + timeout => 10); + } // 'probe-failed'; $samples[$offset][$i] = eval { state_snapshot($quad->node($i), 'pcm', \@pcm_x_pcm_keys); }; diag("L3 mid-leg t+$offset node$i waits=[$waits]" . ($samples[$offset][$i] ? '' : ' snapshot-failed')); + diag("L3 mid-leg t+$offset node$i aux=[$aux]"); + diag("L3 mid-leg t+$offset node$i wire=[$wire]"); } } From 9eec4932d6d7f325aed469999319cebc3a936aeb Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 05:23:33 -0700 Subject: [PATCH 08/56] test(cluster): include lmd and interconnect listener in the aux-wait probe --- src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl index 70ecda49fc..fe59a204f9 100644 --- a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl +++ b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl @@ -450,8 +450,8 @@ sub write_file $quad->node($i)->safe_psql('postgres', q{SELECT backend_type || ':' || coalesce(wait_event, '-') FROM pg_stat_activity - WHERE backend_type IN ('lmon', 'lms', 'lms worker', 'lck', - 'cssd', 'diag', 'cluster stats', 'qvotec') + WHERE backend_type IN ('lmon', 'lms', 'lms worker', 'lck', 'lmd', + 'cssd', 'diag', 'cluster stats', 'qvotec', 'interconnect listener') ORDER BY backend_type}, timeout => 10); } // 'probe-failed'; From 5d686d2b53e0f4fb1f0e6e3aa7c627050561fcf4 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 05:30:55 -0700 Subject: [PATCH 09/56] feat(cluster): per-slot master tag/ticket diagnostic rows in the pcm dump A wedged tag whose terminal legs stall silently leaves no counter or log trace (the terminal path stages IC frames without stats). Emit one text-valued pg_cluster_state row per live master tag slot (FIFO shape: head/tail/active/queued bitmap) and per occupied ticket slot (state, leg bitmaps, reliable-leg opcode/phase/retry/responder) so the stuck leg and the missing ACK node are directly readable; racy-tolerant reads, rows present only while slots are occupied. t/400 mid-leg probe prints them. --- src/backend/cluster/cluster_debug.c | 20 ++++ src/backend/cluster/cluster_pcm_x_convert.c | 106 ++++++++++++++++++ src/include/cluster/cluster_pcm_x_convert.h | 6 + .../t/400_pcm_x_queue_4node_liveness.pl | 10 ++ src/test/cluster_unit/test_cluster_debug.c | 12 ++ .../cluster_unit/test_cluster_pcm_x_convert.c | 28 +++++ 6 files changed, 182 insertions(+) diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 3e3e832542..53c8d6ad4c 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -2010,6 +2010,26 @@ dump_pcm(ReturnSetInfo *rsinfo) if (!cluster_pcm_x_runtime_fail_closed_site(fail_closed_site, sizeof(fail_closed_site))) strlcpy(fail_closed_site, "(none)", sizeof(fail_closed_site)); emit_row(rsinfo, "pcm", "pcm_x_runtime_fail_closed_site", fail_closed_site); + + /* Text-valued per-slot diagnostic rows; present only while master tag + * or ticket slots are occupied, so the quiescent key count above is + * unchanged. */ + { + Size cursor = 0; + Size index = 0; + char line[256]; + char key[64]; + + while (cluster_pcm_x_master_tag_debug_next(&cursor, &index, line, sizeof(line))) { + snprintf(key, sizeof(key), "pcm_x_tag_%zu", index); + emit_row(rsinfo, "pcm", key, line); + } + cursor = 0; + while (cluster_pcm_x_master_ticket_debug_next(&cursor, &index, line, sizeof(line))) { + snprintf(key, sizeof(key), "pcm_x_ticket_%zu", index); + emit_row(rsinfo, "pcm", key, line); + } + } emit_row(rsinfo, "pcm", "pcm_x_queue_enqueue_count", fmt_int64((int64)stats.enqueue_count)); emit_row(rsinfo, "pcm", "pcm_x_queue_admit_count", fmt_int64((int64)stats.admit_count)); emit_row(rsinfo, "pcm", "pcm_x_queue_confirm_count", fmt_int64((int64)stats.confirm_count)); diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index df5ff39f8c..fa045ec977 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -1107,6 +1107,112 @@ cluster_pcm_x_runtime_fail_closed_site(char *buf, Size buflen) } +/* Map an FIFO index to a printable value: PCM_X_INVALID_SLOT_INDEX -> -1. */ +static long +pcm_x_debug_index(Size index) +{ + return index == PCM_X_INVALID_SLOT_INDEX ? -1L : (long)index; +} + + +/* Diagnostic iterator over live master tag slots. Lock-free and + * racy-tolerant: each slot is copied under a generation double-read and + * formatted only from the stable local copy; slots that mutate mid-copy are + * skipped. Diagnostic surface only — never used for protocol decisions. */ +bool +cluster_pcm_x_master_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, Size buflen) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXAllocatorView view; + PcmXMasterTagSlot copy; + Size i; + + if (header == NULL || cursor_io == NULL || index_out == NULL || buf == NULL || buflen == 0) + return false; + if (!pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TAG, &view)) + return false; + for (i = *cursor_io; i < view.capacity; i++) { + PcmXMasterTagSlot *raw = (PcmXMasterTagSlot *)pcm_x_allocator_slot(&view, i); + uint64 gen_before; + uint64 gen_after; + + if (raw == NULL || pcm_x_slot_state_read(&raw->slot) != PCM_X_TAG_LIVE + || !pcm_x_slot_generation_read(&raw->slot, &gen_before)) + continue; + memcpy(©, raw, sizeof(copy)); + pg_read_barrier(); + if (!pcm_x_slot_generation_read(&raw->slot, &gen_after) || gen_after != gen_before + || pcm_x_slot_state_read(&raw->slot) != PCM_X_TAG_LIVE) + continue; + snprintf(buf, buflen, + "rel=%u blk=%u head=%ld tail=%ld active=%ld queued=0x%x qseq=" UINT64_FORMAT + " outstanding=%zu", + copy.tag.relNumber, copy.tag.blockNum, pcm_x_debug_index(copy.head_index), + pcm_x_debug_index(copy.tail_index), pcm_x_debug_index(copy.active_index), + pg_atomic_read_u32(©.queued_node_bitmap), copy.queue_state_sequence, + copy.outstanding_ticket_count); + *index_out = i; + *cursor_io = i + 1; + return true; + } + *cursor_io = view.capacity; + return false; +} + + +/* Diagnostic iterator over occupied master ticket slots; same racy-tolerant + * contract as the tag iterator. */ +bool +cluster_pcm_x_master_ticket_debug_next(Size *cursor_io, Size *index_out, char *buf, Size buflen) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXAllocatorView view; + PcmXMasterTicketSlot copy; + Size i; + + if (header == NULL || cursor_io == NULL || index_out == NULL || buf == NULL || buflen == 0) + return false; + if (!pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TICKET, &view)) + return false; + for (i = *cursor_io; i < view.capacity; i++) { + PcmXMasterTicketSlot *raw = (PcmXMasterTicketSlot *)pcm_x_allocator_slot(&view, i); + uint64 gen_before; + uint64 gen_after; + uint32 state; + + if (raw == NULL) + continue; + state = pcm_x_slot_state_read(&raw->slot); + if (state == PCM_XT_FREE || state == PCM_XT_RESERVED_NONVISIBLE) + continue; + if (!pcm_x_slot_generation_read(&raw->slot, &gen_before)) + continue; + memcpy(©, raw, sizeof(copy)); + pg_read_barrier(); + if (!pcm_x_slot_generation_read(&raw->slot, &gen_after) || gen_after != gen_before) + continue; + snprintf(buf, buflen, + "state=%u flags=0x%x node=%d ticket=" UINT64_FORMAT " grant=" UINT64_FORMAT + " tomb=0x%llx involved=0x%x drained=0x%x retire_acked=0x%x pending_s=0x%x" + " acked_s=0x%x leg_op=%u leg_phase=%u leg_flags=0x%x leg_retry=%u leg_node=%d" + " leg_deadline=" UINT64_FORMAT, + pcm_x_slot_state_read(©.slot), pcm_x_slot_flags_read(©.slot), + copy.ref.identity.node_id, copy.ref.handle.ticket_id, copy.ref.grant_generation, + (unsigned long long)copy.reliable.response_tombstone_mask, + copy.involved_nodes_bitmap, copy.drained_nodes_bitmap, + copy.retire_acked_nodes_bitmap, copy.pending_s_holders_bitmap, + copy.acked_s_holders_bitmap, copy.reliable.pending_opcode, copy.reliable.phase, + copy.reliable.flags, copy.reliable.retry_count, + copy.reliable.expected_responder_node, copy.reliable.retry_deadline_ms); + *index_out = i; + *cursor_io = i + 1; + return true; + } + *cursor_io = view.capacity; + return false; +} + + /* Reserve one slot under the sole allocator authority. */ PcmXAllocatorResult cluster_pcm_x_allocator_reserve(PcmXAllocatorKind kind, PcmXSlotRef *ref_out, diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h index 2e8d55e9ad..6d853ab688 100644 --- a/src/include/cluster/cluster_pcm_x_convert.h +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -1300,6 +1300,12 @@ extern void cluster_pcm_x_runtime_fail_closed_at(const char *site_file, int site #define cluster_pcm_x_runtime_fail_closed() cluster_pcm_x_runtime_fail_closed_at(__FILE__, __LINE__) /* Copies the recorded fail-closed site into buf; false when never fused. */ extern bool cluster_pcm_x_runtime_fail_closed_site(char *buf, Size buflen); +/* Diagnostic iterators over live master tag / occupied ticket slots for the + * pg_cluster_state dump; racy-tolerant, never used for protocol decisions. */ +extern bool cluster_pcm_x_master_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, + Size buflen); +extern bool cluster_pcm_x_master_ticket_debug_next(Size *cursor_io, Size *index_out, char *buf, + Size buflen); extern PcmXStepResult cluster_pcm_x_master_step(PcmXMasterTicketState current, PcmXMasterEvent event, uint32 guards, PcmXMasterTicketState *next); diff --git a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl index fe59a204f9..f912f5a7d5 100644 --- a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl +++ b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl @@ -463,6 +463,15 @@ sub write_file FROM pg_cluster_ic_peers}, timeout => 10); } // 'probe-failed'; + my $slots = eval { + $quad->node($i)->safe_psql('postgres', + q{SELECT string_agg(key || '=[' || value || ']', ' ' ORDER BY key) + FROM pg_cluster_state + WHERE category = 'pcm' + AND (key LIKE 'pcm_x_tag_%' OR key LIKE 'pcm_x_ticket_%')}, + timeout => 10); + } // 'probe-failed'; + $slots =~ s/\n/ | /g; $samples[$offset][$i] = eval { state_snapshot($quad->node($i), 'pcm', \@pcm_x_pcm_keys); }; @@ -470,6 +479,7 @@ sub write_file . ($samples[$offset][$i] ? '' : ' snapshot-failed')); diag("L3 mid-leg t+$offset node$i aux=[$aux]"); diag("L3 mid-leg t+$offset node$i wire=[$wire]"); + diag("L3 mid-leg t+$offset node$i slots=[$slots]"); } } diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 94cb8d42f8..4ed77bfc2c 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -941,6 +941,18 @@ cluster_pcm_x_runtime_fail_closed_site(char *buf, Size buflen) return false; } +bool +cluster_pcm_x_master_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, Size buflen) +{ + return false; +} + +bool +cluster_pcm_x_master_ticket_debug_next(Size *cursor_io, Size *index_out, char *buf, Size buflen) +{ + return false; +} + /* PGRAC spec-2.30 D9 R10 stub audit — 9 transition counter accessors. */ uint64 cluster_pcm_get_trans_n_to_s_count(void) diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 47c711ce28..4cb034d816 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -7343,6 +7343,33 @@ UT_TEST(test_runtime_fail_closed_records_winning_site_only) UT_ASSERT(strcmp(site, first) == 0); } +UT_TEST(test_master_debug_iterators_report_live_slots) +{ + PcmXMasterAdmission admission; + PcmXEnqueuePayload request; + Size cursor = 0; + Size index = 0; + char line[256]; + + init_active_pcm_x(UINT64_C(77)); + /* Nothing admitted yet: both iterators must report no occupied slots. */ + UT_ASSERT(!cluster_pcm_x_master_tag_debug_next(&cursor, &index, line, sizeof(line))); + cursor = 0; + UT_ASSERT(!cluster_pcm_x_master_ticket_debug_next(&cursor, &index, line, sizeof(line))); + request = make_enqueue(make_wait_identity(716, 0, 25, UINT64_C(21001)), UINT64_C(2201), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + cursor = 0; + UT_ASSERT(cluster_pcm_x_master_tag_debug_next(&cursor, &index, line, sizeof(line))); + UT_ASSERT(strstr(line, "queued=") != NULL); + UT_ASSERT(!cluster_pcm_x_master_tag_debug_next(&cursor, &index, line, sizeof(line))); + cursor = 0; + UT_ASSERT(cluster_pcm_x_master_ticket_debug_next(&cursor, &index, line, sizeof(line))); + UT_ASSERT(strstr(line, "state=") != NULL); + UT_ASSERT(strstr(line, "leg_op=") != NULL); +} + UT_TEST(test_master_cancel_is_exact_and_unlinks_middle_without_fifo_damage) { PcmXShmemHeader *header; @@ -14299,6 +14326,7 @@ main(void) UT_RUN(test_recovery_blocked_runtime_cannot_promote_or_begin_transfer); UT_RUN(test_recovery_blocked_runtime_refuses_ack_and_local_mutators); UT_RUN(test_runtime_fail_closed_records_winning_site_only); + UT_RUN(test_master_debug_iterators_report_live_slots); UT_RUN(test_master_cancel_is_exact_and_unlinks_middle_without_fifo_damage); UT_RUN(test_master_prehandle_cancel_replays_exactly_and_never_hits_reused_slot); UT_RUN(test_master_prehandle_identity_alias_is_corruption_not_stale_cancel); From 194f3f7c4d2d80a356a73f73c94b4adc3394429f Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 05:45:54 -0700 Subject: [PATCH 10/56] fix(cluster): coherent slot dump + requester escape-arm evidence Per the t/400 root-cause audit (pgrac scratchpad review): - The per-slot dump iterators now revalidate and copy under the matching master partition shared lock (the same lock every in-place field mutation holds) instead of a generation double-read, so a printed combination is a state that actually existed. - Every non-OK acquire_writer_impl exit records its source line; the nested-wait guard records the held content-lock tag and result that closed it. The bufmgr writer failure report prints fail_line and guard evidence, closing the silent BARRIER_CLOSED(13) client-escape attribution gap. - t/400 mid-leg probe samples at t+1/t+3/t+8 to catch the terminal handoff mid-flight, with movement printed per sample pair. --- src/backend/cluster/cluster_gcs_block.c | 51 +- src/backend/cluster/cluster_pcm_x_convert.c | 85 +- src/backend/storage/buffer/bufmgr.c | 3658 +++++++---------- src/include/cluster/cluster_gcs_block.h | 3 + src/include/cluster/cluster_pcm_x_convert.h | 3 + .../t/400_pcm_x_queue_4node_liveness.pl | 28 +- 6 files changed, 1625 insertions(+), 2203 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index cb267c4aa2..497eff39b5 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -8075,6 +8075,23 @@ typedef struct GcsBlockPcmXRequesterCleanupContext { static GcsBlockPcmXRequesterCleanupContext gcs_block_pcm_x_requester_cleanup_context; static bool gcs_block_pcm_x_requester_exit_hook_registered = false; +/* Source line of this backend's most recent non-OK acquire_writer_impl exit. + * Diagnostic only: consumed by the bufmgr failure report so a client-visible + * writer error names the exact escape arm instead of just the result code. */ +static int gcs_block_pcm_x_requester_fail_line = 0; + +#define GCS_BLOCK_PCM_X_REQUESTER_DONE() \ + do { \ + gcs_block_pcm_x_requester_fail_line = __LINE__; \ + goto requester_done; \ + } while (0) + +int +cluster_gcs_pcm_x_requester_last_fail_line(void) +{ + return gcs_block_pcm_x_requester_fail_line; +} + static bool gcs_block_pcm_x_revalidate_peer_binding(int32 node_id, uint64 epoch, uint64 session); static void @@ -8456,7 +8473,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (wait_seq == 0) { cluster_pcm_x_runtime_fail_closed(); result = PCM_X_QUEUE_COUNTER_EXHAUSTED; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } wait_published = true; gcs_block_pcm_x_requester_cleanup_context.wait_published = true; @@ -8474,11 +8491,11 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (result == PCM_X_QUEUE_CORRUPT) goto requester_fail_closed; if (!cluster_gcs_pcm_x_nested_guard_retryable(result)) - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, cluster_epoch, master_session)) { result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } @@ -8491,11 +8508,11 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (result != PCM_X_QUEUE_OK) { if (result == PCM_X_QUEUE_CORRUPT) goto requester_fail_closed; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } if (!BufferTagsEqual(&initial_own.tag, &buf->tag) || initial_own.generation == UINT64_MAX) { result = PCM_X_QUEUE_STALE; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } live_result = cluster_pcm_own_classify_live_flags(initial_own.flags, initial_own.reservation_token); @@ -8506,17 +8523,17 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, cluster_epoch, master_session)) { result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } continue; } if (result != PCM_X_QUEUE_OK) - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); if (initial_own.pcm_state != (uint8)PCM_STATE_N && initial_own.pcm_state != (uint8)PCM_STATE_S && initial_own.pcm_state != (uint8)PCM_STATE_X) { result = PCM_X_QUEUE_STALE; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } break; } @@ -8544,7 +8561,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, cluster_epoch, master_session)) { result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } gcs_block_pcm_x_requester_cleanup_context.handle = handle; @@ -8569,7 +8586,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, cluster_epoch, master_session)) { result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } for (;;) { @@ -8588,7 +8605,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, cluster_epoch, master_session)) { result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } for (;;) { @@ -8608,7 +8625,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, cluster_epoch, master_session)) { result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } } else if (handle.role == PCM_X_LOCAL_ROLE_FOLLOWER) { @@ -8662,7 +8679,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim master_node, cluster_epoch, master_session)) { result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } continue; } @@ -8739,11 +8756,11 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, cluster_epoch, master_session)) { result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } result = PCM_X_QUEUE_OK; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } else { result = PCM_X_QUEUE_CORRUPT; goto requester_fail_closed; @@ -8967,10 +8984,10 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, cluster_epoch, master_session)) { result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); requester_fail_closed: /* Cleanup may cancel a purely local membership. PREPARE/COMMIT evidence diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index fa045ec977..d635c21d72 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -1115,10 +1115,12 @@ pcm_x_debug_index(Size index) } -/* Diagnostic iterator over live master tag slots. Lock-free and - * racy-tolerant: each slot is copied under a generation double-read and - * formatted only from the stable local copy; slots that mutate mid-copy are - * skipped. Diagnostic surface only — never used for protocol decisions. */ +/* Diagnostic iterator over live master tag slots. Coherence contract: the + * candidate is identified by a racy pre-read, then revalidated and copied + * under the matching master partition shared lock (the same lock every + * in-place tag/ticket field mutation holds), so the printed combination is a + * state that actually existed. Diagnostic surface only — never used for + * protocol decisions. */ bool cluster_pcm_x_master_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, Size buflen) { @@ -1129,21 +1131,32 @@ cluster_pcm_x_master_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, if (header == NULL || cursor_io == NULL || index_out == NULL || buf == NULL || buflen == 0) return false; - if (!pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TAG, &view)) + if (!pcm_x_allocator_entry_unlocked(header) + || !pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TAG, &view)) return false; for (i = *cursor_io; i < view.capacity; i++) { PcmXMasterTagSlot *raw = (PcmXMasterTagSlot *)pcm_x_allocator_slot(&view, i); - uint64 gen_before; - uint64 gen_after; + PcmXMasterTagSlot *locked; + PcmXSlotRef tag_ref; + BufferTag tag; + uint32 partition; if (raw == NULL || pcm_x_slot_state_read(&raw->slot) != PCM_X_TAG_LIVE - || !pcm_x_slot_generation_read(&raw->slot, &gen_before)) + || !pcm_x_slot_generation_read(&raw->slot, &tag_ref.slot_generation)) continue; - memcpy(©, raw, sizeof(copy)); + tag_ref.slot_index = i; + tag = raw->tag; pg_read_barrier(); - if (!pcm_x_slot_generation_read(&raw->slot, &gen_after) || gen_after != gen_before - || pcm_x_slot_state_read(&raw->slot) != PCM_X_TAG_LIVE) + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + locked = (PcmXMasterTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TAG, tag_ref, &tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (locked == NULL) { + LWLockRelease(&header->master_locks[partition].lock); continue; + } + memcpy(©, locked, sizeof(copy)); + LWLockRelease(&header->master_locks[partition].lock); snprintf(buf, buflen, "rel=%u blk=%u head=%ld tail=%ld active=%ld queued=0x%x qseq=" UINT64_FORMAT " outstanding=%zu", @@ -1160,8 +1173,8 @@ cluster_pcm_x_master_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, } -/* Diagnostic iterator over occupied master ticket slots; same racy-tolerant - * contract as the tag iterator. */ +/* Diagnostic iterator over occupied master ticket slots; same partition-lock + * coherence contract as the tag iterator. */ bool cluster_pcm_x_master_ticket_debug_next(Size *cursor_io, Size *index_out, char *buf, Size buflen) { @@ -1172,12 +1185,15 @@ cluster_pcm_x_master_ticket_debug_next(Size *cursor_io, Size *index_out, char *b if (header == NULL || cursor_io == NULL || index_out == NULL || buf == NULL || buflen == 0) return false; - if (!pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TICKET, &view)) + if (!pcm_x_allocator_entry_unlocked(header) + || !pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TICKET, &view)) return false; for (i = *cursor_io; i < view.capacity; i++) { PcmXMasterTicketSlot *raw = (PcmXMasterTicketSlot *)pcm_x_allocator_slot(&view, i); - uint64 gen_before; - uint64 gen_after; + PcmXMasterTicketSlot *locked; + PcmXSlotRef ticket_ref; + BufferTag tag; + uint32 partition; uint32 state; if (raw == NULL) @@ -1185,12 +1201,21 @@ cluster_pcm_x_master_ticket_debug_next(Size *cursor_io, Size *index_out, char *b state = pcm_x_slot_state_read(&raw->slot); if (state == PCM_XT_FREE || state == PCM_XT_RESERVED_NONVISIBLE) continue; - if (!pcm_x_slot_generation_read(&raw->slot, &gen_before)) + if (!pcm_x_slot_generation_read(&raw->slot, &ticket_ref.slot_generation)) continue; - memcpy(©, raw, sizeof(copy)); + ticket_ref.slot_index = i; + tag = raw->ref.identity.tag; pg_read_barrier(); - if (!pcm_x_slot_generation_read(&raw->slot, &gen_after) || gen_after != gen_before) + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + locked = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &tag, PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (locked == NULL) { + LWLockRelease(&header->master_locks[partition].lock); continue; + } + memcpy(©, locked, sizeof(copy)); + LWLockRelease(&header->master_locks[partition].lock); snprintf(buf, buflen, "state=%u flags=0x%x node=%d ticket=" UINT64_FORMAT " grant=" UINT64_FORMAT " tomb=0x%llx involved=0x%x drained=0x%x retire_acked=0x%x pending_s=0x%x" @@ -11620,6 +11645,23 @@ pcm_x_local_nested_wait_guard_tag(const BufferTag *tag) } +/* This backend's most recent nested-guard refusal: the held content-lock tag + * that closed the guard and the refusing result. Diagnostic only. */ +static BufferTag pcm_x_nested_guard_block_tag; +static int pcm_x_nested_guard_block_result = PCM_X_QUEUE_OK; + +bool +cluster_pcm_x_nested_wait_guard_last_block(BufferTag *tag_out, int *result_out) +{ + if (pcm_x_nested_guard_block_result == PCM_X_QUEUE_OK) + return false; + if (tag_out != NULL) + *tag_out = pcm_x_nested_guard_block_tag; + if (result_out != NULL) + *result_out = pcm_x_nested_guard_block_result; + return true; +} + PcmXQueueResult cluster_pcm_x_nested_wait_guard_before_block(void) { @@ -11635,8 +11677,11 @@ cluster_pcm_x_nested_wait_guard_before_block(void) } for (i = 0; i < snapshot.count; i++) { result = pcm_x_local_nested_wait_guard_tag(&snapshot.tags[i]); - if (result != PCM_X_QUEUE_OK) + if (result != PCM_X_QUEUE_OK) { + pcm_x_nested_guard_block_tag = snapshot.tags[i]; + pcm_x_nested_guard_block_result = (int)result; return result; + } } return PCM_X_QUEUE_OK; } diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index b06e340ac6..9b18a8dc2d 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -34,9 +34,9 @@ #include #include "access/tableam.h" -#include "access/xlog.h" /* PGRAC (spec-6.14 D8): GetXLogInsertRecPtr */ +#include "access/xlog.h" /* PGRAC (spec-6.14 D8): GetXLogInsertRecPtr */ #ifdef USE_PGRAC_CLUSTER -#include "access/transam.h" /* FirstNormalObjectId */ +#include "access/transam.h" /* FirstNormalObjectId */ #endif #include "access/xloginsert.h" #include "access/xlogutils.h" @@ -54,26 +54,26 @@ #include "postmaster/bgwriter.h" #include "storage/buf_internals.h" #ifdef USE_PGRAC_CLUSTER -#include "cluster/cluster_catalog_stats.h" /* spec-6.14 D10b — catalog buf hit/miss */ -#include "cluster/cluster_mode.h" /* PGRAC (spec-6.14 D8): storage-mode gate */ +#include "cluster/cluster_catalog_stats.h" /* spec-6.14 D10b — catalog buf hit/miss */ +#include "cluster/cluster_mode.h" /* PGRAC (spec-6.14 D8): storage-mode gate */ #include "cluster/cluster_pcm_direct_init.h" #include "cluster/cluster_gcs_block.h" #include "cluster/cluster_pcm_lock.h" -#include "cluster/cluster_grd.h" /* existing block-path fail-closed counter */ -#include "cluster/cluster_guc.h" /* spec-4.7a D2 — cluster_gcs_block_local_cache */ +#include "cluster/cluster_grd.h" /* existing block-path fail-closed counter */ +#include "cluster/cluster_guc.h" /* spec-4.7a D2 — cluster_gcs_block_local_cache */ #include "cluster/cluster_block_recovery.h" /* spec-4.10 D1 — online block recovery */ #include "cluster/cluster_conf.h" /* spec-5.7 HW — cluster_conf_node_count */ #include "cluster/cluster_hw.h" /* spec-5.7 HW — relation-extend authority */ #include "cluster/cluster_xnode_lever.h" /* spec-6.12h — PI keep counter */ -#include "cluster/cluster_pi_shadow.h" /* spec-6.12h D-h3a — PI ship-SCN stamp */ -#include "cluster/cluster_hw_lease.h" /* spec-6.12d — per-node HW space leases */ -#include "cluster/cluster_extend_gate.h" /* spec-5.7 §3.1d — liveness engage gate */ +#include "cluster/cluster_pi_shadow.h" /* spec-6.12h D-h3a — PI ship-SCN stamp */ +#include "cluster/cluster_hw_lease.h" /* spec-6.12d — per-node HW space leases */ +#include "cluster/cluster_extend_gate.h" /* spec-5.7 §3.1d — liveness engage gate */ #include "cluster/cluster_epoch.h" -#include "cluster/cluster_sf_dep.h" /* spec-6.2 Smart Fusion DBWR brake */ -#include "cluster/cluster_xnode_profile.h" /* spec-5.59 D3/D4 — read probe + relkind hint */ -#include "cluster/cluster_itl.h" /* spec-6.12a — quiescent check for X->S downgrade */ -#include "cluster/cluster_inject.h" /* GCS-race round-4c P1 — yield-notify-drop point */ -#include "cluster/cluster_pcm_own.h" /* ownership-generation wave — per-buffer gen + flags */ +#include "cluster/cluster_sf_dep.h" /* spec-6.2 Smart Fusion DBWR brake */ +#include "cluster/cluster_xnode_profile.h" /* spec-5.59 D3/D4 — read probe + relkind hint */ +#include "cluster/cluster_itl.h" /* spec-6.12a — quiescent check for X->S downgrade */ +#include "cluster/cluster_inject.h" /* GCS-race round-4c P1 — yield-notify-drop point */ +#include "cluster/cluster_pcm_own.h" /* ownership-generation wave — per-buffer gen + flags */ #include "cluster/cluster_pcm_x_bufmgr.h" /* spec-2.36a C1 opaque reservation API */ #include "cluster/cluster_pcm_x_convert.h" @@ -103,18 +103,17 @@ extern bool ignore_checksum_failure; /* Note: these two macros only work on shared buffers, not local ones! */ -#define BufHdrGetBlock(bufHdr) ((Block) (BufferBlocks + ((Size) (bufHdr)->buf_id) * BLCKSZ)) -#define BufferGetLSN(bufHdr) (PageGetLSN(BufHdrGetBlock(bufHdr))) +#define BufHdrGetBlock(bufHdr) ((Block)(BufferBlocks + ((Size)(bufHdr)->buf_id) * BLCKSZ)) +#define BufferGetLSN(bufHdr) (PageGetLSN(BufHdrGetBlock(bufHdr))) /* Note: this macro only works on local buffers, not shared ones! */ -#define LocalBufHdrGetBlock(bufHdr) \ - LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)] +#define LocalBufHdrGetBlock(bufHdr) LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)] /* Bits in SyncOneBuffer's return value */ -#define BUF_WRITTEN 0x01 -#define BUF_REUSABLE 0x02 +#define BUF_WRITTEN 0x01 +#define BUF_REUSABLE 0x02 -#define RELS_BSEARCH_THRESHOLD 20 +#define RELS_BSEARCH_THRESHOLD 20 #ifdef USE_PGRAC_CLUSTER /* @@ -135,7 +134,7 @@ extern bool ignore_checksum_failure; static inline bool cluster_bufmgr_reln_pcm_tracked(RelFileNumber relnum) { - return cluster_shared_catalog || relnum >= (RelFileNumber) FirstNormalObjectId; + return cluster_shared_catalog || relnum >= (RelFileNumber)FirstNormalObjectId; } static inline bool @@ -173,33 +172,33 @@ cluster_pcm_own_bump_failure(BufferDesc *buf, uint64 generation, uint32 *out_fla } static void -cluster_pcm_own_report_bump_failure(BufferDesc *buf, ClusterPcmOwnResult result, - uint64 generation, uint32 flags, const char *context) +cluster_pcm_own_report_bump_failure(BufferDesc *buf, ClusterPcmOwnResult result, uint64 generation, + uint32 flags, const char *context) { if (result == CLUSTER_PCM_OWN_BUSY) ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), errmsg("cluster PCM ownership transition conflicts with an active reservation"), - errdetail("context=%s buffer=%d generation=%llu flags=0x%x", context, - buf->buf_id, (unsigned long long) generation, flags))); + errdetail("context=%s buffer=%d generation=%llu flags=0x%x", context, buf->buf_id, + (unsigned long long)generation, flags))); if (result == CLUSTER_PCM_OWN_EXHAUSTED) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("cluster PCM ownership generation exhausted for buffer %d", buf->buf_id), errdetail("context=%s generation=%llu flags=0x%x", context, - (unsigned long long) generation, flags))); + (unsigned long long)generation, flags))); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("cluster PCM ownership generation bump failed for buffer %d", buf->buf_id), errdetail("context=%s generation=%llu flags=0x%x result=%d", context, - (unsigned long long) generation, flags, (int) result))); + (unsigned long long)generation, flags, (int)result))); } -static inline ClusterPcmOwnResult cluster_pcm_own_bump_locked(BufferDesc *buf, - uint32 set_flags, uint32 clear_flags, - uint64 *out_generation, - uint32 *out_flags); +static inline ClusterPcmOwnResult cluster_pcm_own_bump_locked(BufferDesc *buf, uint32 set_flags, + uint32 clear_flags, + uint64 *out_generation, + uint32 *out_flags); /* * PGRAC ownership-generation wave — coherent @@ -227,12 +226,12 @@ cluster_pcm_own_transition(BufferDesc *buf, uint8 new_pcm_state, uint32 set_flag uint64 new_generation; buf_state = LockBufHdr(buf); - bump_result = cluster_pcm_own_bump_locked(buf, set_flags, clear_flags, - &new_generation, &observed_flags); + bump_result = cluster_pcm_own_bump_locked(buf, set_flags, clear_flags, &new_generation, + &observed_flags); if (bump_result != CLUSTER_PCM_OWN_OK) { UnlockBufHdr(buf, buf_state); - cluster_pcm_own_report_bump_failure(buf, bump_result, new_generation, - observed_flags, "ownership transition"); + cluster_pcm_own_report_bump_failure(buf, bump_result, new_generation, observed_flags, + "ownership transition"); } buf->pcm_state = new_pcm_state; UnlockBufHdr(buf, buf_state); @@ -285,8 +284,7 @@ cluster_pcm_own_snapshot_locked(BufferDesc *buf, ClusterPcmOwnSnapshot *out) static inline bool cluster_bufmgr_pcm_x_retained_image_locked(BufferDesc *buf, uint32 buf_state) { - return buf != NULL && buf->buffer_type == (uint8) BUF_TYPE_PI - && (buf_state & BM_VALID) != 0; + return buf != NULL && buf->buffer_type == (uint8)BUF_TYPE_PI && (buf_state & BM_VALID) != 0; } /* A resident mapping is GCS-shippable only while its node owns matching PCM @@ -296,13 +294,12 @@ static inline bool cluster_bufmgr_pcm_current_image_locked(BufferDesc *buf, uint32 buf_state) { return buf != NULL - && cluster_pcm_x_current_image_shape(buf->pcm_state, buf->buffer_type, - (buf_state & BM_VALID) != 0); + && cluster_pcm_x_current_image_shape(buf->pcm_state, buf->buffer_type, + (buf_state & BM_VALID) != 0); } static inline bool -cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(BufferDesc *buf, - uint32 buf_state) +cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(BufferDesc *buf, uint32 buf_state) { uint32 flags; @@ -315,14 +312,13 @@ cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(BufferDesc *buf, * material. Its monotonic token may legitimately still be zero before * the first reservation, so flags -- not token value -- are the active * lifecycle authority. Every live or malformed shape remains blocked. */ - return buf->pcm_state != (uint8) PCM_STATE_N - || (buf_state - & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR)) != 0 - || flags != 0; + return buf->pcm_state != (uint8)PCM_STATE_N + || (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR)) != 0 + || flags != 0; } -static inline bool cluster_pcm_own_snapshot_matches_locked( - BufferDesc *buf, const ClusterPcmOwnSnapshot *expected); +static inline bool cluster_pcm_own_snapshot_matches_locked(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected); static inline void cluster_pcm_own_eviction_capture_locked(BufferDesc *buf, ClusterPcmOwnEvictionCapture *out) @@ -331,8 +327,7 @@ cluster_pcm_own_eviction_capture_locked(BufferDesc *buf, ClusterPcmOwnEvictionCa } static ClusterPcmOwnResult -cluster_pcm_own_eviction_commit_locked(BufferDesc *buf, - const ClusterPcmOwnEvictionCapture *capture, +cluster_pcm_own_eviction_commit_locked(BufferDesc *buf, const ClusterPcmOwnEvictionCapture *capture, uint64 *out_generation, uint32 *out_flags) { ClusterPcmOwnResult result; @@ -340,8 +335,7 @@ cluster_pcm_own_eviction_commit_locked(BufferDesc *buf, if (!cluster_pcm_own_snapshot_matches_locked(buf, capture)) return CLUSTER_PCM_OWN_STALE; if (!cluster_pcm_own_eviction_reuse_allowed(capture)) { - result = cluster_pcm_own_classify_live_flags(capture->flags, - capture->reservation_token); + result = cluster_pcm_own_classify_live_flags(capture->flags, capture->reservation_token); return result != CLUSTER_PCM_OWN_OK ? result : CLUSTER_PCM_OWN_EXHAUSTED; } @@ -366,7 +360,7 @@ cluster_pcm_own_snapshot_matches_locked(BufferDesc *buf, const ClusterPcmOwnSnap ClusterPcmOwnResult cluster_bufmgr_pcm_own_snapshot(BufferDesc *buf, ClusterPcmOwnSnapshot *out_snapshot) { - uint32 buf_state; + uint32 buf_state; if (buf == NULL || out_snapshot == NULL) return CLUSTER_PCM_OWN_INVALID; @@ -382,17 +376,17 @@ cluster_bufmgr_pcm_own_snapshot(BufferDesc *buf, ClusterPcmOwnSnapshot *out_snap bool cluster_bufmgr_pcm_x_content_write_permitted(BufferDesc *buf) { - bool permitted; - uint32 buf_state; - uint32 flags; + bool permitted; + uint32 buf_state; + uint32 flags; if (buf == NULL) return false; buf_state = LockBufHdr(buf); flags = cluster_pcm_own_flags_get(buf->buf_id); permitted = (flags & PCM_OWN_FLAG_REVOKING) == 0 - && (!cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state) - || flags == PCM_OWN_FLAG_GRANT_PENDING); + && (!cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state) + || flags == PCM_OWN_FLAG_GRANT_PENDING); UnlockBufHdr(buf, buf_state); return permitted; } @@ -402,11 +396,11 @@ cluster_bufmgr_pcm_own_snapshot_by_tag(const BufferTag *tag, int *out_buffer_id, ClusterPcmOwnSnapshot *out_snapshot) { BufferDesc *buf; - BufferTag lookup_tag; - LWLock *partition_lock; - uint32 hashcode; - uint32 buf_state; - int buf_id; + BufferTag lookup_tag; + LWLock *partition_lock; + uint32 hashcode; + uint32 buf_state; + int buf_id; ClusterPcmOwnResult result; if (out_buffer_id != NULL) @@ -423,8 +417,7 @@ cluster_bufmgr_pcm_own_snapshot_by_tag(const BufferTag *tag, int *out_buffer_id, partition_lock = BufMappingPartitionLock(hashcode); LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&lookup_tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); return CLUSTER_PCM_OWN_STALE; } @@ -432,8 +425,7 @@ cluster_bufmgr_pcm_own_snapshot_by_tag(const BufferTag *tag, int *out_buffer_id, buf_state = LockBufHdr(buf); if (!BufferTagsEqual(&buf->tag, tag) || (buf_state & BM_VALID) == 0) result = CLUSTER_PCM_OWN_STALE; - else - { + else { cluster_pcm_own_snapshot_locked(buf, out_snapshot); *out_buffer_id = buf_id; result = CLUSTER_PCM_OWN_OK; @@ -454,31 +446,29 @@ cluster_bufmgr_pcm_own_snapshot_by_tag(const BufferTag *tag, int *out_buffer_id, * present. Queue JOIN/WAIT never sets a reservation flag here. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_finish_s_release_to_n( - BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_s, - ClusterPcmOwnSnapshot *out_n_snapshot) +cluster_bufmgr_pcm_own_finish_s_release_to_n(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected_s, + ClusterPcmOwnSnapshot *out_n_snapshot) { ClusterPcmOwnResult result; - uint32 buf_state; + uint32 buf_state; if (buf == NULL || expected_s == NULL || out_n_snapshot == NULL) return CLUSTER_PCM_OWN_INVALID; memset(out_n_snapshot, 0, sizeof(*out_n_snapshot)); if (ClusterPcmOwnArray == NULL) return CLUSTER_PCM_OWN_NOT_READY; - if (expected_s->pcm_state != (uint8) PCM_STATE_S || expected_s->flags != 0) + if (expected_s->pcm_state != (uint8)PCM_STATE_S || expected_s->flags != 0) return CLUSTER_PCM_OWN_STALE; buf_state = LockBufHdr(buf); if (!cluster_pcm_own_snapshot_matches_locked(buf, expected_s)) result = CLUSTER_PCM_OWN_STALE; - else - { + else { result = cluster_pcm_own_bump_locked(buf, 0, 0, NULL, NULL); - if (result == CLUSTER_PCM_OWN_OK) - { - buf->pcm_state = (uint8) PCM_STATE_N; - buf->buffer_type = (uint8) BUF_TYPE_CURRENT; + if (result == CLUSTER_PCM_OWN_OK) { + buf->pcm_state = (uint8)PCM_STATE_N; + buf->buffer_type = (uint8)BUF_TYPE_CURRENT; cluster_pcm_own_snapshot_locked(buf, out_n_snapshot); } } @@ -500,8 +490,7 @@ cluster_bufmgr_pcm_own_finish_s_release_to_n( * pin-only readers may consume bytes without that content-lock gate. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, - XLogRecPtr *out_page_lsn, +cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, XLogRecPtr *out_page_lsn, uint64 *out_page_scn) { BufferTag lookup_tag; @@ -537,8 +526,7 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, partition_lock = BufMappingPartitionLock(hashcode); LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&lookup_tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); return CLUSTER_PCM_OWN_STALE; } @@ -546,15 +534,14 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, buf_state = LockBufHdr(buf); if (!BufferTagsEqual(&buf->tag, tag) || (buf_state & BM_VALID) == 0) result = CLUSTER_PCM_OWN_STALE; - else - { + else { shared_refcount = BUF_STATE_GET_REFCOUNT(buf_state); token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, token); if (live_result == CLUSTER_PCM_OWN_CORRUPT) result = live_result; - else if (buf->pcm_state != (uint8) PCM_STATE_S) + else if (buf->pcm_state != (uint8)PCM_STATE_S) result = CLUSTER_PCM_OWN_STALE; else if (flags != 0) result = CLUSTER_PCM_OWN_BUSY; @@ -567,8 +554,7 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, else if (cluster_pcm_x_revoke_finish_mode(tag, shared_refcount) != CLUSTER_PCM_X_REVOKE_FINISH_RETAIN) result = CLUSTER_PCM_OWN_BUSY; - else - { + else { cluster_pcm_own_snapshot_locked(buf, &expected_s); cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); buf_state = 0; @@ -591,11 +577,10 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state) || (buf_state & BM_IO_IN_PROGRESS) != 0) result = CLUSTER_PCM_OWN_STALE; - else - { - page = (Page) BufHdrGetBlock(buf); + else { + page = (Page)BufHdrGetBlock(buf); page_lsn = PageGetLSN(page); - page_scn = (uint64) ((PageHeader) page)->pd_block_scn; + page_scn = (uint64)((PageHeader)page)->pd_block_scn; dirty = (buf_state & BM_DIRTY) != 0; } UnlockBufHdr(buf, buf_state); @@ -603,29 +588,24 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, if (result == CLUSTER_PCM_OWN_OK && dirty && !XLogRecPtrIsInvalid(page_lsn)) XLogFlush(cluster_gcs_clamp_ship_flush_lsn(page_lsn)); - if (result == CLUSTER_PCM_OWN_OK) - { + if (result == CLUSTER_PCM_OWN_OK) { buf_state = LockBufHdr(buf); - page = (Page) BufHdrGetBlock(buf); + page = (Page)BufHdrGetBlock(buf); if (!cluster_pcm_own_snapshot_matches_locked(buf, &expected_s) || (buf_state & BM_VALID) == 0 || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state) - || (buf_state & BM_IO_IN_PROGRESS) != 0 - || PageGetLSN(page) != page_lsn) + || (buf_state & BM_IO_IN_PROGRESS) != 0 || PageGetLSN(page) != page_lsn) result = CLUSTER_PCM_OWN_STALE; - else - { + else { result = cluster_pcm_own_bump_locked(buf, 0, 0, NULL, NULL); - if (result == CLUSTER_PCM_OWN_OK) - { - buf->pcm_state = (uint8) PCM_STATE_N; + if (result == CLUSTER_PCM_OWN_OK) { + buf->pcm_state = (uint8)PCM_STATE_N; /* PI+BM_VALID is the established never-write/never-serve shape. * Unlike a source retain it has no live reservation flag (the * monotonic token remains idle): the next exact GRANT_PENDING * install may overwrite it, then republishes CURRENT. */ - buf->buffer_type = (uint8) BUF_TYPE_PI; - buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED - | BM_IO_ERROR); + buf->buffer_type = (uint8)BUF_TYPE_PI; + buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR); } } UnlockBufHdr(buf, buf_state); @@ -644,8 +624,7 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, PG_END_TRY(); cluster_bufmgr_unpin_for_gcs(buf); - if (result == CLUSTER_PCM_OWN_OK) - { + if (result == CLUSTER_PCM_OWN_OK) { *out_page_lsn = page_lsn; *out_page_scn = page_scn; } @@ -655,8 +634,9 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, /* Make freshly installed bytes visible only while the same queue * reservation still owns both the descriptor and content EXCLUSIVE. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_publish_installed_x_image( - BufferDesc *buf, const ClusterPcmOwnSnapshot *expected, uint64 reservation_token) +cluster_bufmgr_pcm_own_publish_installed_x_image(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected, + uint64 reservation_token) { ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; uint32 buf_state; @@ -667,7 +647,7 @@ cluster_bufmgr_pcm_own_publish_installed_x_image( return CLUSTER_PCM_OWN_INVALID; if (ClusterPcmOwnArray == NULL) return CLUSTER_PCM_OWN_NOT_READY; - if (expected->pcm_state != (uint8) PCM_STATE_N || expected->flags != 0 + if (expected->pcm_state != (uint8)PCM_STATE_N || expected->flags != 0 || expected->reservation_token == UINT64_MAX || reservation_token != expected->reservation_token + 1) return CLUSTER_PCM_OWN_STALE; @@ -677,12 +657,11 @@ cluster_bufmgr_pcm_own_publish_installed_x_image( || cluster_pcm_own_gen_get(buf->buf_id) != expected->generation || cluster_pcm_own_reservation_token_get(buf->buf_id) != reservation_token || cluster_pcm_own_flags_get(buf->buf_id) != PCM_OWN_FLAG_GRANT_PENDING - || buf->pcm_state != (uint8) PCM_STATE_N) + || buf->pcm_state != (uint8)PCM_STATE_N) result = CLUSTER_PCM_OWN_STALE; - else - { + else { buf_state |= BM_VALID; - buf->buffer_type = (uint8) BUF_TYPE_CURRENT; + buf->buffer_type = (uint8)BUF_TYPE_CURRENT; } UnlockBufHdr(buf, buf_state); return result; @@ -985,8 +964,7 @@ cluster_pcm_own_abort_grant_after_master_rollback(BufferDesc *buf, return result; } -pg_attribute_noreturn() static void -cluster_pcm_own_rollback_grant_after_error_and_rethrow( +pg_attribute_noreturn() static void cluster_pcm_own_rollback_grant_after_error_and_rethrow( BufferDesc *buf, const ClusterPcmOwnSnapshot *base, uint64 reservation_token, PcmLockMode acquired_mode, bool has_reservation, ErrorData *original_error, MemoryContext caller_context) @@ -1025,15 +1003,15 @@ cluster_pcm_own_rollback_grant_after_error_and_rethrow( if (master_released) { if (has_reservation) { - rollback_result = cluster_pcm_own_abort_grant_after_master_rollback( - buf, base, reservation_token); + rollback_result + = cluster_pcm_own_abort_grant_after_master_rollback(buf, base, reservation_token); if (rollback_result != CLUSTER_PCM_OWN_OK) elog(LOG, "failed to converge local cluster PCM state after content-lock error " "master rollback: buffer=%d token=%llu result=%d; reservation remains " "fail-closed", - buf != NULL ? buf->buf_id : -1, - (unsigned long long)reservation_token, (int)rollback_result); + buf != NULL ? buf->buf_id : -1, (unsigned long long)reservation_token, + (int)rollback_result); } else elog(LOG, "cluster PCM content-lock error rolled back a master grant without local " @@ -1127,7 +1105,7 @@ static bool cluster_bufmgr_in_gcs_drop = false; static inline void cluster_pcm_own_read(BufferDesc *buf, uint8 *out_state, uint64 *out_gen, uint32 *out_flags) { - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf); if (out_state != NULL) @@ -1143,8 +1121,7 @@ cluster_pcm_own_read(BufferDesc *buf, uint8 *out_state, uint64 *out_gen, uint32 * bounded by PG's own held-LWLock ceiling and stores the complete exact * handle returned by PCM-X; error cleanup never reconstructs identity from a * possibly changed BufferDesc ownership mirror. */ -typedef enum ClusterPcmXHolderLedgerPhase -{ +typedef enum ClusterPcmXHolderLedgerPhase { PCM_X_HOLDER_LEDGER_UNUSED = 0, PCM_X_HOLDER_LEDGER_ACQUIRING, PCM_X_HOLDER_LEDGER_ACTIVE, @@ -1152,16 +1129,14 @@ typedef enum ClusterPcmXHolderLedgerPhase PCM_X_HOLDER_LEDGER_DEFERRED } ClusterPcmXHolderLedgerPhase; -typedef struct ClusterPcmXHolderLedgerEntry -{ +typedef struct ClusterPcmXHolderLedgerEntry { ClusterPcmXHolderLedgerPhase phase; - int32 buffer_id; - LWLock *content_lock; + int32 buffer_id; + LWLock *content_lock; PcmXLocalHolderHandle handle; } ClusterPcmXHolderLedgerEntry; -static ClusterPcmXHolderLedgerEntry - cluster_bufmgr_pcm_x_holder_ledger[LWLOCK_MAX_HELD_BY_PROC]; +static ClusterPcmXHolderLedgerEntry cluster_bufmgr_pcm_x_holder_ledger[LWLOCK_MAX_HELD_BY_PROC]; static uint64 cluster_bufmgr_pcm_x_holder_identity = 0; StaticAssertDecl(lengthof(cluster_bufmgr_pcm_x_holder_ledger) == LWLOCK_MAX_HELD_BY_PROC, @@ -1208,10 +1183,9 @@ static void cluster_bufmgr_pcm_direct_init_snapshot_locked(BufferDesc *buf, uint ClusterPcmDirectInitSnapshot *out); static void -cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, - uint32 wait_index) +cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, uint32 wait_index) { - long delay_ms; + long delay_ms; PcmXQueueResult guard_result; /* @@ -1219,10 +1193,11 @@ cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, * lock is held would make RETIRE depend on the holder it is draining. */ if (content_lock == NULL || LWLockHeldByMe(content_lock)) - ereport(ERROR, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("cannot wait for a cluster PCM-X holder gate while holding content authority"), - errdetail("buffer=%d wait_index=%u", buffer_id, wait_index))); + ereport( + ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cannot wait for a cluster PCM-X holder gate while holding content authority"), + errdetail("buffer=%d wait_index=%u", buffer_id, wait_index))); /* A different held content lock may itself be the holder that a frozen * PROBE is draining. Reuse the protocol's exact held-lock snapshot so a * safe lock-coupling path can still wait, but never sleep across a closed @@ -1230,24 +1205,23 @@ cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, */ guard_result = cluster_pcm_x_nested_wait_guard_before_block(); if (guard_result != PCM_X_QUEUE_OK) - cluster_bufmgr_pcm_x_holder_report_failure( - guard_result, GetBufferDescriptor(buffer_id), "retry wait nested guard"); + cluster_bufmgr_pcm_x_holder_report_failure(guard_result, GetBufferDescriptor(buffer_id), + "retry wait nested guard"); delay_ms = cluster_pcm_x_holder_retry_delay_ms(wait_index); CHECK_FOR_INTERRUPTS(); - (void) WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, delay_ms, - WAIT_EVENT_PCM_BLOCK_CONVERT_WAIT); + (void)WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, delay_ms, + WAIT_EVENT_PCM_BLOCK_CONVERT_WAIT); CHECK_FOR_INTERRUPTS(); } static ClusterPcmXHolderLedgerEntry * cluster_bufmgr_pcm_x_holder_find(BufferDesc *buf) { - int i; + int i; if (buf == NULL) return NULL; - for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) - { + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) { ClusterPcmXHolderLedgerEntry *entry = &cluster_bufmgr_pcm_x_holder_ledger[i]; if (entry->phase != PCM_X_HOLDER_LEDGER_UNUSED && entry->buffer_id == buf->buf_id) @@ -1259,7 +1233,7 @@ cluster_bufmgr_pcm_x_holder_find(BufferDesc *buf) static ClusterPcmXHolderLedgerEntry * cluster_bufmgr_pcm_x_holder_free_entry(void) { - int i; + int i; for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) if (cluster_bufmgr_pcm_x_holder_ledger[i].phase == PCM_X_HOLDER_LEDGER_UNUSED) @@ -1268,11 +1242,10 @@ cluster_bufmgr_pcm_x_holder_free_entry(void) } static bool -cluster_bufmgr_pcm_x_holder_entry_exact(const ClusterPcmXHolderLedgerEntry *entry, - BufferDesc *buf) +cluster_bufmgr_pcm_x_holder_entry_exact(const ClusterPcmXHolderLedgerEntry *entry, BufferDesc *buf) { PcmXRuntimeSnapshot runtime; - uint64 cluster_epoch; + uint64 cluster_epoch; if (entry == NULL || buf == NULL) return false; @@ -1283,8 +1256,7 @@ cluster_bufmgr_pcm_x_holder_entry_exact(const ClusterPcmXHolderLedgerEntry *entr return false; if (cluster_node_id < 0 || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT || entry->handle.key.identity.node_id != cluster_node_id || MyProc == NULL - || MyProc->pgprocno < 0 - || entry->handle.key.identity.procno != (uint32) MyProc->pgprocno + || MyProc->pgprocno < 0 || entry->handle.key.identity.procno != (uint32)MyProc->pgprocno || entry->handle.key.identity.request_id == 0 || entry->handle.key.identity.wait_seq != entry->handle.key.identity.request_id || entry->handle.tag_slot.slot_index == PCM_X_INVALID_SLOT_INDEX @@ -1300,22 +1272,20 @@ cluster_bufmgr_pcm_x_holder_entry_exact(const ClusterPcmXHolderLedgerEntry *entr static void cluster_bufmgr_pcm_x_holder_report_failure(PcmXQueueResult result, BufferDesc *buf, - const char *operation) + const char *operation) { if (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY || result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BARRIER_CLOSED || result == PCM_X_QUEUE_NO_CAPACITY) - ereport(ERROR, - (errcode(ERRCODE_OBJECT_IN_USE), - errmsg("cluster PCM-X holder operation is not ready"), - errdetail("operation=%s buffer=%d result=%d", operation, - buf != NULL ? buf->buf_id : -1, (int) result))); + ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), + errmsg("cluster PCM-X holder operation is not ready"), + errdetail("operation=%s buffer=%d result=%d", operation, + buf != NULL ? buf->buf_id : -1, (int)result))); ereport(ERROR, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("cluster PCM-X holder operation failed"), + (errcode(ERRCODE_DATA_CORRUPTED), errmsg("cluster PCM-X holder operation failed"), errdetail("operation=%s buffer=%d result=%d", operation, - buf != NULL ? buf->buf_id : -1, (int) result))); + buf != NULL ? buf->buf_id : -1, (int)result))); } static void @@ -1341,37 +1311,36 @@ cluster_bufmgr_pcm_x_holder_defer_fail_closed(ClusterPcmXHolderLedgerEntry *entr static void cluster_bufmgr_pcm_x_holder_drain_deferred_nowait(void) { - int i; + int i; - for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) - { + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) { ClusterPcmXHolderLedgerEntry *entry = &cluster_bufmgr_pcm_x_holder_ledger[i]; PcmXQueueResult result; if (entry->phase != PCM_X_HOLDER_LEDGER_DEFERRED) continue; - if (entry->content_lock == NULL) - { + if (entry->content_lock == NULL) { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); - elog(LOG, "could not drain deferred cluster PCM-X holder with no content lock: buffer=%d", + elog(LOG, + "could not drain deferred cluster PCM-X holder with no content lock: buffer=%d", entry->buffer_id); continue; } - if (LWLockHeldByMe(entry->content_lock)) - { - elog(LOG, "could not drain deferred cluster PCM-X holder while content lock is held: buffer=%d", + if (LWLockHeldByMe(entry->content_lock)) { + elog(LOG, + "could not drain deferred cluster PCM-X holder while content lock is held: " + "buffer=%d", entry->buffer_id); continue; } result = cluster_pcm_x_local_holder_exceptional_detach_exact(&entry->handle, - entry->content_lock); + entry->content_lock); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_NOT_FOUND) cluster_bufmgr_pcm_x_holder_clear(entry); - else if (result != PCM_X_QUEUE_GATE_RETRY && result != PCM_X_QUEUE_BUSY) - { + else if (result != PCM_X_QUEUE_GATE_RETRY && result != PCM_X_QUEUE_BUSY) { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); elog(LOG, "could not drain deferred exact cluster PCM-X holder: buffer=%d result=%d", - entry->buffer_id, (int) result); + entry->buffer_id, (int)result); } } } @@ -1384,51 +1353,42 @@ static void cluster_bufmgr_pcm_x_holder_drain_deferred(ClusterPcmXHolderLedgerEntry *entry) { PcmXRuntimeSnapshot runtime; - uint32 wait_index = 0; + uint32 wait_index = 0; if (entry == NULL || entry->phase != PCM_X_HOLDER_LEDGER_DEFERRED) return; - for (;;) - { + for (;;) { PcmXQueueResult result; ClusterPcmXHolderRetryAction action; - if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)) - { + if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)) { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, - GetBufferDescriptor(entry->buffer_id), - "deferred detach lock order"); + GetBufferDescriptor(entry->buffer_id), + "deferred detach lock order"); } result = cluster_pcm_x_local_holder_exceptional_detach_exact(&entry->handle, - entry->content_lock); + entry->content_lock); action = cluster_pcm_x_holder_unregister_retry_action(result, 0); - if (action == CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE) - { + if (action == CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE) { cluster_bufmgr_pcm_x_holder_clear(entry); return; } - if (action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT) - { + if (action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT) { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); - cluster_bufmgr_pcm_x_holder_report_failure(result, - GetBufferDescriptor(entry->buffer_id), - "deferred detach"); + cluster_bufmgr_pcm_x_holder_report_failure( + result, GetBufferDescriptor(entry->buffer_id), "deferred detach"); } - cluster_bufmgr_pcm_x_holder_retry_wait( - entry->content_lock, entry->buffer_id, - wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); + cluster_bufmgr_pcm_x_holder_retry_wait(entry->content_lock, entry->buffer_id, + wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); wait_index++; - if (wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS == 0) - { + if (wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS == 0) { runtime = cluster_pcm_x_runtime_snapshot(); - if (runtime.state != PCM_X_RUNTIME_ACTIVE - || runtime.master_session_incarnation == 0) - { + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); - cluster_bufmgr_pcm_x_holder_report_failure( - PCM_X_QUEUE_NOT_READY, GetBufferDescriptor(entry->buffer_id), - "deferred detach runtime"); + cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_NOT_READY, + GetBufferDescriptor(entry->buffer_id), + "deferred detach runtime"); } } } @@ -1454,19 +1414,16 @@ cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf) cluster_bufmgr_pcm_x_holder_drain_deferred_nowait(); entry = cluster_bufmgr_pcm_x_holder_find(buf); - if (entry != NULL && entry->phase == PCM_X_HOLDER_LEDGER_DEFERRED) - { + if (entry != NULL && entry->phase == PCM_X_HOLDER_LEDGER_DEFERRED) { cluster_bufmgr_pcm_x_holder_drain_deferred(entry); entry = NULL; } - if (entry != NULL) - { + if (entry != NULL) { if (entry->phase != PCM_X_HOLDER_LEDGER_ACQUIRING - || !cluster_bufmgr_pcm_x_holder_entry_exact(entry, buf)) - { + || !cluster_bufmgr_pcm_x_holder_entry_exact(entry, buf)) { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, buf, - "reuse existing holder ledger"); + "reuse existing holder ledger"); } if (LWLockHeldByMe(entry->content_lock)) ereport(ERROR, @@ -1506,18 +1463,15 @@ cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf) errmsg("cannot register a cluster PCM-X holder for a pre-held content lock"), errdetail("buffer=%d", buf->buf_id))); entry = cluster_bufmgr_pcm_x_holder_free_entry(); - if (entry == NULL) - { - int i; + if (entry == NULL) { + int i; /* * Deferred exact handles do not consume the ledger forever. If every * slot is occupied, wait for one safe detach before declaring a real * held-LWLock bound violation. */ - for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) - { - ClusterPcmXHolderLedgerEntry *candidate - = &cluster_bufmgr_pcm_x_holder_ledger[i]; + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) { + ClusterPcmXHolderLedgerEntry *candidate = &cluster_bufmgr_pcm_x_holder_ledger[i]; if (candidate->phase != PCM_X_HOLDER_LEDGER_DEFERRED) continue; @@ -1527,10 +1481,9 @@ cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf) } } if (entry == NULL) - ereport(ERROR, - (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg("cluster PCM-X holder ledger is full"), - errdetail("maximum entries=%d", LWLOCK_MAX_HELD_BY_PROC))); + ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("cluster PCM-X holder ledger is full"), + errdetail("maximum entries=%d", LWLOCK_MAX_HELD_BY_PROC))); cluster_pcm_own_read(buf, NULL, &own_generation, NULL); cluster_epoch = cluster_epoch_get_current(); @@ -1562,8 +1515,7 @@ cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf) key.identity.base_own_generation = own_generation; } key.buffer_id = buf->buf_id; - for (;;) - { + for (;;) { ClusterPcmXHolderRetryAction action; PcmXRuntimeSnapshot current_runtime; @@ -1595,8 +1547,7 @@ cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf) if (action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT) cluster_bufmgr_pcm_x_holder_report_failure(result, buf, "register"); cluster_bufmgr_pcm_x_holder_retry_wait( - content_lock, buf->buf_id, - wait_index++ % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); + content_lock, buf->buf_id, wait_index++ % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); } return entry; @@ -1610,12 +1561,12 @@ cluster_bufmgr_pcm_x_holder_activate(ClusterPcmXHolderLedgerEntry *entry) if (entry == NULL) return; if (entry->phase != PCM_X_HOLDER_LEDGER_ACQUIRING) - cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, - GetBufferDescriptor(entry->buffer_id), "activate phase"); + cluster_bufmgr_pcm_x_holder_report_failure( + PCM_X_QUEUE_BAD_STATE, GetBufferDescriptor(entry->buffer_id), "activate phase"); result = cluster_pcm_x_local_holder_activate_exact(&entry->handle); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) - cluster_bufmgr_pcm_x_holder_report_failure(result, - GetBufferDescriptor(entry->buffer_id), "activate"); + cluster_bufmgr_pcm_x_holder_report_failure(result, GetBufferDescriptor(entry->buffer_id), + "activate"); entry->phase = PCM_X_HOLDER_LEDGER_ACTIVE; } @@ -1627,13 +1578,12 @@ cluster_bufmgr_pcm_x_holder_mark_releasing(ClusterPcmXHolderLedgerEntry *entry) if (entry == NULL) return; if (entry->phase != PCM_X_HOLDER_LEDGER_ACTIVE) - cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, - GetBufferDescriptor(entry->buffer_id), - "mark releasing phase"); + cluster_bufmgr_pcm_x_holder_report_failure( + PCM_X_QUEUE_BAD_STATE, GetBufferDescriptor(entry->buffer_id), "mark releasing phase"); result = cluster_pcm_x_local_holder_mark_releasing_exact(&entry->handle); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) - cluster_bufmgr_pcm_x_holder_report_failure(result, - GetBufferDescriptor(entry->buffer_id), "mark releasing"); + cluster_bufmgr_pcm_x_holder_report_failure(result, GetBufferDescriptor(entry->buffer_id), + "mark releasing"); entry->phase = PCM_X_HOLDER_LEDGER_RELEASING; } @@ -1641,42 +1591,35 @@ static void cluster_bufmgr_pcm_x_holder_unregister(ClusterPcmXHolderLedgerEntry *entry) { PcmXQueueResult result; - uint32 waits_used = 0; + uint32 waits_used = 0; if (entry == NULL) return; if (entry->phase != PCM_X_HOLDER_LEDGER_RELEASING) - cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, - GetBufferDescriptor(entry->buffer_id), - "unregister phase"); + cluster_bufmgr_pcm_x_holder_report_failure( + PCM_X_QUEUE_BAD_STATE, GetBufferDescriptor(entry->buffer_id), "unregister phase"); if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)) - cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, - GetBufferDescriptor(entry->buffer_id), - "unregister lock order"); - for (;;) - { + cluster_bufmgr_pcm_x_holder_report_failure( + PCM_X_QUEUE_BAD_STATE, GetBufferDescriptor(entry->buffer_id), "unregister lock order"); + for (;;) { ClusterPcmXHolderRetryAction action; result = cluster_pcm_x_local_holder_unregister_exact(&entry->handle); action = cluster_pcm_x_holder_unregister_retry_action(result, waits_used); - if (action == CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE) - { + if (action == CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE) { cluster_bufmgr_pcm_x_holder_clear(entry); return; } - if (action == CLUSTER_PCM_X_HOLDER_RETRY_DEFER) - { + if (action == CLUSTER_PCM_X_HOLDER_RETRY_DEFER) { entry->phase = PCM_X_HOLDER_LEDGER_DEFERRED; return; } - if (action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT) - { + if (action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT) { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); cluster_bufmgr_pcm_x_holder_report_failure( result, GetBufferDescriptor(entry->buffer_id), "unregister"); } - cluster_bufmgr_pcm_x_holder_retry_wait(entry->content_lock, entry->buffer_id, - waits_used++); + cluster_bufmgr_pcm_x_holder_retry_wait(entry->content_lock, entry->buffer_id, waits_used++); } } @@ -1687,8 +1630,7 @@ cluster_bufmgr_pcm_x_holder_abort_acquiring(ClusterPcmXHolderLedgerEntry *entry) if (entry == NULL) return; - if (entry->content_lock == NULL) - { + if (entry->content_lock == NULL) { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); elog(LOG, "could not abort cluster PCM-X ACQUIRING holder with no content lock: buffer=%d", entry->buffer_id); @@ -1701,46 +1643,45 @@ cluster_bufmgr_pcm_x_holder_abort_acquiring(ClusterPcmXHolderLedgerEntry *entry) cluster_bufmgr_pcm_x_holder_clear(entry); else if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BUSY) entry->phase = PCM_X_HOLDER_LEDGER_DEFERRED; - else - { + else { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); elog(LOG, "could not abort exact cluster PCM-X ACQUIRING holder: buffer=%d result=%d", - entry->buffer_id, (int) result); + entry->buffer_id, (int)result); } } static void cluster_bufmgr_pcm_x_holder_exception_cleanup_all(void) { - int i; + int i; - for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) - { + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) { ClusterPcmXHolderLedgerEntry *entry = &cluster_bufmgr_pcm_x_holder_ledger[i]; PcmXQueueResult result; if (entry->phase == PCM_X_HOLDER_LEDGER_UNUSED) continue; - if (entry->content_lock == NULL) - { + if (entry->content_lock == NULL) { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); - elog(LOG, "could not detach cluster PCM-X holder after error with no content lock: buffer=%d", - entry->buffer_id); + elog( + LOG, + "could not detach cluster PCM-X holder after error with no content lock: buffer=%d", + entry->buffer_id); continue; } if (LWLockHeldByMe(entry->content_lock)) continue; result = cluster_pcm_x_local_holder_exceptional_detach_exact(&entry->handle, - entry->content_lock); + entry->content_lock); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_NOT_FOUND) cluster_bufmgr_pcm_x_holder_clear(entry); else if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BUSY) entry->phase = PCM_X_HOLDER_LEDGER_DEFERRED; - else - { + else { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); - elog(LOG, "could not detach exact cluster PCM-X holder after error: buffer=%d result=%d", - entry->buffer_id, (int) result); + elog(LOG, + "could not detach exact cluster PCM-X holder after error: buffer=%d result=%d", + entry->buffer_id, (int)result); } } } @@ -1748,8 +1689,7 @@ cluster_bufmgr_pcm_x_holder_exception_cleanup_all(void) static void cluster_bufmgr_pcm_x_holder_reset(void) { - MemSet(cluster_bufmgr_pcm_x_holder_ledger, 0, - sizeof(cluster_bufmgr_pcm_x_holder_ledger)); + MemSet(cluster_bufmgr_pcm_x_holder_ledger, 0, sizeof(cluster_bufmgr_pcm_x_holder_ledger)); cluster_bufmgr_pcm_x_holder_identity = 0; } @@ -1812,18 +1752,30 @@ static void cluster_bufmgr_pcm_x_writer_report_failure(PcmXQueueResult result, BufferDesc *buf, const char *operation) { + BufferTag guard_tag; + int guard_result = 0; + char guard_detail[96]; + + /* Name the exact requester escape arm and, for a nested-guard refusal, + * the held content-lock tag that closed the guard. */ + guard_detail[0] = '\0'; + if (cluster_pcm_x_nested_wait_guard_last_block(&guard_tag, &guard_result)) + snprintf(guard_detail, sizeof(guard_detail), " guard_result=%d guard_rel=%u guard_blk=%u", + guard_result, guard_tag.relNumber, guard_tag.blockNum); if (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY || result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BARRIER_CLOSED || result == PCM_X_QUEUE_NO_CAPACITY) ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), errmsg("cluster PCM-X writer operation is not ready"), - errdetail("operation=%s buffer=%d result=%d", operation, - buf != NULL ? buf->buf_id : -1, (int)result))); + errdetail("operation=%s buffer=%d result=%d fail_line=%d%s", operation, + buf != NULL ? buf->buf_id : -1, (int)result, + cluster_gcs_pcm_x_requester_last_fail_line(), guard_detail))); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("cluster PCM-X writer operation failed"), - errdetail("operation=%s buffer=%d result=%d", operation, - buf != NULL ? buf->buf_id : -1, (int)result))); + errdetail("operation=%s buffer=%d result=%d fail_line=%d%s", operation, + buf != NULL ? buf->buf_id : -1, (int)result, + cluster_gcs_pcm_x_requester_last_fail_line(), guard_detail))); } static void @@ -2240,8 +2192,7 @@ static inline int32 GetPrivateRefCount(Buffer buffer); * serialized by the BufferDesc header lock. The private refcount table is * process-local and cannot change concurrently inside this backend. */ static void -cluster_bufmgr_pcm_direct_init_snapshot_locked(BufferDesc *buf, uint32 buf_state, - bool page_is_new, +cluster_bufmgr_pcm_direct_init_snapshot_locked(BufferDesc *buf, uint32 buf_state, bool page_is_new, ClusterPcmDirectInitSnapshot *out) { memset(out, 0, sizeof(*out)); @@ -2257,37 +2208,34 @@ cluster_bufmgr_pcm_direct_init_snapshot_locked(BufferDesc *buf, uint32 buf_state out->page_is_new = page_is_new; } -pg_attribute_noreturn() static void -cluster_bufmgr_pcm_direct_init_report_failure(BufferDesc *buf, ClusterPcmOwnResult result, - const ClusterPcmDirectInitSnapshot *observed, - const char *context) +pg_attribute_noreturn() static void cluster_bufmgr_pcm_direct_init_report_failure( + BufferDesc *buf, ClusterPcmOwnResult result, const ClusterPcmDirectInitSnapshot *observed, + const char *context) { - uint64 generation = observed != NULL ? observed->generation : 0; - uint32 flags = observed != NULL ? observed->flags : 0; + uint64 generation = observed != NULL ? observed->generation : 0; + uint32 flags = observed != NULL ? observed->flags : 0; cluster_grd_inc_block_path_failclosed(); - if (result == CLUSTER_PCM_OWN_BUSY || result == CLUSTER_PCM_OWN_CORRUPT || - result == CLUSTER_PCM_OWN_EXHAUSTED || result == CLUSTER_PCM_OWN_NOT_READY) + if (result == CLUSTER_PCM_OWN_BUSY || result == CLUSTER_PCM_OWN_CORRUPT + || result == CLUSTER_PCM_OWN_EXHAUSTED || result == CLUSTER_PCM_OWN_NOT_READY) cluster_pcm_own_report_bump_failure(buf, result, generation, flags, context); - ereport(ERROR, - (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("cluster PCM direct initialization lacks an exact live proof"), - errdetail("context=%s buffer=%d result=%d generation=%llu flags=0x%x", - context, buf != NULL ? buf->buf_id : -1, (int) result, - (unsigned long long) generation, flags))); + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cluster PCM direct initialization lacks an exact live proof"), + errdetail("context=%s buffer=%d result=%d generation=%llu flags=0x%x", context, + buf != NULL ? buf->buf_id : -1, (int)result, + (unsigned long long)generation, flags))); } -pg_attribute_noreturn() static void -cluster_bufmgr_pcm_direct_init_no_grant_failclosed(BufferDesc *buf, - ClusterPcmDirectInitKind kind) +pg_attribute_noreturn() static void cluster_bufmgr_pcm_direct_init_no_grant_failclosed( + BufferDesc *buf, ClusterPcmDirectInitKind kind) { cluster_grd_inc_block_path_failclosed(); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("cluster PCM direct initialization did not obtain X ownership"), errdetail("buffer=%d operation=%d returned a one-shot image without a durable grant", - buf != NULL ? buf->buf_id : -1, (int) kind))); + buf != NULL ? buf->buf_id : -1, (int)kind))); } /* Arm only at a source-authorized operation site. This does not reserve the @@ -2299,21 +2247,20 @@ cluster_bufmgr_pcm_arm_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind kin { ClusterPcmDirectInitSnapshot observed; ClusterPcmOwnResult result; - uint32 buf_state; + uint32 buf_state; memset(proof, 0, sizeof(*proof)); if (!cluster_pcm_is_active() || !cluster_bufmgr_should_pcm_track(buf)) return; buf_state = LockBufHdr(buf); - cluster_bufmgr_pcm_direct_init_snapshot_locked( - buf, buf_state, PageIsNew((Page) BufHdrGetBlock(buf)), &observed); + cluster_bufmgr_pcm_direct_init_snapshot_locked(buf, buf_state, + PageIsNew((Page)BufHdrGetBlock(buf)), &observed); result = cluster_pcm_direct_init_proof_arm(kind, &observed, proof); UnlockBufHdr(buf, buf_state); if (result != CLUSTER_PCM_OWN_OK) - cluster_bufmgr_pcm_direct_init_report_failure(buf, result, &observed, - "direct-init arm"); + cluster_bufmgr_pcm_direct_init_report_failure(buf, result, &observed, "direct-init arm"); } /* @@ -2330,26 +2277,25 @@ cluster_bufmgr_pcm_gate_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind ki ClusterPcmDirectInitSnapshot observed; ClusterPcmOwnSnapshot pending_base; ClusterPcmOwnResult pending_result; - bool grant_acquired = false; - uint32 buf_state; - uint64 pending_token = 0; - uint64 committed_generation; + bool grant_acquired = false; + uint32 buf_state; + uint64 pending_token = 0; + uint64 committed_generation; if (!cluster_pcm_is_active() || !cluster_bufmgr_should_pcm_track(buf)) return; if (proof == NULL) cluster_bufmgr_pcm_direct_init_report_failure(buf, CLUSTER_PCM_OWN_INVALID, NULL, - "direct-init missing proof"); + "direct-init missing proof"); buf_state = LockBufHdr(buf); - cluster_bufmgr_pcm_direct_init_snapshot_locked( - buf, buf_state, PageIsNew((Page) BufHdrGetBlock(buf)), &observed); + cluster_bufmgr_pcm_direct_init_snapshot_locked(buf, buf_state, + PageIsNew((Page)BufHdrGetBlock(buf)), &observed); pending_result = cluster_pcm_direct_init_proof_consume(kind, &observed, proof); if (pending_result == CLUSTER_PCM_OWN_OK) pending_result = cluster_pcm_own_reservation_begin_exact( buf->buf_id, observed.generation, PCM_OWN_FLAG_GRANT_PENDING, &pending_token); - if (pending_result == CLUSTER_PCM_OWN_OK) - { + if (pending_result == CLUSTER_PCM_OWN_OK) { memset(&pending_base, 0, sizeof(pending_base)); pending_base.tag = observed.tag; pending_base.generation = observed.generation; @@ -2361,7 +2307,7 @@ cluster_bufmgr_pcm_gate_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind ki if (pending_result != CLUSTER_PCM_OWN_OK) cluster_bufmgr_pcm_direct_init_report_failure(buf, pending_result, &observed, - "direct-init consume"); + "direct-init consume"); PG_TRY(); { @@ -2377,12 +2323,11 @@ cluster_bufmgr_pcm_gate_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind ki if (grant_acquired) cluster_pcm_own_finish_grant_or_rollback(buf, &pending_base, pending_token, - (uint8) PCM_STATE_X, PCM_LOCK_MODE_X, - &committed_generation); - else - { + (uint8)PCM_STATE_X, PCM_LOCK_MODE_X, + &committed_generation); + else { cluster_pcm_own_abort_grant_or_error(buf, &pending_base, pending_token, - "direct-init read-image"); + "direct-init read-image"); cluster_bufmgr_pcm_direct_init_no_grant_failclosed(buf, kind); } } @@ -2395,12 +2340,11 @@ cluster_bufmgr_pcm_gate_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind ki * being dropped. For the relations with size below this threshold, we find * the buffers by doing lookups in BufMapping table. */ -#define BUF_DROP_FULL_SCAN_THRESHOLD (uint64) (NBuffers / 32) +#define BUF_DROP_FULL_SCAN_THRESHOLD (uint64)(NBuffers / 32) -typedef struct PrivateRefCountEntry -{ - Buffer buffer; - int32 refcount; +typedef struct PrivateRefCountEntry { + Buffer buffer; + int32 refcount; } PrivateRefCountEntry; /* 64 bytes, about the size of a cache line on common systems */ @@ -2410,10 +2354,9 @@ typedef struct PrivateRefCountEntry * Status of buffers to checkpoint for a particular tablespace, used * internally in BufferSync. */ -typedef struct CkptTsStatus -{ +typedef struct CkptTsStatus { /* oid of the tablespace */ - Oid tsId; + Oid tsId; /* * Checkpoint progress for this tablespace. To make progress comparable @@ -2422,16 +2365,16 @@ typedef struct CkptTsStatus * page checkpointed in this tablespace increments this space's progress * by progress_slice. */ - float8 progress; - float8 progress_slice; + float8 progress; + float8 progress_slice; /* number of to-be checkpointed pages in this tablespace */ - int num_to_scan; + int num_to_scan; /* already processed pages in this tablespace */ - int num_scanned; + int num_scanned; /* current offset in CkptBufferIds for this tablespace */ - int index; + int index; } CkptTsStatus; /* @@ -2441,17 +2384,16 @@ typedef struct CkptTsStatus * DropRelationsAllBuffers. Pointer to this struct and RelFileLocator must be * compatible. */ -typedef struct SMgrSortArray -{ - RelFileLocator rlocator; /* This must be the first member */ +typedef struct SMgrSortArray { + RelFileLocator rlocator; /* This must be the first member */ SMgrRelation srel; } SMgrSortArray; /* GUC variables */ -bool zero_damaged_pages = false; -int bgwriter_lru_maxpages = 100; -double bgwriter_lru_multiplier = 2.0; -bool track_io_timing = false; +bool zero_damaged_pages = false; +int bgwriter_lru_maxpages = 100; +double bgwriter_lru_multiplier = 2.0; +bool track_io_timing = false; /* * How many buffers PrefetchBuffer callers should try to stay ahead of their @@ -2459,22 +2401,22 @@ bool track_io_timing = false; * for buffers not belonging to tablespaces that have their * effective_io_concurrency parameter set. */ -int effective_io_concurrency = DEFAULT_EFFECTIVE_IO_CONCURRENCY; +int effective_io_concurrency = DEFAULT_EFFECTIVE_IO_CONCURRENCY; /* * Like effective_io_concurrency, but used by maintenance code paths that might * benefit from a higher setting because they work on behalf of many sessions. * Overridden by the tablespace setting of the same name. */ -int maintenance_io_concurrency = DEFAULT_MAINTENANCE_IO_CONCURRENCY; +int maintenance_io_concurrency = DEFAULT_MAINTENANCE_IO_CONCURRENCY; /* * GUC variables about triggering kernel writeback for buffers written; OS * dependent defaults are set via the GUC mechanism. */ -int checkpoint_flush_after = DEFAULT_CHECKPOINT_FLUSH_AFTER; -int bgwriter_flush_after = DEFAULT_BGWRITER_FLUSH_AFTER; -int backend_flush_after = DEFAULT_BACKEND_FLUSH_AFTER; +int checkpoint_flush_after = DEFAULT_CHECKPOINT_FLUSH_AFTER; +int bgwriter_flush_after = DEFAULT_BGWRITER_FLUSH_AFTER; +int backend_flush_after = DEFAULT_BACKEND_FLUSH_AFTER; /* local state for LockBufferForCleanup */ static BufferDesc *PinCountWaitBuf = NULL; @@ -2538,16 +2480,14 @@ ReservePrivateRefCountEntry(void) * majority of cases. */ { - int i; + int i; - for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++) - { + for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++) { PrivateRefCountEntry *res; res = &PrivateRefCountArray[i]; - if (res->buffer == InvalidBuffer) - { + if (res->buffer == InvalidBuffer) { ReservedRefCountEntry = res; return; } @@ -2564,19 +2504,17 @@ ReservePrivateRefCountEntry(void) * hashtable. Use that slot. */ PrivateRefCountEntry *hashent; - bool found; + bool found; /* select victim slot */ - ReservedRefCountEntry = - &PrivateRefCountArray[PrivateRefCountClock++ % REFCOUNT_ARRAY_ENTRIES]; + ReservedRefCountEntry + = &PrivateRefCountArray[PrivateRefCountClock++ % REFCOUNT_ARRAY_ENTRIES]; /* Better be used, otherwise we shouldn't get here. */ Assert(ReservedRefCountEntry->buffer != InvalidBuffer); /* enter victim array entry into hashtable */ - hashent = hash_search(PrivateRefCountHash, - &(ReservedRefCountEntry->buffer), - HASH_ENTER, + hashent = hash_search(PrivateRefCountHash, &(ReservedRefCountEntry->buffer), HASH_ENTER, &found); Assert(!found); hashent->refcount = ReservedRefCountEntry->refcount; @@ -2622,7 +2560,7 @@ static PrivateRefCountEntry * GetPrivateRefCountEntry(Buffer buffer, bool do_move) { PrivateRefCountEntry *res; - int i; + int i; Assert(BufferIsValid(buffer)); Assert(!BufferIsLocal(buffer)); @@ -2631,8 +2569,7 @@ GetPrivateRefCountEntry(Buffer buffer, bool do_move) * First search for references in the array, that'll be sufficient in the * majority of cases. */ - for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++) - { + for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++) { res = &PrivateRefCountArray[i]; if (res->buffer == buffer) @@ -2653,15 +2590,12 @@ GetPrivateRefCountEntry(Buffer buffer, bool do_move) if (res == NULL) return NULL; - else if (!do_move) - { + else if (!do_move) { /* caller doesn't want us to move the hash entry into the array */ return res; - } - else - { + } else { /* move buffer from hashtable into the free array slot */ - bool found; + bool found; PrivateRefCountEntry *free; /* Ensure there's a free array slot */ @@ -2720,9 +2654,7 @@ ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref) { Assert(ref->refcount == 0); - if (ref >= &PrivateRefCountArray[0] && - ref < &PrivateRefCountArray[REFCOUNT_ARRAY_ENTRIES]) - { + if (ref >= &PrivateRefCountArray[0] && ref < &PrivateRefCountArray[REFCOUNT_ARRAY_ENTRIES]) { ref->buffer = InvalidBuffer; /* @@ -2731,11 +2663,9 @@ ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref) * entries. */ ReservedRefCountEntry = ref; - } - else - { - bool found; - Buffer buffer = ref->buffer; + } else { + bool found; + Buffer buffer = ref->buffer; hash_search(PrivateRefCountHash, &buffer, HASH_REMOVE, &found); Assert(found); @@ -2751,38 +2681,23 @@ ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref) * NOTE: what we check here is that *this* backend holds a pin on * the buffer. We do not care whether some other backend does. */ -#define BufferIsPinned(bufnum) \ -( \ - !BufferIsValid(bufnum) ? \ - false \ - : \ - BufferIsLocal(bufnum) ? \ - (LocalRefCount[-(bufnum) - 1] > 0) \ - : \ - (GetPrivateRefCount(bufnum) > 0) \ -) - - -static Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, - ForkNumber forkNum, BlockNumber blockNum, - ReadBufferMode mode, BufferAccessStrategy strategy, - bool *hit); -static BlockNumber ExtendBufferedRelCommon(BufferManagerRelation bmr, - ForkNumber fork, - BufferAccessStrategy strategy, - uint32 flags, - uint32 extend_by, - BlockNumber extend_upto, - Buffer *buffers, - uint32 *extended_by); -static BlockNumber ExtendBufferedRelShared(BufferManagerRelation bmr, - ForkNumber fork, - BufferAccessStrategy strategy, - uint32 flags, - uint32 extend_by, - BlockNumber extend_upto, - Buffer *buffers, - uint32 *extended_by); +#define BufferIsPinned(bufnum) \ + (!BufferIsValid(bufnum) ? false \ + : BufferIsLocal(bufnum) ? (LocalRefCount[-(bufnum) - 1] > 0) \ + : (GetPrivateRefCount(bufnum) > 0)) + + +static Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, + BlockNumber blockNum, ReadBufferMode mode, + BufferAccessStrategy strategy, bool *hit); +static BlockNumber ExtendBufferedRelCommon(BufferManagerRelation bmr, ForkNumber fork, + BufferAccessStrategy strategy, uint32 flags, + uint32 extend_by, BlockNumber extend_upto, + Buffer *buffers, uint32 *extended_by); +static BlockNumber ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, + BufferAccessStrategy strategy, uint32 flags, + uint32 extend_by, BlockNumber extend_upto, + Buffer *buffers, uint32 *extended_by); static bool PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy); static void PinBuffer_Locked(BufferDesc *buf); static void UnpinBuffer(BufferDesc *buf); @@ -2800,12 +2715,10 @@ extern void cluster_gcs_block_pi_write_note(BufferTag tag, SCN page_scn); extern bool cluster_gcs_block_test_deliver_self_invalidate(BufferTag tag); #endif static uint32 WaitBufHdrUnlocked(BufferDesc *buf); -static int SyncOneBuffer(int buf_id, bool skip_recently_used, - WritebackContext *wb_context); +static int SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context); static void WaitIO(BufferDesc *buf); static bool StartBufferIO(BufferDesc *buf, bool forInput); -static void TerminateBufferIO(BufferDesc *buf, bool clear_dirty, - uint32 set_flag_bits); +static void TerminateBufferIO(BufferDesc *buf, bool clear_dirty, uint32 set_flag_bits); static void shared_buffer_write_error_callback(void *arg); static void local_buffer_write_error_callback(void *arg); @@ -2814,56 +2727,46 @@ static void local_buffer_write_error_callback(void *arg); static bool InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, LWLock *oldPartitionLock, uint32 buf_state); static void InvalidateBufferCommitTailLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, - LWLock *oldPartitionLock, uint32 buf_state, - uint8 old_pcm_mode, bool release_pcm_holder); + LWLock *oldPartitionLock, uint32 buf_state, + uint8 old_pcm_mode, bool release_pcm_holder); static bool InvalidateBufferTry(BufferDesc *buf); -static BufferDesc *BufferAlloc(SMgrRelation smgr, - char relpersistence, - ForkNumber forkNum, - BlockNumber blockNum, - BufferAccessStrategy strategy, - bool *foundPtr, IOContext io_context); +static BufferDesc *BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, + BlockNumber blockNum, BufferAccessStrategy strategy, bool *foundPtr, + IOContext io_context); static Buffer GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context); -static void FlushBuffer(BufferDesc *buf, SMgrRelation reln, - IOObject io_object, IOContext io_context); -static void FindAndDropRelationBuffers(RelFileLocator rlocator, - ForkNumber forkNum, - BlockNumber nForkBlock, - BlockNumber firstDelBlock); -static void RelationCopyStorageUsingBuffer(RelFileLocator srclocator, - RelFileLocator dstlocator, +static void FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, + IOContext io_context); +static void FindAndDropRelationBuffers(RelFileLocator rlocator, ForkNumber forkNum, + BlockNumber nForkBlock, BlockNumber firstDelBlock); +static void RelationCopyStorageUsingBuffer(RelFileLocator srclocator, RelFileLocator dstlocator, ForkNumber forkNum, bool permanent); static void AtProcExit_Buffers(int code, Datum arg); static void CheckForBufferLeaks(void); #ifdef USE_ASSERT_CHECKING -static void AssertNotCatalogBufferLock(LWLock *lock, LWLockMode mode, - void *unused_context); +static void AssertNotCatalogBufferLock(LWLock *lock, LWLockMode mode, void *unused_context); #endif -static int rlocator_comparator(const void *p1, const void *p2); +static int rlocator_comparator(const void *p1, const void *p2); static inline int buffertag_comparator(const BufferTag *ba, const BufferTag *bb); static inline int ckpt_buforder_comparator(const CkptSortItem *a, const CkptSortItem *b); -static int ts_ckpt_progress_comparator(Datum a, Datum b, void *arg); +static int ts_ckpt_progress_comparator(Datum a, Datum b, void *arg); /* * Implementation of PrefetchBuffer() for shared buffers. */ PrefetchBufferResult -PrefetchSharedBuffer(SMgrRelation smgr_reln, - ForkNumber forkNum, - BlockNumber blockNum) +PrefetchSharedBuffer(SMgrRelation smgr_reln, ForkNumber forkNum, BlockNumber blockNum) { - PrefetchBufferResult result = {InvalidBuffer, false}; - BufferTag newTag; /* identity of requested block */ - uint32 newHash; /* hash value for newTag */ - LWLock *newPartitionLock; /* buffer partition lock for it */ - int buf_id; + PrefetchBufferResult result = { InvalidBuffer, false }; + BufferTag newTag; /* identity of requested block */ + uint32 newHash; /* hash value for newTag */ + LWLock *newPartitionLock; /* buffer partition lock for it */ + int buf_id; Assert(BlockNumberIsValid(blockNum)); /* create a tag so we can lookup the buffer */ - InitBufferTag(&newTag, &smgr_reln->smgr_rlocator.locator, - forkNum, blockNum); + InitBufferTag(&newTag, &smgr_reln->smgr_rlocator.locator, forkNum, blockNum); /* determine its hash code and partition lock ID */ newHash = BufTableHashCode(&newTag); @@ -2875,22 +2778,17 @@ PrefetchSharedBuffer(SMgrRelation smgr_reln, LWLockRelease(newPartitionLock); /* If not in buffers, initiate prefetch */ - if (buf_id < 0) - { + if (buf_id < 0) { #ifdef USE_PREFETCH /* * Try to initiate an asynchronous read. This returns false in * recovery if the relation file doesn't exist. */ - if ((io_direct_flags & IO_DIRECT_DATA) == 0 && - smgrprefetch(smgr_reln, forkNum, blockNum)) - { + if ((io_direct_flags & IO_DIRECT_DATA) == 0 && smgrprefetch(smgr_reln, forkNum, blockNum)) { result.initiated_io = true; } -#endif /* USE_PREFETCH */ - } - else - { +#endif /* USE_PREFETCH */ + } else { /* * Report the buffer it was in at that time. The caller may be able * to avoid a buffer table lookup, but it's not pinned and it must be @@ -2944,19 +2842,15 @@ PrefetchBuffer(Relation reln, ForkNumber forkNum, BlockNumber blockNum) Assert(RelationIsValid(reln)); Assert(BlockNumberIsValid(blockNum)); - if (RelationUsesLocalBuffers(reln)) - { + if (RelationUsesLocalBuffers(reln)) { /* see comments in ReadBufferExtended */ if (RELATION_IS_OTHER_TEMP(reln)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot access temporary tables of other sessions"))); + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot access temporary tables of other sessions"))); /* pass it off to localbuf.c */ return PrefetchLocalBuffer(RelationGetSmgr(reln), forkNum, blockNum); - } - else - { + } else { /* pass it to the shared buffer version */ return PrefetchSharedBuffer(RelationGetSmgr(reln), forkNum, blockNum); } @@ -2974,9 +2868,9 @@ ReadRecentBuffer(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockN Buffer recent_buffer) { BufferDesc *bufHdr; - BufferTag tag; - uint32 buf_state; - bool have_private_ref; + BufferTag tag; + uint32 buf_state; + bool have_private_ref; Assert(BufferIsValid(recent_buffer)); @@ -2984,25 +2878,21 @@ ReadRecentBuffer(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockN ReservePrivateRefCountEntry(); InitBufferTag(&tag, &rlocator, forkNum, blockNum); - if (BufferIsLocal(recent_buffer)) - { - int b = -recent_buffer - 1; + if (BufferIsLocal(recent_buffer)) { + int b = -recent_buffer - 1; bufHdr = GetLocalBufferDescriptor(b); buf_state = pg_atomic_read_u32(&bufHdr->state); /* Is it still valid and holding the right tag? */ - if ((buf_state & BM_VALID) && BufferTagsEqual(&tag, &bufHdr->tag)) - { + if ((buf_state & BM_VALID) && BufferTagsEqual(&tag, &bufHdr->tag)) { PinLocalBuffer(bufHdr, true); pgBufferUsage.local_blks_hit++; return true; } - } - else - { + } else { bufHdr = GetBufferDescriptor(recent_buffer - 1); have_private_ref = GetPrivateRefCount(recent_buffer) > 0; @@ -3016,17 +2906,16 @@ ReadRecentBuffer(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockN else buf_state = LockBufHdr(bufHdr); - if ((buf_state & BM_VALID) && BufferTagsEqual(&tag, &bufHdr->tag)) - { + if ((buf_state & BM_VALID) && BufferTagsEqual(&tag, &bufHdr->tag)) { /* * It's now safe to pin the buffer. We can't pin first and ask * questions later, because it might confuse code paths like * InvalidateBuffer() if we pinned a random non-matching buffer. */ if (have_private_ref) - PinBuffer(bufHdr, NULL); /* bump pin count */ + PinBuffer(bufHdr, NULL); /* bump pin count */ else - PinBuffer_Locked(bufHdr); /* pin for first time */ + PinBuffer_Locked(bufHdr); /* pin for first time */ pgBufferUsage.shared_blks_hit++; @@ -3093,11 +2982,11 @@ ReadBuffer(Relation reln, BlockNumber blockNum) * See buffer/README for details. */ Buffer -ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, - ReadBufferMode mode, BufferAccessStrategy strategy) +ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode, + BufferAccessStrategy strategy) { - bool hit; - Buffer buf; + bool hit; + Buffer buf; /* * Reject attempts to read non-local temporary relations; we would be @@ -3105,9 +2994,8 @@ ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, * session's local buffers. */ if (RELATION_IS_OTHER_TEMP(reln)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot access temporary tables of other sessions"))); + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot access temporary tables of other sessions"))); #ifdef USE_PGRAC_CLUSTER @@ -3118,15 +3006,12 @@ ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, * hint by exact BufferTag match. Profiling-only, GUC-gated inside the * setter, no behavior change. */ - if (unlikely(cluster_xnode_profile_enabled) && - blockNum != P_NEW && - !RelationUsesLocalBuffers(reln)) - { - BufferTag xp_tag; + if (unlikely(cluster_xnode_profile_enabled) && blockNum != P_NEW + && !RelationUsesLocalBuffers(reln)) { + BufferTag xp_tag; InitBufferTag(&xp_tag, &reln->rd_locator, forkNum, blockNum); - cluster_xp_relkind_hint_set(&xp_tag, - reln->rd_rel->relkind == RELKIND_INDEX); + cluster_xp_relkind_hint_set(&xp_tag, reln->rd_rel->relkind == RELKIND_INDEX); } #endif @@ -3135,8 +3020,8 @@ ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, * miss. */ pgstat_count_buffer_read(reln); - buf = ReadBuffer_common(RelationGetSmgr(reln), reln->rd_rel->relpersistence, - forkNum, blockNum, mode, strategy, &hit); + buf = ReadBuffer_common(RelationGetSmgr(reln), reln->rd_rel->relpersistence, forkNum, blockNum, + mode, strategy, &hit); if (hit) pgstat_count_buffer_hit(reln); return buf; @@ -3154,33 +3039,28 @@ ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, * BackendId). */ Buffer -ReadBufferWithoutRelcache(RelFileLocator rlocator, ForkNumber forkNum, - BlockNumber blockNum, ReadBufferMode mode, - BufferAccessStrategy strategy, bool permanent) +ReadBufferWithoutRelcache(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockNum, + ReadBufferMode mode, BufferAccessStrategy strategy, bool permanent) { - bool hit; + bool hit; SMgrRelation smgr = smgropen(rlocator, InvalidBackendId); - return ReadBuffer_common(smgr, permanent ? RELPERSISTENCE_PERMANENT : - RELPERSISTENCE_UNLOGGED, forkNum, blockNum, - mode, strategy, &hit); + return ReadBuffer_common(smgr, permanent ? RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED, + forkNum, blockNum, mode, strategy, &hit); } /* * Convenience wrapper around ExtendBufferedRelBy() extending by one block. */ Buffer -ExtendBufferedRel(BufferManagerRelation bmr, - ForkNumber forkNum, - BufferAccessStrategy strategy, +ExtendBufferedRel(BufferManagerRelation bmr, ForkNumber forkNum, BufferAccessStrategy strategy, uint32 flags) { - Buffer buf; - uint32 extend_by = 1; + Buffer buf; + uint32 extend_by = 1; - ExtendBufferedRelBy(bmr, forkNum, strategy, flags, extend_by, - &buf, &extend_by); + ExtendBufferedRelBy(bmr, forkNum, strategy, flags, extend_by, &buf, &extend_by); return buf; } @@ -3203,26 +3083,19 @@ ExtendBufferedRel(BufferManagerRelation bmr, * be empty. */ BlockNumber -ExtendBufferedRelBy(BufferManagerRelation bmr, - ForkNumber fork, - BufferAccessStrategy strategy, - uint32 flags, - uint32 extend_by, - Buffer *buffers, - uint32 *extended_by) +ExtendBufferedRelBy(BufferManagerRelation bmr, ForkNumber fork, BufferAccessStrategy strategy, + uint32 flags, uint32 extend_by, Buffer *buffers, uint32 *extended_by) { Assert((bmr.rel != NULL) != (bmr.smgr != NULL)); Assert(bmr.smgr == NULL || bmr.relpersistence != 0); Assert(extend_by > 0); - if (bmr.smgr == NULL) - { + if (bmr.smgr == NULL) { bmr.smgr = RelationGetSmgr(bmr.rel); bmr.relpersistence = bmr.rel->rd_rel->relpersistence; } - return ExtendBufferedRelCommon(bmr, fork, strategy, flags, - extend_by, InvalidBlockNumber, + return ExtendBufferedRelCommon(bmr, fork, strategy, flags, extend_by, InvalidBlockNumber, buffers, extended_by); } @@ -3235,24 +3108,19 @@ ExtendBufferedRelBy(BufferManagerRelation bmr, * crash recovery). */ Buffer -ExtendBufferedRelTo(BufferManagerRelation bmr, - ForkNumber fork, - BufferAccessStrategy strategy, - uint32 flags, - BlockNumber extend_to, - ReadBufferMode mode) +ExtendBufferedRelTo(BufferManagerRelation bmr, ForkNumber fork, BufferAccessStrategy strategy, + uint32 flags, BlockNumber extend_to, ReadBufferMode mode) { BlockNumber current_size; - uint32 extended_by = 0; - Buffer buffer = InvalidBuffer; - Buffer buffers[64]; + uint32 extended_by = 0; + Buffer buffer = InvalidBuffer; + Buffer buffers[64]; Assert((bmr.rel != NULL) != (bmr.smgr != NULL)); Assert(bmr.smgr == NULL || bmr.relpersistence != 0); Assert(extend_to != InvalidBlockNumber && extend_to > 0); - if (bmr.smgr == NULL) - { + if (bmr.smgr == NULL) { bmr.smgr = RelationGetSmgr(bmr.rel); bmr.relpersistence = bmr.rel->rd_rel->relpersistence; } @@ -3262,11 +3130,10 @@ ExtendBufferedRelTo(BufferManagerRelation bmr, * smgr_cached_nblocks[fork] is positive then it must exist, no need for * an smgrexists call. */ - if ((flags & EB_CREATE_FORK_IF_NEEDED) && - (bmr.smgr->smgr_cached_nblocks[fork] == 0 || - bmr.smgr->smgr_cached_nblocks[fork] == InvalidBlockNumber) && - !smgrexists(bmr.smgr, fork)) - { + if ((flags & EB_CREATE_FORK_IF_NEEDED) + && (bmr.smgr->smgr_cached_nblocks[fork] == 0 + || bmr.smgr->smgr_cached_nblocks[fork] == InvalidBlockNumber) + && !smgrexists(bmr.smgr, fork)) { LockRelationForExtension(bmr.rel, ExclusiveLock); /* could have been closed while waiting for lock */ @@ -3302,23 +3169,20 @@ ExtendBufferedRelTo(BufferManagerRelation bmr, if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) flags |= EB_LOCK_TARGET; - while (current_size < extend_to) - { - uint32 num_pages = lengthof(buffers); + while (current_size < extend_to) { + uint32 num_pages = lengthof(buffers); BlockNumber first_block; - if ((uint64) current_size + num_pages > extend_to) + if ((uint64)current_size + num_pages > extend_to) num_pages = extend_to - current_size; - first_block = ExtendBufferedRelCommon(bmr, fork, strategy, flags, - num_pages, extend_to, + first_block = ExtendBufferedRelCommon(bmr, fork, strategy, flags, num_pages, extend_to, buffers, &extended_by); current_size = first_block + extended_by; Assert(num_pages != 0 || current_size >= extend_to); - for (int i = 0; i < extended_by; i++) - { + for (int i = 0; i < extended_by; i++) { if (first_block + i != extend_to - 1) ReleaseBuffer(buffers[i]); else @@ -3332,14 +3196,12 @@ ExtendBufferedRelTo(BufferManagerRelation bmr, * * XXX: Should we control this via a flag? */ - if (buffer == InvalidBuffer) - { - bool hit; + if (buffer == InvalidBuffer) { + bool hit; Assert(extended_by == 0); - buffer = ReadBuffer_common(bmr.smgr, bmr.relpersistence, - fork, extend_to - 1, mode, strategy, - &hit); + buffer = ReadBuffer_common(bmr.smgr, bmr.relpersistence, fork, extend_to - 1, mode, + strategy, &hit); } return buffer; @@ -3351,16 +3213,15 @@ ExtendBufferedRelTo(BufferManagerRelation bmr, * *hit is set to true if the request was satisfied from shared buffer cache. */ static Buffer -ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, - BlockNumber blockNum, ReadBufferMode mode, - BufferAccessStrategy strategy, bool *hit) +ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, + ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit) { BufferDesc *bufHdr; - Block bufBlock; - bool found; - IOContext io_context; - IOObject io_object; - bool isLocalBuf = SmgrIsTemp(smgr); + Block bufBlock; + bool found; + IOContext io_context; + IOObject io_object; + bool isLocalBuf = SmgrIsTemp(smgr); #ifdef USE_PGRAC_CLUSTER ClusterPcmDirectInitProof direct_init_proof; #endif @@ -3372,9 +3233,8 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * instead, as acquiring the extension lock inside ExtendBufferedRel() * scales a lot better. */ - if (unlikely(blockNum == P_NEW)) - { - uint32 flags = EB_SKIP_EXTENSION_LOCK; + if (unlikely(blockNum == P_NEW)) { + uint32 flags = EB_SKIP_EXTENSION_LOCK; /* * Since no-one else can be looking at the page contents yet, there is @@ -3384,21 +3244,17 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) flags |= EB_LOCK_FIRST; - return ExtendBufferedRel(BMR_SMGR(smgr, relpersistence), - forkNum, strategy, flags); + return ExtendBufferedRel(BMR_SMGR(smgr, relpersistence), forkNum, strategy, flags); } /* Make sure we will have room to remember the buffer pin */ ResourceOwnerEnlargeBuffers(CurrentResourceOwner); - TRACE_POSTGRESQL_BUFFER_READ_START(forkNum, blockNum, - smgr->smgr_rlocator.locator.spcOid, - smgr->smgr_rlocator.locator.dbOid, - smgr->smgr_rlocator.locator.relNumber, - smgr->smgr_rlocator.backend); + TRACE_POSTGRESQL_BUFFER_READ_START( + forkNum, blockNum, smgr->smgr_rlocator.locator.spcOid, smgr->smgr_rlocator.locator.dbOid, + smgr->smgr_rlocator.locator.relNumber, smgr->smgr_rlocator.backend); - if (isLocalBuf) - { + if (isLocalBuf) { /* * We do not use a BufferAccessStrategy for I/O of temporary tables. * However, in some cases, the "strategy" may not be NULL, so we can't @@ -3410,24 +3266,19 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, bufHdr = LocalBufferAlloc(smgr, forkNum, blockNum, &found); if (found) pgBufferUsage.local_blks_hit++; - else if (mode == RBM_NORMAL || mode == RBM_NORMAL_NO_LOG || - mode == RBM_ZERO_ON_ERROR) + else if (mode == RBM_NORMAL || mode == RBM_NORMAL_NO_LOG || mode == RBM_ZERO_ON_ERROR) pgBufferUsage.local_blks_read++; - } - else - { + } else { /* * lookup the buffer. IO_IN_PROGRESS is set if the requested block is * not currently in memory. */ io_context = IOContextForStrategy(strategy); io_object = IOOBJECT_RELATION; - bufHdr = BufferAlloc(smgr, relpersistence, forkNum, blockNum, - strategy, &found, io_context); + bufHdr = BufferAlloc(smgr, relpersistence, forkNum, blockNum, strategy, &found, io_context); if (found) pgBufferUsage.shared_blks_hit++; - else if (mode == RBM_NORMAL || mode == RBM_NORMAL_NO_LOG || - mode == RBM_ZERO_ON_ERROR) + else if (mode == RBM_NORMAL || mode == RBM_NORMAL_NO_LOG || mode == RBM_ZERO_ON_ERROR) pgBufferUsage.shared_blks_read++; #ifdef USE_PGRAC_CLUSTER @@ -3439,10 +3290,8 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * transfer -- the cache-locality signal the A-L9 observability leg * watches. */ - if (cluster_shared_catalog && - smgr->smgr_rlocator.locator.relNumber < - (RelFileNumber) FirstNormalObjectId) - { + if (cluster_shared_catalog + && smgr->smgr_rlocator.locator.relNumber < (RelFileNumber)FirstNormalObjectId) { if (found) cluster_catalog_stats_buf_hit_inc(); else @@ -3454,8 +3303,7 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, /* At this point we do NOT hold any locks. */ /* if it was already in the buffer pool, we're done */ - if (found) - { + if (found) { /* Just need to update stats before we exit */ *hit = true; VacuumPageHit++; @@ -3464,19 +3312,16 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, if (VacuumCostActive) VacuumCostBalance += VacuumCostPageHit; - TRACE_POSTGRESQL_BUFFER_READ_DONE(forkNum, blockNum, - smgr->smgr_rlocator.locator.spcOid, + TRACE_POSTGRESQL_BUFFER_READ_DONE(forkNum, blockNum, smgr->smgr_rlocator.locator.spcOid, smgr->smgr_rlocator.locator.dbOid, smgr->smgr_rlocator.locator.relNumber, - smgr->smgr_rlocator.backend, - found); + smgr->smgr_rlocator.backend, found); /* * In RBM_ZERO_AND_LOCK mode the caller expects the page to be locked * on return. */ - if (!isLocalBuf) - { + if (!isLocalBuf) { if (mode == RBM_ZERO_AND_LOCK) LockBuffer(BufferDescriptorGetBuffer(bufHdr), BUFFER_LOCK_EXCLUSIVE); else if (mode == RBM_ZERO_AND_CLEANUP_LOCK) @@ -3491,7 +3336,7 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * page but its contents are not yet valid. IO_IN_PROGRESS is set for it, * if it's a shared buffer. */ - Assert(!(pg_atomic_read_u32(&bufHdr->state) & BM_VALID)); /* spinlock not needed */ + Assert(!(pg_atomic_read_u32(&bufHdr->state) & BM_VALID)); /* spinlock not needed */ bufBlock = isLocalBuf ? LocalBufHdrGetBlock(bufHdr) : BufHdrGetBlock(bufHdr); @@ -3499,28 +3344,24 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * Read in the page, unless the caller intends to overwrite it and just * wants us to allocate a buffer. */ - if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) - { - MemSet((char *) bufBlock, 0, BLCKSZ); + if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) { + MemSet((char *)bufBlock, 0, BLCKSZ); #ifdef USE_PGRAC_CLUSTER if (!isLocalBuf) - cluster_bufmgr_pcm_arm_direct_init( - bufHdr, CLUSTER_PCM_DIRECT_INIT_READ_MISS, &direct_init_proof); + cluster_bufmgr_pcm_arm_direct_init(bufHdr, CLUSTER_PCM_DIRECT_INIT_READ_MISS, + &direct_init_proof); #endif - } - else - { - instr_time io_start = pgstat_prepare_io_time(); - bool verified; + } else { + instr_time io_start = pgstat_prepare_io_time(); + bool verified; smgrread(smgr, forkNum, blockNum, bufBlock); - pgstat_count_io_op_time(io_object, io_context, - IOOP_READ, io_start, 1); + pgstat_count_io_op_time(io_object, io_context, IOOP_READ, io_start, 1); /* check for garbage data */ - verified = PageIsVerifiedExtended((Page) bufBlock, blockNum, - PIV_LOG_WARNING | PIV_REPORT_STAT); + verified + = PageIsVerifiedExtended((Page)bufBlock, blockNum, PIV_LOG_WARNING | PIV_REPORT_STAT); #ifdef USE_PGRAC_CLUSTER @@ -3538,34 +3379,27 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * ignore_checksum_failure's "return the page as-is" remains the * fallback (verified stays true). */ - if (!verified) - { + if (!verified) { if (relpersistence == RELPERSISTENCE_PERMANENT && !isLocalBuf - && cluster_block_recovery_on_read(smgr, forkNum, blockNum, (char *) bufBlock)) + && cluster_block_recovery_on_read(smgr, forkNum, blockNum, (char *)bufBlock)) verified = true; - } - else if (relpersistence == RELPERSISTENCE_PERMANENT && !isLocalBuf - && ignore_checksum_failure && cluster_online_block_recovery - && cluster_block_recovery_checksum_mismatch((char *) bufBlock, blockNum)) - { - (void) cluster_block_recovery_on_read(smgr, forkNum, blockNum, (char *) bufBlock); + } else if (relpersistence == RELPERSISTENCE_PERMANENT && !isLocalBuf + && ignore_checksum_failure && cluster_online_block_recovery + && cluster_block_recovery_checksum_mismatch((char *)bufBlock, blockNum)) { + (void)cluster_block_recovery_on_read(smgr, forkNum, blockNum, (char *)bufBlock); } #endif - if (!verified) - { - if (mode == RBM_ZERO_ON_ERROR || zero_damaged_pages) - { + if (!verified) { + if (mode == RBM_ZERO_ON_ERROR || zero_damaged_pages) { ereport(WARNING, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid page in block %u of relation %s; zeroing out page", - blockNum, - relpath(smgr->smgr_rlocator, forkNum)))); - MemSet((char *) bufBlock, 0, BLCKSZ); + blockNum, relpath(smgr->smgr_rlocator, forkNum)))); + MemSet((char *)bufBlock, 0, BLCKSZ); } #ifdef USE_PGRAC_CLUSTER - else if (cluster_online_block_recovery) - { + else if (cluster_online_block_recovery) { /* * Online recovery on but the block could not be rebuilt. * PANIC escalation (operator opt-in) is restricted to the @@ -3573,14 +3407,14 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * reconstructible from scratch, so an unrebuildable one * should fail the query (ERROR), not crash the instance. */ - int elevel = (cluster_block_recovery_on_unrecoverable == CLUSTER_BLKREC_ACTION_PANIC - && forkNum == MAIN_FORKNUM) - ? PANIC : ERROR; + int elevel = (cluster_block_recovery_on_unrecoverable == CLUSTER_BLKREC_ACTION_PANIC + && forkNum == MAIN_FORKNUM) + ? PANIC + : ERROR; ereport(elevel, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("invalid page in block %u of relation %s", - blockNum, + errmsg("invalid page in block %u of relation %s", blockNum, relpath(smgr->smgr_rlocator, forkNum)), errhint("online block recovery could not rebuild this block from WAL " "(no full-page-image base in retained WAL, an unsupported " @@ -3588,11 +3422,9 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, } #endif else - ereport(ERROR, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("invalid page in block %u of relation %s", - blockNum, - relpath(smgr->smgr_rlocator, forkNum)))); + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("invalid page in block %u of relation %s", blockNum, + relpath(smgr->smgr_rlocator, forkNum)))); } } @@ -3607,26 +3439,21 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * that we cannot use LockBuffer() or LockBufferForCleanup() here, because * they assert that the buffer is already valid.) */ - if ((mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) && - !isLocalBuf) - { + if ((mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) && !isLocalBuf) { #ifdef USE_PGRAC_CLUSTER - cluster_bufmgr_pcm_gate_direct_init( - bufHdr, CLUSTER_PCM_DIRECT_INIT_READ_MISS, &direct_init_proof); + cluster_bufmgr_pcm_gate_direct_init(bufHdr, CLUSTER_PCM_DIRECT_INIT_READ_MISS, + &direct_init_proof); #endif LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_EXCLUSIVE); } - if (isLocalBuf) - { + if (isLocalBuf) { /* Only need to adjust flags */ - uint32 buf_state = pg_atomic_read_u32(&bufHdr->state); + uint32 buf_state = pg_atomic_read_u32(&bufHdr->state); buf_state |= BM_VALID; pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state); - } - else - { + } else { /* Set BM_VALID, terminate IO, and wake up any waiters */ TerminateBufferIO(bufHdr, false, BM_VALID); } @@ -3635,12 +3462,9 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, if (VacuumCostActive) VacuumCostBalance += VacuumCostPageMiss; - TRACE_POSTGRESQL_BUFFER_READ_DONE(forkNum, blockNum, - smgr->smgr_rlocator.locator.spcOid, - smgr->smgr_rlocator.locator.dbOid, - smgr->smgr_rlocator.locator.relNumber, - smgr->smgr_rlocator.backend, - found); + TRACE_POSTGRESQL_BUFFER_READ_DONE( + forkNum, blockNum, smgr->smgr_rlocator.locator.spcOid, smgr->smgr_rlocator.locator.dbOid, + smgr->smgr_rlocator.locator.relNumber, smgr->smgr_rlocator.backend, found); return BufferDescriptorGetBuffer(bufHdr); } @@ -3669,18 +3493,16 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * No locks are held either at entry or exit. */ static BufferDesc * -BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, - BlockNumber blockNum, - BufferAccessStrategy strategy, - bool *foundPtr, IOContext io_context) -{ - BufferTag newTag; /* identity of requested block */ - uint32 newHash; /* hash value for newTag */ - LWLock *newPartitionLock; /* buffer partition lock for it */ - int existing_buf_id; - Buffer victim_buffer; +BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, + BufferAccessStrategy strategy, bool *foundPtr, IOContext io_context) +{ + BufferTag newTag; /* identity of requested block */ + uint32 newHash; /* hash value for newTag */ + LWLock *newPartitionLock; /* buffer partition lock for it */ + int existing_buf_id; + Buffer victim_buffer; BufferDesc *victim_buf_hdr; - uint32 victim_buf_state; + uint32 victim_buf_state; /* create a tag so we can lookup the buffer */ InitBufferTag(&newTag, &smgr->smgr_rlocator.locator, forkNum, blockNum); @@ -3692,10 +3514,9 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, /* see if the block is in the buffer pool already */ LWLockAcquire(newPartitionLock, LW_SHARED); existing_buf_id = BufTableLookup(&newTag, newHash); - if (existing_buf_id >= 0) - { + if (existing_buf_id >= 0) { BufferDesc *buf; - bool valid; + bool valid; /* * Found it. Now, pin the buffer so no one can steal it from the @@ -3711,8 +3532,7 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, *foundPtr = true; - if (!valid) - { + if (!valid) { /* * We can only get here if (a) someone else is still reading in * the page, or (b) a previous read attempt failed. We have to @@ -3720,8 +3540,7 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * own read attempt if the page is still not BM_VALID. * StartBufferIO does it all. */ - if (StartBufferIO(buf, true)) - { + if (StartBufferIO(buf, true)) { /* * If we get here, previous attempts to read the buffer must * have failed ... but we shall bravely try again. @@ -3754,10 +3573,9 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, */ LWLockAcquire(newPartitionLock, LW_EXCLUSIVE); existing_buf_id = BufTableInsert(&newTag, newHash, victim_buf_hdr->buf_id); - if (existing_buf_id >= 0) - { + if (existing_buf_id >= 0) { BufferDesc *existing_buf_hdr; - bool valid; + bool valid; /* * Got a collision. Someone has already done what we were about to do. @@ -3789,8 +3607,7 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, *foundPtr = true; - if (!valid) - { + if (!valid) { /* * We can only get here if (a) someone else is still reading in * the page, or (b) a previous read attempt failed. We have to @@ -3798,8 +3615,7 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * own read attempt if the page is still not BM_VALID. * StartBufferIO does it all. */ - if (StartBufferIO(existing_buf_hdr, true)) - { + if (StartBufferIO(existing_buf_hdr, true)) { /* * If we get here, previous attempts to read the buffer must * have failed ... but we shall bravely try again. @@ -3882,10 +3698,10 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, static void InvalidateBuffer(BufferDesc *buf) { - BufferTag oldTag; - uint32 oldHash; /* hash value for oldTag */ - LWLock *oldPartitionLock; /* buffer partition lock for it */ - uint32 buf_state; + BufferTag oldTag; + uint32 oldHash; /* hash value for oldTag */ + LWLock *oldPartitionLock; /* buffer partition lock for it */ + uint32 buf_state; /* Save the original buffer tag before dropping the spinlock */ oldTag = buf->tag; @@ -3914,8 +3730,7 @@ InvalidateBuffer(BufferDesc *buf) buf_state = LockBufHdr(buf); /* If it's changed while we were waiting for lock, do nothing */ - if (!BufferTagsEqual(&buf->tag, &oldTag)) - { + if (!BufferTagsEqual(&buf->tag, &oldTag)) { UnlockBufHdr(buf, buf_state); LWLockRelease(oldPartitionLock); return; @@ -3930,8 +3745,7 @@ InvalidateBuffer(BufferDesc *buf) * yet done StartBufferIO, WaitIO will fall through and we'll effectively * be busy-looping here.) */ - if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) - { + if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) { UnlockBufHdr(buf, buf_state); LWLockRelease(oldPartitionLock); /* safety check: should definitely not be our *own* pin */ @@ -3941,8 +3755,7 @@ InvalidateBuffer(BufferDesc *buf) goto retry; } - if (!InvalidateBufferCommitLocked(buf, &oldTag, oldHash, oldPartitionLock, buf_state)) - { + if (!InvalidateBufferCommitLocked(buf, &oldTag, oldHash, oldPartitionLock, buf_state)) { pg_usleep(1000L); goto retry; } @@ -3959,15 +3772,15 @@ InvalidateBuffer(BufferDesc *buf) */ static bool InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, - LWLock *oldPartitionLock, uint32 buf_state) + LWLock *oldPartitionLock, uint32 buf_state) { - uint8 old_pcm_mode = 0; - bool release_pcm_holder = false; + uint8 old_pcm_mode = 0; + bool release_pcm_holder = false; #ifdef USE_PGRAC_CLUSTER ClusterPcmOwnEvictionCapture eviction_capture; ClusterPcmOwnResult eviction_result; - uint32 observed_flags = 0; - uint64 observed_generation = 0; + uint32 observed_flags = 0; + uint64 observed_generation = 0; /* * D5a: descriptor reuse is an exact ownership-tuple commit. Refuse to @@ -3977,24 +3790,22 @@ InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, * succeed. */ cluster_pcm_own_eviction_capture_locked(buf, &eviction_capture); - eviction_result = cluster_pcm_own_eviction_commit_locked( - buf, &eviction_capture, &observed_generation, &observed_flags); - if (eviction_result != CLUSTER_PCM_OWN_OK) - { + eviction_result = cluster_pcm_own_eviction_commit_locked(buf, &eviction_capture, + &observed_generation, &observed_flags); + if (eviction_result != CLUSTER_PCM_OWN_OK) { UnlockBufHdr(buf, buf_state); LWLockRelease(oldPartitionLock); if (eviction_result == CLUSTER_PCM_OWN_BUSY || eviction_result == CLUSTER_PCM_OWN_STALE) return false; - cluster_pcm_own_report_bump_failure(buf, eviction_result, - observed_generation != 0 ? observed_generation - : eviction_capture.generation, - observed_flags != 0 ? observed_flags : eviction_capture.flags, - "buffer eviction"); + cluster_pcm_own_report_bump_failure( + buf, eviction_result, + observed_generation != 0 ? observed_generation : eviction_capture.generation, + observed_flags != 0 ? observed_flags : eviction_capture.flags, "buffer eviction"); } old_pcm_mode = eviction_capture.pcm_state; - release_pcm_holder = cluster_pcm_is_active() - && cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(oldTag)) - && (old_pcm_mode == (uint8) PCM_LOCK_MODE_S || old_pcm_mode == (uint8) PCM_LOCK_MODE_X); + release_pcm_holder + = cluster_pcm_is_active() && cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(oldTag)) + && (old_pcm_mode == (uint8)PCM_LOCK_MODE_S || old_pcm_mode == (uint8)PCM_LOCK_MODE_X); #endif InvalidateBufferCommitTailLocked(buf, oldTag, oldHash, oldPartitionLock, buf_state, @@ -4012,10 +3823,10 @@ InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, */ static void InvalidateBufferCommitTailLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, - LWLock *oldPartitionLock, uint32 buf_state, - uint8 old_pcm_mode, bool release_pcm_holder) + LWLock *oldPartitionLock, uint32 buf_state, uint8 old_pcm_mode, + bool release_pcm_holder) { - uint32 oldFlags; + uint32 oldFlags; /* * Clear out the buffer's tag and flags. We must do this to ensure that @@ -4032,7 +3843,7 @@ InvalidateBufferCommitTailLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldH * inherit a stale BUF_TYPE_PI (the PI shape must stay reachable ONLY * through cluster_bufmgr_convert_to_pi_locked, or the D-h3 shadow stamp * could be paired with another residency's bytes). */ - buf->buffer_type = (uint8) BUF_TYPE_CURRENT; + buf->buffer_type = (uint8)BUF_TYPE_CURRENT; #endif UnlockBufHdr(buf, buf_state); @@ -4053,18 +3864,17 @@ InvalidateBufferCommitTailLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldH * immutable capture avoids the old temporary tag restoration race. On a * throwing release the buffer is absent from the mapping table but is not * yet reusable, which is the fail-closed state. */ - if (release_pcm_holder) - { + if (release_pcm_holder) { PG_TRY(); { - cluster_pcm_lock_release_saved_tag_for_eviction(*oldTag, (PcmLockMode) old_pcm_mode); + cluster_pcm_lock_release_saved_tag_for_eviction(*oldTag, (PcmLockMode)old_pcm_mode); } PG_CATCH(); { elog(LOG, "cluster PCM saved-tag eviction release failed for buffer %d mode=%d; " "old mapping is removed and descriptor was not returned to the freelist", - buf->buf_id, (int) old_pcm_mode); + buf->buf_id, (int)old_pcm_mode); PG_RE_THROW(); } PG_END_TRY(); @@ -4095,10 +3905,10 @@ InvalidateBufferCommitTailLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldH static bool InvalidateBufferTry(BufferDesc *buf) { - BufferTag oldTag; - uint32 oldHash; /* hash value for oldTag */ - LWLock *oldPartitionLock; /* buffer partition lock for it */ - uint32 buf_state; + BufferTag oldTag; + uint32 oldHash; /* hash value for oldTag */ + LWLock *oldPartitionLock; /* buffer partition lock for it */ + uint32 buf_state; /* Save the original buffer tag before dropping the spinlock */ oldTag = buf->tag; @@ -4122,8 +3932,7 @@ InvalidateBufferTry(BufferDesc *buf) * deterministically. Gated to GCS drops only — plain evictions must * not stall. No lock is held across the sleep. */ - if (cluster_bufmgr_in_gcs_drop - && BufTagGetForkNum(&oldTag) == MAIN_FORKNUM) + if (cluster_bufmgr_in_gcs_drop && BufTagGetForkNum(&oldTag) == MAIN_FORKNUM) CLUSTER_INJECTION_POINT("cluster-pcm-drop-prepin-window"); #endif @@ -4136,21 +3945,19 @@ InvalidateBufferTry(BufferDesc *buf) buf_state = LockBufHdr(buf); /* If it's changed while we were waiting for lock, do nothing */ - if (!BufferTagsEqual(&buf->tag, &oldTag)) - { + if (!BufferTagsEqual(&buf->tag, &oldTag)) { UnlockBufHdr(buf, buf_state); LWLockRelease(oldPartitionLock); return true; } - if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) - { + if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) { UnlockBufHdr(buf, buf_state); LWLockRelease(oldPartitionLock); /* safety check: should definitely not be our *own* pin */ if (GetPrivateRefCount(BufferDescriptorGetBuffer(buf)) > 0) elog(ERROR, "buffer is pinned in InvalidateBufferTry"); - return false; /* foreign pin — caller parks / fail-closes */ + return false; /* foreign pin — caller parks / fail-closes */ } return InvalidateBufferCommitLocked(buf, &oldTag, oldHash, oldPartitionLock, buf_state); @@ -4168,10 +3975,10 @@ InvalidateBufferTry(BufferDesc *buf) static bool InvalidateVictimBuffer(BufferDesc *buf_hdr) { - uint32 buf_state; - uint32 hash; - LWLock *partition_lock; - BufferTag tag; + uint32 buf_state; + uint32 hash; + LWLock *partition_lock; + BufferTag tag; Assert(GetPrivateRefCount(BufferDescriptorGetBuffer(buf_hdr)) == 1); @@ -4198,8 +4005,7 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) /* PCM-X retained images deliberately keep BM_VALID and may still have this * victim pin. The live REVOKING token, not passive refcount, is the reuse * authority; never let the clock-sweep shortcut bypass D5a. */ - if (cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(buf_hdr, buf_state)) - { + if (cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(buf_hdr, buf_state)) { UnlockBufHdr(buf_hdr, buf_state); LWLockRelease(partition_lock); return false; @@ -4210,8 +4016,7 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) * If somebody else pinned the buffer since, or even worse, dirtied it, * give up on this buffer: It's clearly in use. */ - if (BUF_STATE_GET_REFCOUNT(buf_state) != 1 || (buf_state & BM_DIRTY)) - { + if (BUF_STATE_GET_REFCOUNT(buf_state) != 1 || (buf_state & BM_DIRTY)) { Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0); UnlockBufHdr(buf_hdr, buf_state); @@ -4235,7 +4040,7 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) * PGRAC: spec-6.12h D-h3a — victim reuse retags this buffer; reset the * cluster copy label under the header lock (same stale-BUF_TYPE_PI * containment as InvalidateBuffer above). */ - buf_hdr->buffer_type = (uint8) BUF_TYPE_CURRENT; + buf_hdr->buffer_type = (uint8)BUF_TYPE_CURRENT; #endif UnlockBufHdr(buf_hdr, buf_state); @@ -4249,13 +4054,11 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) * the cache-residency bit + propagate master_holder lifecycle (HC110) * before BufTableDelete completes the eviction. */ - if (cluster_pcm_is_active() - && cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(&tag)) - && buf_hdr->pcm_state != (uint8) PCM_STATE_N) - { - PcmLockMode old_mode = (PcmLockMode) buf_hdr->pcm_state; + if (cluster_pcm_is_active() && cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(&tag)) + && buf_hdr->pcm_state != (uint8)PCM_STATE_N) { + PcmLockMode old_mode = (PcmLockMode)buf_hdr->pcm_state; - buf_hdr->tag = tag; /* restore for release helper (cleared above) */ + buf_hdr->tag = tag; /* restore for release helper (cleared above) */ cluster_pcm_lock_release_buffer_for_eviction(buf_hdr, old_mode); ClearBufferTag(&buf_hdr->tag); @@ -4263,7 +4066,7 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) * PGRAC ownership-gen: coherent N-flip + generation bump (the header * spinlock was dropped above at UnlockBufHdr) so a buf_id reuse after * this victim eviction cannot alias a stale captured generation. */ - cluster_pcm_own_transition(buf_hdr, (uint8) PCM_STATE_N, 0, + cluster_pcm_own_transition(buf_hdr, (uint8)PCM_STATE_N, 0, PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING); } #endif @@ -4284,9 +4087,9 @@ static Buffer GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) { BufferDesc *buf_hdr; - Buffer buf; - uint32 buf_state; - bool from_ring; + Buffer buf; + uint32 buf_state; + bool from_ring; /* * Ensure, while the spinlock's not yet held, that there's a free refcount @@ -4322,9 +4125,8 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) * our share-lock won't prevent hint-bit updates). We will recheck the * dirty bit after re-locking the buffer header. */ - if (buf_state & BM_DIRTY) - { - LWLock *content_lock; + if (buf_state & BM_DIRTY) { + LWLock *content_lock; Assert(buf_state & BM_TAG_VALID); Assert(buf_state & BM_VALID); @@ -4344,8 +4146,7 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) * from StrategyGetBuffer.) */ content_lock = BufferDescriptorGetContentLock(buf_hdr); - if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) - { + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) { /* * Someone else has locked the buffer, so give it up and loop back * to get another one. @@ -4361,18 +4162,15 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) * lock to inspect the page LSN, so this can't be done inside * StrategyGetBuffer. */ - if (strategy != NULL) - { - XLogRecPtr lsn; + if (strategy != NULL) { + XLogRecPtr lsn; /* Read the LSN while holding buffer header lock */ buf_state = LockBufHdr(buf_hdr); lsn = BufferGetLSN(buf_hdr); UnlockBufHdr(buf_hdr, buf_state); - if (XLogNeedsFlush(lsn) - && StrategyRejectBuffer(strategy, buf_hdr, from_ring)) - { + if (XLogNeedsFlush(lsn) && StrategyRejectBuffer(strategy, buf_hdr, from_ring)) { LWLockRelease(content_lock); UnpinBuffer(buf_hdr); goto again; @@ -4383,13 +4181,11 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) FlushBuffer(buf_hdr, NULL, IOOBJECT_RELATION, io_context); LWLockRelease(content_lock); - ScheduleBufferTagForWriteback(&BackendWritebackContext, io_context, - &buf_hdr->tag); + ScheduleBufferTagForWriteback(&BackendWritebackContext, io_context, &buf_hdr->tag); } - if (buf_state & BM_VALID) - { + if (buf_state & BM_VALID) { /* * When a BufferAccessStrategy is in use, blocks evicted from shared * buffers are counted as IOOP_EVICT in the corresponding context @@ -4406,8 +4202,7 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) * we may have been forced to release the buffer due to concurrent * pinners or erroring out. */ - pgstat_count_io_op(IOOBJECT_RELATION, io_context, - from_ring ? IOOP_REUSE : IOOP_EVICT); + pgstat_count_io_op(IOOBJECT_RELATION, io_context, from_ring ? IOOP_REUSE : IOOP_EVICT); } /* @@ -4415,8 +4210,7 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) * can fail because another backend could have pinned or dirtied the * buffer. */ - if ((buf_state & BM_TAG_VALID) && !InvalidateVictimBuffer(buf_hdr)) - { + if ((buf_state & BM_TAG_VALID) && !InvalidateVictimBuffer(buf_hdr)) { UnpinBuffer(buf_hdr); goto again; } @@ -4449,8 +4243,8 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) static void LimitAdditionalPins(uint32 *additional_pins) { - uint32 max_backends; - int max_proportional_pins; + uint32 max_backends; + int max_proportional_pins; if (*additional_pins <= 1) return; @@ -4478,41 +4272,28 @@ LimitAdditionalPins(uint32 *additional_pins) * avoid duplicating the tracing and relpersistence related logic. */ static BlockNumber -ExtendBufferedRelCommon(BufferManagerRelation bmr, - ForkNumber fork, - BufferAccessStrategy strategy, - uint32 flags, - uint32 extend_by, - BlockNumber extend_upto, - Buffer *buffers, +ExtendBufferedRelCommon(BufferManagerRelation bmr, ForkNumber fork, BufferAccessStrategy strategy, + uint32 flags, uint32 extend_by, BlockNumber extend_upto, Buffer *buffers, uint32 *extended_by) { BlockNumber first_block; - TRACE_POSTGRESQL_BUFFER_EXTEND_START(fork, - bmr.smgr->smgr_rlocator.locator.spcOid, - bmr.smgr->smgr_rlocator.locator.dbOid, - bmr.smgr->smgr_rlocator.locator.relNumber, - bmr.smgr->smgr_rlocator.backend, - extend_by); + TRACE_POSTGRESQL_BUFFER_EXTEND_START( + fork, bmr.smgr->smgr_rlocator.locator.spcOid, bmr.smgr->smgr_rlocator.locator.dbOid, + bmr.smgr->smgr_rlocator.locator.relNumber, bmr.smgr->smgr_rlocator.backend, extend_by); if (bmr.relpersistence == RELPERSISTENCE_TEMP) - first_block = ExtendBufferedRelLocal(bmr, fork, flags, - extend_by, extend_upto, - buffers, &extend_by); + first_block + = ExtendBufferedRelLocal(bmr, fork, flags, extend_by, extend_upto, buffers, &extend_by); else - first_block = ExtendBufferedRelShared(bmr, fork, strategy, flags, - extend_by, extend_upto, + first_block = ExtendBufferedRelShared(bmr, fork, strategy, flags, extend_by, extend_upto, buffers, &extend_by); *extended_by = extend_by; - TRACE_POSTGRESQL_BUFFER_EXTEND_DONE(fork, - bmr.smgr->smgr_rlocator.locator.spcOid, + TRACE_POSTGRESQL_BUFFER_EXTEND_DONE(fork, bmr.smgr->smgr_rlocator.locator.spcOid, bmr.smgr->smgr_rlocator.locator.dbOid, bmr.smgr->smgr_rlocator.locator.relNumber, - bmr.smgr->smgr_rlocator.backend, - *extended_by, - first_block); + bmr.smgr->smgr_rlocator.backend, *extended_by, first_block); return first_block; } @@ -4522,18 +4303,13 @@ ExtendBufferedRelCommon(BufferManagerRelation bmr, * shared buffers. */ static BlockNumber -ExtendBufferedRelShared(BufferManagerRelation bmr, - ForkNumber fork, - BufferAccessStrategy strategy, - uint32 flags, - uint32 extend_by, - BlockNumber extend_upto, - Buffer *buffers, +ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccessStrategy strategy, + uint32 flags, uint32 extend_by, BlockNumber extend_upto, Buffer *buffers, uint32 *extended_by) { BlockNumber first_block; - IOContext io_context = IOContextForStrategy(strategy); - instr_time io_start; + IOContext io_context = IOContextForStrategy(strategy); + instr_time io_start; #ifdef USE_PGRAC_CLUSTER /*---------- @@ -4565,14 +4341,13 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, *---------- */ ClusterHwClass hwc = CLUSTER_HW_NATIVE_LOCAL; - HwLock hwlk; + HwLock hwlk; ClusterResId hw_resid; - uint32 hw_lease_tail = 0; /* spec-6.12d: parked grant tail */ + uint32 hw_lease_tail = 0; /* spec-6.12d: parked grant tail */ if (cluster_relation_extend_lock_enabled && bmr.rel != NULL && cluster_node_id >= 0 && fork == MAIN_FORKNUM && !RecoveryInProgress() - && bmr.rel->rd_rel->relpersistence != RELPERSISTENCE_TEMP) - { + && bmr.rel->rd_rel->relpersistence != RELPERSISTENCE_TEMP) { /* * Decide whether to engage the cross-node authority from runtime * liveness, not the static configured node count (spec-5.7 §3.1d, @@ -4583,14 +4358,13 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ClusterExtendEngage engage = cluster_extend_liveness_engage(true); if (engage == CLUSTER_EXTEND_ENGAGE_FAIL_CLOSED) - ereport(ERROR, - (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), - errmsg("could not acquire the cluster relation-extend lock for \"%s\"", - RelationGetRelationName(bmr.rel)), - errdetail("The cluster coordination substrate is not ready and an alive peer could not be ruled out."))); + ereport(ERROR, (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), + errmsg("could not acquire the cluster relation-extend lock for \"%s\"", + RelationGetRelationName(bmr.rel)), + errdetail("The cluster coordination substrate is not ready and an " + "alive peer could not be ruled out."))); - if (engage == CLUSTER_EXTEND_ENGAGE_COORDINATE) - { + if (engage == CLUSTER_EXTEND_ENGAGE_COORDINATE) { /* * A peer is alive and the substrate is ready: a permanent * relation is GLOBALIZE (authority-owned from block 0); an @@ -4599,11 +4373,12 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, */ hwc = cluster_hw_classify_persistence(bmr.rel->rd_rel->relpersistence, true); if (hwc == CLUSTER_HW_FAIL_CLOSED) - ereport(ERROR, - (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), - errmsg("unlogged relation \"%s\" cannot be safely extended in a multi-node cluster", - RelationGetRelationName(bmr.rel)), - errhint("Unlogged relations have no WAL authority to coordinate cross-node extension."))); + ereport(ERROR, (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), + errmsg("unlogged relation \"%s\" cannot be safely extended in a " + "multi-node cluster", + RelationGetRelationName(bmr.rel)), + errhint("Unlogged relations have no WAL authority to coordinate " + "cross-node extension."))); } /* @@ -4625,15 +4400,14 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, * These pages are pinned by us and not valid. While we hold the pin they * can't be acquired as victim buffers by another backend. */ - for (uint32 i = 0; i < extend_by; i++) - { - Block buf_block; + for (uint32 i = 0; i < extend_by; i++) { + Block buf_block; buffers[i] = GetVictimBuffer(strategy, io_context); buf_block = BufHdrGetBlock(GetBufferDescriptor(buffers[i] - 1)); /* new buffers are zero-filled */ - MemSet((char *) buf_block, 0, BLCKSZ); + MemSet((char *)buf_block, 0, BLCKSZ); } /* in case we need to pin an existing buffer below */ @@ -4661,31 +4435,26 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, * reconfig-revoke (AD-013) + backend-exit cleanup as the backstop on an * error path, mirroring CF (spec-5.6). */ - if (hwc == CLUSTER_HW_GLOBALIZE) - { + if (hwc == CLUSTER_HW_GLOBALIZE) { cluster_hw_resid_encode(RelationGetSmgr(bmr.rel)->smgr_rlocator.locator, fork, &hw_resid); - if (!cluster_hw_lock(&hw_resid, &hwlk)) - { + if (!cluster_hw_lock(&hw_resid, &hwlk)) { /* release the victim buffers pinned above (the extension lock is not * held yet) before failing closed, symmetric with the allocate-fail * path below (review P2). */ - for (uint32 i = 0; i < extend_by; i++) - { + for (uint32 i = 0; i < extend_by; i++) { BufferDesc *buf_hdr = GetBufferDescriptor(buffers[i] - 1); StrategyFreeBuffer(buf_hdr); UnpinBuffer(buf_hdr); } - ereport(ERROR, - (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), - errmsg("could not acquire the cluster relation-extend lock for \"%s\"", - RelationGetRelationName(bmr.rel)))); + ereport(ERROR, (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), + errmsg("could not acquire the cluster relation-extend lock for \"%s\"", + RelationGetRelationName(bmr.rel)))); } } #endif - if (!(flags & EB_SKIP_EXTENSION_LOCK)) - { + if (!(flags & EB_SKIP_EXTENSION_LOCK)) { LockRelationForExtension(bmr.rel, ExclusiveLock); if (bmr.rel) bmr.smgr = RelationGetSmgr(bmr.rel); @@ -4699,9 +4468,8 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, bmr.smgr->smgr_cached_nblocks[fork] = InvalidBlockNumber; #ifdef USE_PGRAC_CLUSTER - if (hwc == CLUSTER_HW_GLOBALIZE) - { - uint32 hw_granted = 0; + if (hwc == CLUSTER_HW_GLOBALIZE) { + uint32 hw_granted = 0; BlockNumber seed_nblocks; /* @@ -4736,52 +4504,45 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, * the only shape the hio.c lease consumer serves. */ { - uint32 hw_want = extend_by; + uint32 hw_want = extend_by; - if (cluster_hw_lease_active() && - extend_upto == InvalidBlockNumber && - bmr.rel->rd_rel->relkind == RELKIND_RELATION && - (uint32) cluster_space_lease_blocks > extend_by) - hw_want = (uint32) cluster_space_lease_blocks; + if (cluster_hw_lease_active() && extend_upto == InvalidBlockNumber + && bmr.rel->rd_rel->relkind == RELKIND_RELATION + && (uint32)cluster_space_lease_blocks > extend_by) + hw_want = (uint32)cluster_space_lease_blocks; first_block = cluster_hw_allocate(RelationGetSmgr(bmr.rel)->smgr_rlocator.locator, fork, hw_want, seed_nblocks, &hw_granted); } cluster_hw_unlock(&hwlk); - if (first_block == InvalidBlockNumber || hw_granted == 0) - { + if (first_block == InvalidBlockNumber || hw_granted == 0) { /* fail closed: release the extension lock + all victim buffers, then * raise 53RA6 (we are outside any critical section). */ if (!(flags & EB_SKIP_EXTENSION_LOCK)) UnlockRelationForExtension(bmr.rel, ExclusiveLock); - for (uint32 i = 0; i < extend_by; i++) - { + for (uint32 i = 0; i < extend_by; i++) { BufferDesc *buf_hdr = GetBufferDescriptor(buffers[i] - 1); StrategyFreeBuffer(buf_hdr); UnpinBuffer(buf_hdr); } - ereport(ERROR, - (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), - errmsg("cluster relation-extend authority unavailable for \"%s\"", - RelationGetRelationName(bmr.rel)), - errhint("The HW_ALLOC round trip to the resource master could not be proven; retry."))); + ereport(ERROR, (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), + errmsg("cluster relation-extend authority unavailable for \"%s\"", + RelationGetRelationName(bmr.rel)), + errhint("The HW_ALLOC round trip to the resource master could not be " + "proven; retry."))); } - if (hw_granted < extend_by) - { - for (uint32 i = hw_granted; i < extend_by; i++) - { + if (hw_granted < extend_by) { + for (uint32 i = hw_granted; i < extend_by; i++) { BufferDesc *buf_hdr = GetBufferDescriptor(buffers[i] - 1); StrategyFreeBuffer(buf_hdr); UnpinBuffer(buf_hdr); } extend_by = hw_granted; - } - else - { + } else { /* * PGRAC: spec-6.12d -- the over-ask surplus becomes this node's * lease. It MUST also be zero-extended below: the HWM advance @@ -4792,10 +4553,9 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, */ hw_lease_tail = hw_granted - extend_by; } - } - else + } else #endif - first_block = smgrnblocks(bmr.smgr, fork); + first_block = smgrnblocks(bmr.smgr, fork); /* * Now that we have the accurate relation size, check if the caller wants @@ -4803,17 +4563,15 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, * extensions, we might have acquired too many buffers and need to release * them. */ - if (extend_upto != InvalidBlockNumber) - { - uint32 orig_extend_by = extend_by; + if (extend_upto != InvalidBlockNumber) { + uint32 orig_extend_by = extend_by; if (first_block > extend_upto) extend_by = 0; - else if ((uint64) first_block + extend_by > extend_upto) + else if ((uint64)first_block + extend_by > extend_upto) extend_by = extend_upto - first_block; - for (uint32 i = extend_by; i < orig_extend_by; i++) - { + for (uint32 i = extend_by; i < orig_extend_by; i++) { BufferDesc *buf_hdr = GetBufferDescriptor(buffers[i] - 1); /* @@ -4824,8 +4582,7 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, UnpinBuffer(buf_hdr); } - if (extend_by == 0) - { + if (extend_by == 0) { if (!(flags & EB_SKIP_EXTENSION_LOCK)) UnlockRelationForExtension(bmr.rel, ExclusiveLock); *extended_by = extend_by; @@ -4834,12 +4591,10 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, } /* Fail if relation is already at maximum possible length */ - if ((uint64) first_block + extend_by >= MaxBlockNumber) - ereport(ERROR, - (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg("cannot extend relation %s beyond %u blocks", - relpath(bmr.smgr->smgr_rlocator, fork), - MaxBlockNumber))); + if ((uint64)first_block + extend_by >= MaxBlockNumber) + ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("cannot extend relation %s beyond %u blocks", + relpath(bmr.smgr->smgr_rlocator, fork), MaxBlockNumber))); /* * Insert buffers into buffer table, mark as IO_IN_PROGRESS. @@ -4847,14 +4602,13 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, * This needs to happen before we extend the relation, because as soon as * we do, other backends can start to read in those pages. */ - for (int i = 0; i < extend_by; i++) - { - Buffer victim_buf = buffers[i]; + for (int i = 0; i < extend_by; i++) { + Buffer victim_buf = buffers[i]; BufferDesc *victim_buf_hdr = GetBufferDescriptor(victim_buf - 1); - BufferTag tag; - uint32 hash; - LWLock *partition_lock; - int existing_id; + BufferTag tag; + uint32 hash; + LWLock *partition_lock; + int existing_id; InitBufferTag(&tag, &bmr.smgr->smgr_rlocator.locator, fork, first_block + i); hash = BufTableHashCode(&tag); @@ -4878,11 +4632,10 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, * overwrite. Since the legitimate cases should always have left a * zero-filled buffer, complain if not PageIsNew. */ - if (existing_id >= 0) - { + if (existing_id >= 0) { BufferDesc *existing_hdr = GetBufferDescriptor(existing_id); - Block buf_block; - bool valid; + Block buf_block; + bool valid; /* * Pin the existing buffer before releasing the partition lock, @@ -4902,13 +4655,14 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, buffers[i] = BufferDescriptorGetBuffer(existing_hdr); buf_block = BufHdrGetBlock(existing_hdr); - if (valid && !PageIsNew((Page) buf_block)) + if (valid && !PageIsNew((Page)buf_block)) ereport(ERROR, (errmsg("unexpected data beyond EOF in block %u of relation %s", existing_hdr->tag.blockNum, relpath(bmr.smgr->smgr_rlocator, fork)), - errhint("This has been seen to occur with buggy kernels; consider updating your system."))); + errhint("This has been seen to occur with buggy kernels; consider " + "updating your system."))); - /* + /* * We *must* do smgr[zero]extend before succeeding, else the page * will not be reserved by the kernel, and the next P_NEW call * will decide to return the same page. Clear the BM_VALID bit, @@ -4920,31 +4674,27 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, */ #ifdef USE_PGRAC_CLUSTER { - uint32 cluster_state = LockBufHdr(existing_hdr); - bool blocked + uint32 cluster_state = LockBufHdr(existing_hdr); + bool blocked = cluster_bufmgr_pcm_x_retained_image_locked(existing_hdr, cluster_state) - || (cluster_pcm_own_flags_get(existing_hdr->buf_id) - & PCM_OWN_FLAG_REVOKING) != 0; + || (cluster_pcm_own_flags_get(existing_hdr->buf_id) & PCM_OWN_FLAG_REVOKING) + != 0; UnlockBufHdr(existing_hdr, cluster_state); if (blocked) - ereport(ERROR, - (errcode(ERRCODE_OBJECT_IN_USE), - errmsg("cannot reuse retained cluster PCM image during " - "relation extension"))); + ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), + errmsg("cannot reuse retained cluster PCM image during " + "relation extension"))); } #endif - do - { - uint32 buf_state = LockBufHdr(existing_hdr); + do { + uint32 buf_state = LockBufHdr(existing_hdr); buf_state &= ~BM_VALID; UnlockBufHdr(existing_hdr, buf_state); } while (!StartBufferIO(existing_hdr, true)); - } - else - { - uint32 buf_state; + } else { + uint32 buf_state; buf_state = LockBufHdr(victim_buf_hdr); @@ -4993,12 +4743,10 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, * no lease is installed and the range degrades to the documented * orphan-zero-page fail-safe once a later extend covers it. */ - if (hw_lease_tail > 0) - { - smgrzeroextend(bmr.smgr, fork, first_block + extend_by, - (int) hw_lease_tail, false); - cluster_hw_lease_install(bmr.smgr->smgr_rlocator.locator, fork, - first_block + extend_by, hw_lease_tail); + if (hw_lease_tail > 0) { + smgrzeroextend(bmr.smgr, fork, first_block + extend_by, (int)hw_lease_tail, false); + cluster_hw_lease_install(bmr.smgr->smgr_rlocator.locator, fork, first_block + extend_by, + hw_lease_tail); } #endif @@ -5012,36 +4760,32 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, if (!(flags & EB_SKIP_EXTENSION_LOCK)) UnlockRelationForExtension(bmr.rel, ExclusiveLock); - pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, IOOP_EXTEND, - io_start, extend_by); + pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, IOOP_EXTEND, io_start, extend_by); /* Set BM_VALID, terminate IO, and wake up any waiters */ - for (int i = 0; i < extend_by; i++) - { - Buffer buf = buffers[i]; + for (int i = 0; i < extend_by; i++) { + Buffer buf = buffers[i]; BufferDesc *buf_hdr = GetBufferDescriptor(buf - 1); - bool lock = false; + bool lock = false; if (flags & EB_LOCK_FIRST && i == 0) lock = true; - else if (flags & EB_LOCK_TARGET) - { + else if (flags & EB_LOCK_TARGET) { Assert(extend_upto != InvalidBlockNumber); if (first_block + i + 1 == extend_upto) lock = true; } - if (lock) - { + if (lock) { #ifdef USE_PGRAC_CLUSTER ClusterPcmDirectInitProof direct_init_proof; /* This exact buffer is in the range successfully zeroextended above * and still owns BM_IO_IN_PROGRESS + !BM_VALID. */ - cluster_bufmgr_pcm_arm_direct_init( - buf_hdr, CLUSTER_PCM_DIRECT_INIT_EXTEND, &direct_init_proof); - cluster_bufmgr_pcm_gate_direct_init( - buf_hdr, CLUSTER_PCM_DIRECT_INIT_EXTEND, &direct_init_proof); + cluster_bufmgr_pcm_arm_direct_init(buf_hdr, CLUSTER_PCM_DIRECT_INIT_EXTEND, + &direct_init_proof); + cluster_bufmgr_pcm_gate_direct_init(buf_hdr, CLUSTER_PCM_DIRECT_INIT_EXTEND, + &direct_init_proof); #endif LWLockAcquire(BufferDescriptorGetContentLock(buf_hdr), LW_EXCLUSIVE); } @@ -5069,14 +4813,13 @@ void MarkBufferDirty(Buffer buffer) { BufferDesc *bufHdr; - uint32 buf_state; - uint32 old_buf_state; + uint32 buf_state; + uint32 old_buf_state; if (!BufferIsValid(buffer)) elog(ERROR, "bad buffer ID: %d", buffer); - if (BufferIsLocal(buffer)) - { + if (BufferIsLocal(buffer)) { MarkLocalBufferDirty(buffer); return; } @@ -5084,8 +4827,7 @@ MarkBufferDirty(Buffer buffer) bufHdr = GetBufferDescriptor(buffer - 1); Assert(BufferIsPinned(buffer)); - Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr), - LW_EXCLUSIVE)); + Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr), LW_EXCLUSIVE)); #ifdef USE_PGRAC_CLUSTER @@ -5095,20 +4837,17 @@ MarkBufferDirty(Buffer buffer) * retain/release boundary. */ buf_state = LockBufHdr(bufHdr); - if (cluster_bufmgr_pcm_x_retained_image_locked(bufHdr, buf_state)) - { + if (cluster_bufmgr_pcm_x_retained_image_locked(bufHdr, buf_state)) { UnlockBufHdr(bufHdr, buf_state); - ereport(ERROR, - (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("cannot dirty a retained cluster PCM image"), - errdetail("buffer=%d", bufHdr->buf_id))); + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot dirty a retained cluster PCM image"), + errdetail("buffer=%d", bufHdr->buf_id))); } UnlockBufHdr(bufHdr, buf_state); #endif old_buf_state = pg_atomic_read_u32(&bufHdr->state); - for (;;) - { + for (;;) { if (old_buf_state & BM_LOCKED) old_buf_state = WaitBufHdrUnlocked(bufHdr); @@ -5117,16 +4856,14 @@ MarkBufferDirty(Buffer buffer) Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0); buf_state |= BM_DIRTY | BM_JUST_DIRTIED; - if (pg_atomic_compare_exchange_u32(&bufHdr->state, &old_buf_state, - buf_state)) + if (pg_atomic_compare_exchange_u32(&bufHdr->state, &old_buf_state, buf_state)) break; } /* * If the buffer was not dirty already, do vacuum accounting. */ - if (!(old_buf_state & BM_DIRTY)) - { + if (!(old_buf_state & BM_DIRTY)) { VacuumPageDirty++; pgBufferUsage.shared_blks_dirtied++; if (VacuumCostActive) @@ -5148,32 +4885,26 @@ MarkBufferDirty(Buffer buffer) * but can save some tests in the caller. */ Buffer -ReleaseAndReadBuffer(Buffer buffer, - Relation relation, - BlockNumber blockNum) +ReleaseAndReadBuffer(Buffer buffer, Relation relation, BlockNumber blockNum) { - ForkNumber forkNum = MAIN_FORKNUM; + ForkNumber forkNum = MAIN_FORKNUM; BufferDesc *bufHdr; - if (BufferIsValid(buffer)) - { + if (BufferIsValid(buffer)) { Assert(BufferIsPinned(buffer)); - if (BufferIsLocal(buffer)) - { + if (BufferIsLocal(buffer)) { bufHdr = GetLocalBufferDescriptor(-buffer - 1); - if (bufHdr->tag.blockNum == blockNum && - BufTagMatchesRelFileLocator(&bufHdr->tag, &relation->rd_locator) && - BufTagGetForkNum(&bufHdr->tag) == forkNum) + if (bufHdr->tag.blockNum == blockNum + && BufTagMatchesRelFileLocator(&bufHdr->tag, &relation->rd_locator) + && BufTagGetForkNum(&bufHdr->tag) == forkNum) return buffer; UnpinLocalBuffer(buffer); - } - else - { + } else { bufHdr = GetBufferDescriptor(buffer - 1); /* we have pin, so it's ok to examine tag without spinlock */ - if (bufHdr->tag.blockNum == blockNum && - BufTagMatchesRelFileLocator(&bufHdr->tag, &relation->rd_locator) && - BufTagGetForkNum(&bufHdr->tag) == forkNum) + if (bufHdr->tag.blockNum == blockNum + && BufTagMatchesRelFileLocator(&bufHdr->tag, &relation->rd_locator) + && BufTagGetForkNum(&bufHdr->tag) == forkNum) return buffer; UnpinBuffer(bufHdr); } @@ -5207,25 +4938,23 @@ ReleaseAndReadBuffer(Buffer buffer, static bool PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy) { - Buffer b = BufferDescriptorGetBuffer(buf); - bool result; + Buffer b = BufferDescriptorGetBuffer(buf); + bool result; PrivateRefCountEntry *ref; Assert(!BufferIsLocal(b)); ref = GetPrivateRefCountEntry(b, true); - if (ref == NULL) - { - uint32 buf_state; - uint32 old_buf_state; + if (ref == NULL) { + uint32 buf_state; + uint32 old_buf_state; ReservePrivateRefCountEntry(); ref = NewPrivateRefCountEntry(b); old_buf_state = pg_atomic_read_u32(&buf->state); - for (;;) - { + for (;;) { if (old_buf_state & BM_LOCKED) old_buf_state = WaitBufHdrUnlocked(buf); @@ -5234,14 +4963,11 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy) /* increase refcount */ buf_state += BUF_REFCOUNT_ONE; - if (strategy == NULL) - { + if (strategy == NULL) { /* Default case: increase usagecount unless already max. */ if (BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT) buf_state += BUF_USAGECOUNT_ONE; - } - else - { + } else { /* * Ring buffers shouldn't evict others from pool. Thus we * don't make usagecount more than 1. @@ -5250,9 +4976,7 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy) buf_state += BUF_USAGECOUNT_ONE; } - if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state, - buf_state)) - { + if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state, buf_state)) { result = (buf_state & BM_VALID) != 0; /* @@ -5266,9 +4990,7 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy) break; } } - } - else - { + } else { /* * If we previously pinned the buffer, it must surely be valid. * @@ -5312,9 +5034,9 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy) static void PinBuffer_Locked(BufferDesc *buf) { - Buffer b; + Buffer b; PrivateRefCountEntry *ref; - uint32 buf_state; + uint32 buf_state; /* * As explained, We don't expect any preexisting pins. That allows us to @@ -5356,7 +5078,7 @@ static void UnpinBuffer(BufferDesc *buf) { PrivateRefCountEntry *ref; - Buffer b = BufferDescriptorGetBuffer(buf); + Buffer b = BufferDescriptorGetBuffer(buf); Assert(!BufferIsLocal(b)); @@ -5368,10 +5090,9 @@ UnpinBuffer(BufferDesc *buf) Assert(ref->refcount > 0); ref->refcount--; - if (ref->refcount == 0) - { - uint32 buf_state; - uint32 old_buf_state; + if (ref->refcount == 0) { + uint32 buf_state; + uint32 old_buf_state; /* * Mark buffer non-accessible to Valgrind. @@ -5392,8 +5113,7 @@ UnpinBuffer(BufferDesc *buf) * it's not safe to use atomic decrement here; thus use a CAS loop. */ old_buf_state = pg_atomic_read_u32(&buf->state); - for (;;) - { + for (;;) { if (old_buf_state & BM_LOCKED) old_buf_state = WaitBufHdrUnlocked(buf); @@ -5401,14 +5121,12 @@ UnpinBuffer(BufferDesc *buf) buf_state -= BUF_REFCOUNT_ONE; - if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state, - buf_state)) + if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state, buf_state)) break; } /* Support LockBufferForCleanup() */ - if (buf_state & BM_PIN_COUNT_WAITER) - { + if (buf_state & BM_PIN_COUNT_WAITER) { /* * Acquire the buffer header lock, re-check that there's a waiter. * Another backend could have unpinned this buffer, and already @@ -5418,17 +5136,14 @@ UnpinBuffer(BufferDesc *buf) */ buf_state = LockBufHdr(buf); - if ((buf_state & BM_PIN_COUNT_WAITER) && - BUF_STATE_GET_REFCOUNT(buf_state) == 1) - { + if ((buf_state & BM_PIN_COUNT_WAITER) && BUF_STATE_GET_REFCOUNT(buf_state) == 1) { /* we just released the last pin other than the waiter's */ - int wait_backend_pgprocno = buf->wait_backend_pgprocno; + int wait_backend_pgprocno = buf->wait_backend_pgprocno; buf_state &= ~BM_PIN_COUNT_WAITER; UnlockBufHdr(buf, buf_state); ProcSendSignal(wait_backend_pgprocno); - } - else + } else UnlockBufHdr(buf, buf_state); } ForgetPrivateRefCountEntry(ref); @@ -5455,17 +5170,17 @@ UnpinBuffer(BufferDesc *buf) static void BufferSync(int flags) { - uint32 buf_state; - int buf_id; - int num_to_scan; - int num_spaces; - int num_processed; - int num_written; + uint32 buf_state; + int buf_id; + int num_to_scan; + int num_spaces; + int num_processed; + int num_written; CkptTsStatus *per_ts_stat = NULL; - Oid last_tsid; + Oid last_tsid; binaryheap *ts_heap; - int i; - int mask = BM_DIRTY; + int i; + int mask = BM_DIRTY; WritebackContext wb_context; /* Make sure we can handle the pin inside SyncOneBuffer */ @@ -5476,8 +5191,7 @@ BufferSync(int flags) * we write only permanent, dirty buffers. But at shutdown or end of * recovery, we write all dirty buffers. */ - if (!((flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY | - CHECKPOINT_FLUSH_ALL)))) + if (!((flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY | CHECKPOINT_FLUSH_ALL)))) mask |= BM_PERMANENT; /* @@ -5497,8 +5211,7 @@ BufferSync(int flags) * certainly need to be written for the next checkpoint attempt, too. */ num_to_scan = 0; - for (buf_id = 0; buf_id < NBuffers; buf_id++) - { + for (buf_id = 0; buf_id < NBuffers; buf_id++) { BufferDesc *bufHdr = GetBufferDescriptor(buf_id); /* @@ -5507,8 +5220,7 @@ BufferSync(int flags) */ buf_state = LockBufHdr(bufHdr); - if ((buf_state & mask) == mask) - { + if ((buf_state & mask) == mask) { CkptSortItem *item; buf_state |= BM_CHECKPOINT_NEEDED; @@ -5529,7 +5241,7 @@ BufferSync(int flags) } if (num_to_scan == 0) - return; /* nothing to do */ + return; /* nothing to do */ WritebackContextInit(&wb_context, &checkpoint_flush_after); @@ -5551,10 +5263,9 @@ BufferSync(int flags) * be flushed. This requires the to-be-flushed array to be sorted. */ last_tsid = InvalidOid; - for (i = 0; i < num_to_scan; i++) - { + for (i = 0; i < num_to_scan; i++) { CkptTsStatus *s; - Oid cur_tsid; + Oid cur_tsid; cur_tsid = CkptBufferIds[i].tsId; @@ -5562,9 +5273,8 @@ BufferSync(int flags) * Grow array of per-tablespace status structs, every time a new * tablespace is found. */ - if (last_tsid == InvalidOid || last_tsid != cur_tsid) - { - Size sz; + if (last_tsid == InvalidOid || last_tsid != cur_tsid) { + Size sz; num_spaces++; @@ -5575,9 +5285,9 @@ BufferSync(int flags) sz = sizeof(CkptTsStatus) * num_spaces; if (per_ts_stat == NULL) - per_ts_stat = (CkptTsStatus *) palloc(sz); + per_ts_stat = (CkptTsStatus *)palloc(sz); else - per_ts_stat = (CkptTsStatus *) repalloc(per_ts_stat, sz); + per_ts_stat = (CkptTsStatus *)repalloc(per_ts_stat, sz); s = &per_ts_stat[num_spaces - 1]; memset(s, 0, sizeof(*s)); @@ -5596,9 +5306,7 @@ BufferSync(int flags) */ last_tsid = cur_tsid; - } - else - { + } else { s = &per_ts_stat[num_spaces - 1]; } @@ -5616,15 +5324,12 @@ BufferSync(int flags) * and compute how large a portion of the total progress a single * processed buffer is. */ - ts_heap = binaryheap_allocate(num_spaces, - ts_ckpt_progress_comparator, - NULL); + ts_heap = binaryheap_allocate(num_spaces, ts_ckpt_progress_comparator, NULL); - for (i = 0; i < num_spaces; i++) - { + for (i = 0; i < num_spaces; i++) { CkptTsStatus *ts_stat = &per_ts_stat[i]; - ts_stat->progress_slice = (float8) num_to_scan / ts_stat->num_to_scan; + ts_stat->progress_slice = (float8)num_to_scan / ts_stat->num_to_scan; binaryheap_add_unordered(ts_heap, PointerGetDatum(ts_stat)); } @@ -5639,11 +5344,9 @@ BufferSync(int flags) */ num_processed = 0; num_written = 0; - while (!binaryheap_empty(ts_heap)) - { + while (!binaryheap_empty(ts_heap)) { BufferDesc *bufHdr = NULL; - CkptTsStatus *ts_stat = (CkptTsStatus *) - DatumGetPointer(binaryheap_first(ts_heap)); + CkptTsStatus *ts_stat = (CkptTsStatus *)DatumGetPointer(binaryheap_first(ts_heap)); buf_id = CkptBufferIds[ts_stat->index].buf_id; Assert(buf_id != -1); @@ -5664,10 +5367,8 @@ BufferSync(int flags) * write the buffer though we didn't need to. It doesn't seem worth * guarding against this, though. */ - if (pg_atomic_read_u32(&bufHdr->state) & BM_CHECKPOINT_NEEDED) - { - if (SyncOneBuffer(buf_id, false, &wb_context) & BUF_WRITTEN) - { + if (pg_atomic_read_u32(&bufHdr->state) & BM_CHECKPOINT_NEEDED) { + if (SyncOneBuffer(buf_id, false, &wb_context) & BUF_WRITTEN) { TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id); PendingCheckpointerStats.buf_written_checkpoints++; num_written++; @@ -5683,12 +5384,9 @@ BufferSync(int flags) ts_stat->index++; /* Have all the buffers from the tablespace been processed? */ - if (ts_stat->num_scanned == ts_stat->num_to_scan) - { + if (ts_stat->num_scanned == ts_stat->num_to_scan) { binaryheap_remove_first(ts_heap); - } - else - { + } else { /* update heap with the new progress */ binaryheap_replace_first(ts_heap, PointerGetDatum(ts_stat)); } @@ -5698,7 +5396,7 @@ BufferSync(int flags) * * (This will check for barrier events even if it doesn't sleep.) */ - CheckpointWriteDelay(flags, (double) num_processed / num_to_scan); + CheckpointWriteDelay(flags, (double)num_processed / num_to_scan); } /* @@ -5735,18 +5433,18 @@ bool BgBufferSync(WritebackContext *wb_context) { /* info obtained from freelist.c */ - int strategy_buf_id; - uint32 strategy_passes; - uint32 recent_alloc; + int strategy_buf_id; + uint32 strategy_passes; + uint32 recent_alloc; /* * Information saved between calls so we can determine the strategy * point's advance rate and avoid scanning already-cleaned buffers. */ static bool saved_info_valid = false; - static int prev_strategy_buf_id; + static int prev_strategy_buf_id; static uint32 prev_strategy_passes; - static int next_to_clean; + static int next_to_clean; static uint32 next_passes; /* Moving averages of allocation rate and clean-buffer density */ @@ -5754,26 +5452,26 @@ BgBufferSync(WritebackContext *wb_context) static float smoothed_density = 10.0; /* Potentially these could be tunables, but for now, not */ - float smoothing_samples = 16; - float scan_whole_pool_milliseconds = 120000.0; + float smoothing_samples = 16; + float scan_whole_pool_milliseconds = 120000.0; /* Used to compute how far we scan ahead */ - long strategy_delta; - int bufs_to_lap; - int bufs_ahead; - float scans_per_alloc; - int reusable_buffers_est; - int upcoming_alloc_est; - int min_scan_buffers; + long strategy_delta; + int bufs_to_lap; + int bufs_ahead; + float scans_per_alloc; + int reusable_buffers_est; + int upcoming_alloc_est; + int min_scan_buffers; /* Variables for the scanning loop proper */ - int num_to_scan; - int num_written; - int reusable_buffers; + int num_to_scan; + int num_written; + int reusable_buffers; /* Variables for final smoothed_density update */ - long new_strategy_delta; - uint32 new_recent_alloc; + long new_strategy_delta; + uint32 new_recent_alloc; /* * Find out where the freelist clock sweep currently is, and how many @@ -5789,8 +5487,7 @@ BgBufferSync(WritebackContext *wb_context) * stuff. We mark the saved state invalid so that we can recover sanely * if LRU scan is turned back on later. */ - if (bgwriter_lru_maxpages <= 0) - { + if (bgwriter_lru_maxpages <= 0) { saved_info_valid = false; return true; } @@ -5803,64 +5500,48 @@ BgBufferSync(WritebackContext *wb_context) * weird-looking coding of xxx_passes comparisons are to avoid bogus * behavior when the passes counts wrap around. */ - if (saved_info_valid) - { - int32 passes_delta = strategy_passes - prev_strategy_passes; + if (saved_info_valid) { + int32 passes_delta = strategy_passes - prev_strategy_passes; strategy_delta = strategy_buf_id - prev_strategy_buf_id; - strategy_delta += (long) passes_delta * NBuffers; + strategy_delta += (long)passes_delta * NBuffers; Assert(strategy_delta >= 0); - if ((int32) (next_passes - strategy_passes) > 0) - { + if ((int32)(next_passes - strategy_passes) > 0) { /* we're one pass ahead of the strategy point */ bufs_to_lap = strategy_buf_id - next_to_clean; #ifdef BGW_DEBUG - elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d", - next_passes, next_to_clean, - strategy_passes, strategy_buf_id, - strategy_delta, bufs_to_lap); + elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d", next_passes, + next_to_clean, strategy_passes, strategy_buf_id, strategy_delta, bufs_to_lap); #endif - } - else if (next_passes == strategy_passes && - next_to_clean >= strategy_buf_id) - { + } else if (next_passes == strategy_passes && next_to_clean >= strategy_buf_id) { /* on same pass, but ahead or at least not behind */ bufs_to_lap = NBuffers - (next_to_clean - strategy_buf_id); #ifdef BGW_DEBUG - elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d", - next_passes, next_to_clean, - strategy_passes, strategy_buf_id, - strategy_delta, bufs_to_lap); + elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d", next_passes, + next_to_clean, strategy_passes, strategy_buf_id, strategy_delta, bufs_to_lap); #endif - } - else - { + } else { /* * We're behind, so skip forward to the strategy point and start * cleaning from there. */ #ifdef BGW_DEBUG - elog(DEBUG2, "bgwriter behind: bgw %u-%u strategy %u-%u delta=%ld", - next_passes, next_to_clean, - strategy_passes, strategy_buf_id, - strategy_delta); + elog(DEBUG2, "bgwriter behind: bgw %u-%u strategy %u-%u delta=%ld", next_passes, + next_to_clean, strategy_passes, strategy_buf_id, strategy_delta); #endif next_to_clean = strategy_buf_id; next_passes = strategy_passes; bufs_to_lap = NBuffers; } - } - else - { + } else { /* * Initializing at startup or after LRU scanning had been off. Always * start at the strategy point. */ #ifdef BGW_DEBUG - elog(DEBUG2, "bgwriter initializing: strategy %u-%u", - strategy_passes, strategy_buf_id); + elog(DEBUG2, "bgwriter initializing: strategy %u-%u", strategy_passes, strategy_buf_id); #endif strategy_delta = 0; next_to_clean = strategy_buf_id; @@ -5879,11 +5560,9 @@ BgBufferSync(WritebackContext *wb_context) * * If the strategy point didn't move, we don't update the density estimate */ - if (strategy_delta > 0 && recent_alloc > 0) - { - scans_per_alloc = (float) strategy_delta / (float) recent_alloc; - smoothed_density += (scans_per_alloc - smoothed_density) / - smoothing_samples; + if (strategy_delta > 0 && recent_alloc > 0) { + scans_per_alloc = (float)strategy_delta / (float)recent_alloc; + smoothed_density += (scans_per_alloc - smoothed_density) / smoothing_samples; } /* @@ -5892,21 +5571,20 @@ BgBufferSync(WritebackContext *wb_context) * density estimate. */ bufs_ahead = NBuffers - bufs_to_lap; - reusable_buffers_est = (float) bufs_ahead / smoothed_density; + reusable_buffers_est = (float)bufs_ahead / smoothed_density; /* * Track a moving average of recent buffer allocations. Here, rather than * a true average we want a fast-attack, slow-decline behavior: we * immediately follow any increase. */ - if (smoothed_alloc <= (float) recent_alloc) + if (smoothed_alloc <= (float)recent_alloc) smoothed_alloc = recent_alloc; else - smoothed_alloc += ((float) recent_alloc - smoothed_alloc) / - smoothing_samples; + smoothed_alloc += ((float)recent_alloc - smoothed_alloc) / smoothing_samples; /* Scale the estimate by a GUC to allow more aggressive tuning. */ - upcoming_alloc_est = (int) (smoothed_alloc * bgwriter_lru_multiplier); + upcoming_alloc_est = (int)(smoothed_alloc * bgwriter_lru_multiplier); /* * If recent_alloc remains at zero for many cycles, smoothed_alloc will @@ -5929,10 +5607,9 @@ BgBufferSync(WritebackContext *wb_context) * the BGW will be called during the scan_whole_pool time; slice the * buffer pool into that many sections. */ - min_scan_buffers = (int) (NBuffers / (scan_whole_pool_milliseconds / BgWriterDelay)); + min_scan_buffers = (int)(NBuffers / (scan_whole_pool_milliseconds / BgWriterDelay)); - if (upcoming_alloc_est < (min_scan_buffers + reusable_buffers_est)) - { + if (upcoming_alloc_est < (min_scan_buffers + reusable_buffers_est)) { #ifdef BGW_DEBUG elog(DEBUG2, "bgwriter: alloc_est=%d too small, using min=%d + reusable_est=%d", upcoming_alloc_est, min_scan_buffers, reusable_buffers_est); @@ -5955,39 +5632,33 @@ BgBufferSync(WritebackContext *wb_context) reusable_buffers = reusable_buffers_est; /* Execute the LRU scan */ - while (num_to_scan > 0 && reusable_buffers < upcoming_alloc_est) - { - int sync_state = SyncOneBuffer(next_to_clean, true, - wb_context); + while (num_to_scan > 0 && reusable_buffers < upcoming_alloc_est) { + int sync_state = SyncOneBuffer(next_to_clean, true, wb_context); - if (++next_to_clean >= NBuffers) - { + if (++next_to_clean >= NBuffers) { next_to_clean = 0; next_passes++; } num_to_scan--; - if (sync_state & BUF_WRITTEN) - { + if (sync_state & BUF_WRITTEN) { reusable_buffers++; - if (++num_written >= bgwriter_lru_maxpages) - { + if (++num_written >= bgwriter_lru_maxpages) { PendingBgWriterStats.maxwritten_clean++; break; } - } - else if (sync_state & BUF_REUSABLE) + } else if (sync_state & BUF_REUSABLE) reusable_buffers++; } PendingBgWriterStats.buf_written_clean += num_written; #ifdef BGW_DEBUG - elog(DEBUG1, "bgwriter: recent_alloc=%u smoothed=%.2f delta=%ld ahead=%d density=%.2f reusable_est=%d upcoming_est=%d scanned=%d wrote=%d reusable=%d", - recent_alloc, smoothed_alloc, strategy_delta, bufs_ahead, - smoothed_density, reusable_buffers_est, upcoming_alloc_est, - bufs_to_lap - num_to_scan, - num_written, + elog(DEBUG1, + "bgwriter: recent_alloc=%u smoothed=%.2f delta=%ld ahead=%d density=%.2f reusable_est=%d " + "upcoming_est=%d scanned=%d wrote=%d reusable=%d", + recent_alloc, smoothed_alloc, strategy_delta, bufs_ahead, smoothed_density, + reusable_buffers_est, upcoming_alloc_est, bufs_to_lap - num_to_scan, num_written, reusable_buffers - reusable_buffers_est); #endif @@ -6001,16 +5672,13 @@ BgBufferSync(WritebackContext *wb_context) */ new_strategy_delta = bufs_to_lap - num_to_scan; new_recent_alloc = reusable_buffers - reusable_buffers_est; - if (new_strategy_delta > 0 && new_recent_alloc > 0) - { - scans_per_alloc = (float) new_strategy_delta / (float) new_recent_alloc; - smoothed_density += (scans_per_alloc - smoothed_density) / - smoothing_samples; + if (new_strategy_delta > 0 && new_recent_alloc > 0) { + scans_per_alloc = (float)new_strategy_delta / (float)new_recent_alloc; + smoothed_density += (scans_per_alloc - smoothed_density) / smoothing_samples; #ifdef BGW_DEBUG elog(DEBUG2, "bgwriter: cleaner density alloc=%u scan=%ld density=%.2f new smoothed=%.2f", - new_recent_alloc, new_strategy_delta, - scans_per_alloc, smoothed_density); + new_recent_alloc, new_strategy_delta, scans_per_alloc, smoothed_density); #endif } @@ -6038,9 +5706,9 @@ static int SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) { BufferDesc *bufHdr = GetBufferDescriptor(buf_id); - int result = 0; - uint32 buf_state; - BufferTag tag; + int result = 0; + uint32 buf_state; + BufferTag tag; ReservePrivateRefCountEntry(); @@ -6061,27 +5729,21 @@ SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) * A live retained image is neither reusable nor writable. Test before * advertising BUF_REUSABLE; finish/release serialize the later race with * the content-lock recheck below. */ - if (cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(bufHdr, buf_state)) - { + if (cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(bufHdr, buf_state)) { UnlockBufHdr(bufHdr, buf_state); return 0; } #endif - if (BUF_STATE_GET_REFCOUNT(buf_state) == 0 && - BUF_STATE_GET_USAGECOUNT(buf_state) == 0) - { + if (BUF_STATE_GET_REFCOUNT(buf_state) == 0 && BUF_STATE_GET_USAGECOUNT(buf_state) == 0) { result |= BUF_REUSABLE; - } - else if (skip_recently_used) - { + } else if (skip_recently_used) { /* Caller told us not to write recently-used buffers */ UnlockBufHdr(bufHdr, buf_state); return result; } - if (!(buf_state & BM_VALID) || !(buf_state & BM_DIRTY)) - { + if (!(buf_state & BM_VALID) || !(buf_state & BM_DIRTY)) { /* It's clean, so nothing to do */ UnlockBufHdr(bufHdr, buf_state); return result; @@ -6101,8 +5763,7 @@ SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) * the transition is now complete and stable. */ buf_state = LockBufHdr(bufHdr); - if (cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(bufHdr, buf_state)) - { + if (cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(bufHdr, buf_state)) { UnlockBufHdr(bufHdr, buf_state); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); UnpinBuffer(bufHdr); @@ -6166,7 +5827,7 @@ AtEOXact_Buffers(bool isCommit) void InitBufferPoolAccess(void) { - HASHCTL hash_ctl; + HASHCTL hash_ctl; memset(&PrivateRefCountArray, 0, sizeof(PrivateRefCountArray)); #ifdef USE_PGRAC_CLUSTER @@ -6177,8 +5838,7 @@ InitBufferPoolAccess(void) hash_ctl.keysize = sizeof(int32); hash_ctl.entrysize = sizeof(PrivateRefCountEntry); - PrivateRefCountHash = hash_create("PrivateRefCount", 100, &hash_ctl, - HASH_ELEM | HASH_BLOBS); + PrivateRefCountHash = hash_create("PrivateRefCount", 100, &hash_ctl, HASH_ELEM | HASH_BLOBS); /* * AtProcExit_Buffers needs LWLock access, and thereby has to be called at @@ -6218,30 +5878,26 @@ static void CheckForBufferLeaks(void) { #ifdef USE_ASSERT_CHECKING - int RefCountErrors = 0; + int RefCountErrors = 0; PrivateRefCountEntry *res; - int i; + int i; /* check the array */ - for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++) - { + for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++) { res = &PrivateRefCountArray[i]; - if (res->buffer != InvalidBuffer) - { + if (res->buffer != InvalidBuffer) { PrintBufferLeakWarning(res->buffer); RefCountErrors++; } } /* if necessary search the hash */ - if (PrivateRefCountOverflowed) - { + if (PrivateRefCountOverflowed) { HASH_SEQ_STATUS hstat; hash_seq_init(&hstat, PrivateRefCountHash); - while ((res = (PrivateRefCountEntry *) hash_seq_search(&hstat)) != NULL) - { + while ((res = (PrivateRefCountEntry *)hash_seq_search(&hstat)) != NULL) { PrintBufferLeakWarning(res->buffer); RefCountErrors++; } @@ -6276,11 +5932,10 @@ AssertBufferLocksPermitCatalogRead(void) } static void -AssertNotCatalogBufferLock(LWLock *lock, LWLockMode mode, - void *unused_context) +AssertNotCatalogBufferLock(LWLock *lock, LWLockMode mode, void *unused_context) { BufferDesc *bufHdr; - BufferTag tag; + BufferTag tag; Oid relid; if (mode != LW_EXCLUSIVE) @@ -6336,34 +5991,29 @@ void PrintBufferLeakWarning(Buffer buffer) { BufferDesc *buf; - int32 loccount; - char *path; - BackendId backend; - uint32 buf_state; + int32 loccount; + char *path; + BackendId backend; + uint32 buf_state; Assert(BufferIsValid(buffer)); - if (BufferIsLocal(buffer)) - { + if (BufferIsLocal(buffer)) { buf = GetLocalBufferDescriptor(-buffer - 1); loccount = LocalRefCount[-buffer - 1]; backend = MyBackendId; - } - else - { + } else { buf = GetBufferDescriptor(buffer - 1); loccount = GetPrivateRefCount(buffer); backend = InvalidBackendId; } /* theoretically we should lock the bufhdr here */ - path = relpathbackend(BufTagGetRelFileLocator(&buf->tag), backend, - BufTagGetForkNum(&buf->tag)); + path = relpathbackend(BufTagGetRelFileLocator(&buf->tag), backend, BufTagGetForkNum(&buf->tag)); buf_state = pg_atomic_read_u32(&buf->state); elog(WARNING, "buffer refcount leak: [%03d] " "(rel=%s, blockNum=%u, flags=0x%x, refcount=%u %d)", - buffer, path, - buf->tag.blockNum, buf_state & BUF_FLAG_MASK, + buffer, path, buf->tag.blockNum, buf_state & BUF_FLAG_MASK, BUF_STATE_GET_REFCOUNT(buf_state), loccount); pfree(path); } @@ -6412,8 +6062,7 @@ BufferGetBlockNumber(Buffer buffer) * a buffer. */ void -BufferGetTag(Buffer buffer, RelFileLocator *rlocator, ForkNumber *forknum, - BlockNumber *blknum) +BufferGetTag(Buffer buffer, RelFileLocator *rlocator, ForkNumber *forknum, BlockNumber *blknum) { BufferDesc *bufHdr; @@ -6451,15 +6100,14 @@ BufferGetTag(Buffer buffer, RelFileLocator *rlocator, ForkNumber *forknum, * as the second parameter. If not, pass NULL. */ static void -FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, - IOContext io_context) +FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, IOContext io_context) { - XLogRecPtr recptr; + XLogRecPtr recptr; ErrorContextCallback errcallback; - instr_time io_start; - Block bufBlock; - char *bufToWrite; - uint32 buf_state; + instr_time io_start; + Block bufBlock; + char *bufToWrite; + uint32 buf_state; #ifdef USE_PGRAC_CLUSTER @@ -6469,8 +6117,7 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, * enter StartBufferIO for old transfer bytes. */ buf_state = LockBufHdr(buf); - if (cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state)) - { + if (cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state)) { UnlockBufHdr(buf, buf_state); return; } @@ -6501,7 +6148,7 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, /* Setup error traceback support for ereport() */ errcallback.callback = shared_buffer_write_error_callback; - errcallback.arg = (void *) buf; + errcallback.arg = (void *)buf; errcallback.previous = error_context_stack; error_context_stack = &errcallback; @@ -6509,11 +6156,9 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, if (reln == NULL) reln = smgropen(BufTagGetRelFileLocator(&buf->tag), InvalidBackendId); - TRACE_POSTGRESQL_BUFFER_FLUSH_START(BufTagGetForkNum(&buf->tag), - buf->tag.blockNum, - reln->smgr_rlocator.locator.spcOid, - reln->smgr_rlocator.locator.dbOid, - reln->smgr_rlocator.locator.relNumber); + TRACE_POSTGRESQL_BUFFER_FLUSH_START( + BufTagGetForkNum(&buf->tag), buf->tag.blockNum, reln->smgr_rlocator.locator.spcOid, + reln->smgr_rlocator.locator.dbOid, reln->smgr_rlocator.locator.relNumber); buf_state = LockBufHdr(buf); @@ -6562,8 +6207,7 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, * the only local additions on top of a shipped image are hint-class * changes (ITL lazy cleanout, index kill bits), which emit no WAL. */ - if (cluster_storage_mode_enabled() && !RecoveryInProgress() - && recptr > GetXLogInsertRecPtr()) + if (cluster_storage_mode_enabled() && !RecoveryInProgress() && recptr > GetXLogInsertRecPtr()) recptr = InvalidXLogRecPtr; #endif if (buf_state & BM_PERMANENT) @@ -6581,18 +6225,14 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, * buffer, other processes might be updating hint bits in it, so we must * copy the page to private storage if we do checksumming. */ - bufToWrite = PageSetChecksumCopy((Page) bufBlock, buf->tag.blockNum); + bufToWrite = PageSetChecksumCopy((Page)bufBlock, buf->tag.blockNum); io_start = pgstat_prepare_io_time(); /* * bufToWrite is either the shared buffer or a copy, as appropriate. */ - smgrwrite(reln, - BufTagGetForkNum(&buf->tag), - buf->tag.blockNum, - bufToWrite, - false); + smgrwrite(reln, BufTagGetForkNum(&buf->tag), buf->tag.blockNum, bufToWrite, false); #ifdef USE_PGRAC_CLUSTER @@ -6610,9 +6250,8 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, * the master's watermark check and the holder's strict PI-only drop own * correctness). */ - if (cluster_past_image && buf->pcm_state != (uint8) PCM_STATE_N) - cluster_gcs_block_pi_write_note(buf->tag, - ((PageHeader) bufToWrite)->pd_block_scn); + if (cluster_past_image && buf->pcm_state != (uint8)PCM_STATE_N) + cluster_gcs_block_pi_write_note(buf->tag, ((PageHeader)bufToWrite)->pd_block_scn); #endif /* @@ -6633,8 +6272,7 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, * When a strategy is not in use, the write can only be a "regular" write * of a dirty shared buffer (IOCONTEXT_NORMAL IOOP_WRITE). */ - pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, - IOOP_WRITE, io_start, 1); + pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, IOOP_WRITE, io_start, 1); pgBufferUsage.shared_blks_written++; @@ -6644,11 +6282,9 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, */ TerminateBufferIO(buf, true, 0); - TRACE_POSTGRESQL_BUFFER_FLUSH_DONE(BufTagGetForkNum(&buf->tag), - buf->tag.blockNum, - reln->smgr_rlocator.locator.spcOid, - reln->smgr_rlocator.locator.dbOid, - reln->smgr_rlocator.locator.relNumber); + TRACE_POSTGRESQL_BUFFER_FLUSH_DONE( + BufTagGetForkNum(&buf->tag), buf->tag.blockNum, reln->smgr_rlocator.locator.spcOid, + reln->smgr_rlocator.locator.dbOid, reln->smgr_rlocator.locator.relNumber); /* Pop the error context stack */ error_context_stack = errcallback.previous; @@ -6665,28 +6301,24 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, BlockNumber RelationGetNumberOfBlocksInFork(Relation relation, ForkNumber forkNum) { - if (RELKIND_HAS_TABLE_AM(relation->rd_rel->relkind)) - { + if (RELKIND_HAS_TABLE_AM(relation->rd_rel->relkind)) { /* * Not every table AM uses BLCKSZ wide fixed size blocks. Therefore * tableam returns the size in bytes - but for the purpose of this * routine, we want the number of blocks. Therefore divide, rounding * up. */ - uint64 szbytes; + uint64 szbytes; szbytes = table_relation_size(relation, forkNum); return (szbytes + (BLCKSZ - 1)) / BLCKSZ; - } - else if (RELKIND_HAS_STORAGE(relation->rd_rel->relkind)) - { + } else if (RELKIND_HAS_STORAGE(relation->rd_rel->relkind)) { return smgrnblocks(RelationGetSmgr(relation), forkNum); - } - else + } else Assert(false); - return 0; /* keep compiler quiet */ + return 0; /* keep compiler quiet */ } /* @@ -6727,10 +6359,10 @@ BufferIsPermanent(Buffer buffer) XLogRecPtr BufferGetLSNAtomic(Buffer buffer) { - char *page = BufferGetPage(buffer); + char *page = BufferGetPage(buffer); BufferDesc *bufHdr; - XLogRecPtr lsn; - uint32 buf_state; + XLogRecPtr lsn; + uint32 buf_state; /* * If we don't need locking for correctness, fastpath out. @@ -6772,25 +6404,22 @@ BufferGetLSNAtomic(Buffer buffer) * -------------------------------------------------------------------- */ void -DropRelationBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum, - int nforks, BlockNumber *firstDelBlock) +DropRelationBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum, int nforks, + BlockNumber *firstDelBlock) { - int i; - int j; + int i; + int j; RelFileLocatorBackend rlocator; BlockNumber nForkBlock[MAX_FORKNUM]; - uint64 nBlocksToInvalidate = 0; + uint64 nBlocksToInvalidate = 0; rlocator = smgr_reln->smgr_rlocator; /* If it's a local relation, it's localbuf.c's problem. */ - if (RelFileLocatorBackendIsTemp(rlocator)) - { - if (rlocator.backend == MyBackendId) - { + if (RelFileLocatorBackendIsTemp(rlocator)) { + if (rlocator.backend == MyBackendId) { for (j = 0; j < nforks; j++) - DropRelationLocalBuffers(rlocator.locator, forkNum[j], - firstDelBlock[j]); + DropRelationLocalBuffers(rlocator.locator, forkNum[j], firstDelBlock[j]); } return; } @@ -6817,13 +6446,11 @@ DropRelationBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum, * that should be fine because there must not be any buffers after that * file size. */ - for (i = 0; i < nforks; i++) - { + for (i = 0; i < nforks; i++) { /* Get the number of blocks for a relation's fork */ nForkBlock[i] = smgrnblocks_cached(smgr_reln, forkNum[i]); - if (nForkBlock[i] == InvalidBlockNumber) - { + if (nForkBlock[i] == InvalidBlockNumber) { nBlocksToInvalidate = InvalidBlockNumber; break; } @@ -6836,19 +6463,17 @@ DropRelationBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum, * We apply the optimization iff the total number of blocks to invalidate * is below the BUF_DROP_FULL_SCAN_THRESHOLD. */ - if (BlockNumberIsValid(nBlocksToInvalidate) && - nBlocksToInvalidate < BUF_DROP_FULL_SCAN_THRESHOLD) - { + if (BlockNumberIsValid(nBlocksToInvalidate) + && nBlocksToInvalidate < BUF_DROP_FULL_SCAN_THRESHOLD) { for (j = 0; j < nforks; j++) - FindAndDropRelationBuffers(rlocator.locator, forkNum[j], - nForkBlock[j], firstDelBlock[j]); + FindAndDropRelationBuffers(rlocator.locator, forkNum[j], nForkBlock[j], + firstDelBlock[j]); return; } - for (i = 0; i < NBuffers; i++) - { + for (i = 0; i < NBuffers; i++) { BufferDesc *bufHdr = GetBufferDescriptor(i); - uint32 buf_state; + uint32 buf_state; /* * We can make this a tad faster by prechecking the buffer tag before @@ -6871,13 +6496,11 @@ DropRelationBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum, buf_state = LockBufHdr(bufHdr); - for (j = 0; j < nforks; j++) - { - if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator.locator) && - BufTagGetForkNum(&bufHdr->tag) == forkNum[j] && - bufHdr->tag.blockNum >= firstDelBlock[j]) - { - InvalidateBuffer(bufHdr); /* releases spinlock */ + for (j = 0; j < nforks; j++) { + if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator.locator) + && BufTagGetForkNum(&bufHdr->tag) == forkNum[j] + && bufHdr->tag.blockNum >= firstDelBlock[j]) { + InvalidateBuffer(bufHdr); /* releases spinlock */ break; } } @@ -6897,29 +6520,26 @@ DropRelationBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum, void DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) { - int i; - int n = 0; + int i; + int n = 0; SMgrRelation *rels; - BlockNumber (*block)[MAX_FORKNUM + 1]; - uint64 nBlocksToInvalidate = 0; + BlockNumber(*block)[MAX_FORKNUM + 1]; + uint64 nBlocksToInvalidate = 0; RelFileLocator *locators; - bool cached = true; - bool use_bsearch; + bool cached = true; + bool use_bsearch; if (nlocators == 0) return; - rels = palloc(sizeof(SMgrRelation) * nlocators); /* non-local relations */ + rels = palloc(sizeof(SMgrRelation) * nlocators); /* non-local relations */ /* If it's a local relation, it's localbuf.c's problem. */ - for (i = 0; i < nlocators; i++) - { - if (RelFileLocatorBackendIsTemp(smgr_reln[i]->smgr_rlocator)) - { + for (i = 0; i < nlocators; i++) { + if (RelFileLocatorBackendIsTemp(smgr_reln[i]->smgr_rlocator)) { if (smgr_reln[i]->smgr_rlocator.backend == MyBackendId) DropRelationAllLocalBuffers(smgr_reln[i]->smgr_rlocator.locator); - } - else + } else rels[n++] = smgr_reln[i]; } @@ -6927,8 +6547,7 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) * If there are no non-local relations, then we're done. Release the * memory and return. */ - if (n == 0) - { + if (n == 0) { pfree(rels); return; } @@ -6937,23 +6556,19 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) * This is used to remember the number of blocks for all the relations * forks. */ - block = (BlockNumber (*)[MAX_FORKNUM + 1]) - palloc(sizeof(BlockNumber) * n * (MAX_FORKNUM + 1)); + block = (BlockNumber(*)[MAX_FORKNUM + 1]) palloc(sizeof(BlockNumber) * n * (MAX_FORKNUM + 1)); /* * We can avoid scanning the entire buffer pool if we know the exact size * of each of the given relation forks. See DropRelationBuffers. */ - for (i = 0; i < n && cached; i++) - { - for (int j = 0; j <= MAX_FORKNUM; j++) - { + for (i = 0; i < n && cached; i++) { + for (int j = 0; j <= MAX_FORKNUM; j++) { /* Get the number of blocks for a relation's fork. */ block[i][j] = smgrnblocks_cached(rels[i], j); /* We need to only consider the relation forks that exists. */ - if (block[i][j] == InvalidBlockNumber) - { + if (block[i][j] == InvalidBlockNumber) { if (!smgrexists(rels[i], j)) continue; cached = false; @@ -6969,19 +6584,15 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) * We apply the optimization iff the total number of blocks to invalidate * is below the BUF_DROP_FULL_SCAN_THRESHOLD. */ - if (cached && nBlocksToInvalidate < BUF_DROP_FULL_SCAN_THRESHOLD) - { - for (i = 0; i < n; i++) - { - for (int j = 0; j <= MAX_FORKNUM; j++) - { + if (cached && nBlocksToInvalidate < BUF_DROP_FULL_SCAN_THRESHOLD) { + for (i = 0; i < n; i++) { + for (int j = 0; j <= MAX_FORKNUM; j++) { /* ignore relation forks that doesn't exist */ if (!BlockNumberIsValid(block[i][j])) continue; /* drop all the buffers for a particular relation fork */ - FindAndDropRelationBuffers(rels[i]->smgr_rlocator.locator, - j, block[i][j], 0); + FindAndDropRelationBuffers(rels[i]->smgr_rlocator.locator, j, block[i][j], 0); } } @@ -6991,7 +6602,7 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) } pfree(block); - locators = palloc(sizeof(RelFileLocator) * n); /* non-local relations */ + locators = palloc(sizeof(RelFileLocator) * n); /* non-local relations */ for (i = 0; i < n; i++) locators[i] = rels[i]->smgr_rlocator.locator; @@ -7007,37 +6618,30 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) if (use_bsearch) pg_qsort(locators, n, sizeof(RelFileLocator), rlocator_comparator); - for (i = 0; i < NBuffers; i++) - { + for (i = 0; i < NBuffers; i++) { RelFileLocator *rlocator = NULL; BufferDesc *bufHdr = GetBufferDescriptor(i); - uint32 buf_state; + uint32 buf_state; /* * As in DropRelationBuffers, an unlocked precheck should be safe and * saves some cycles. */ - if (!use_bsearch) - { - int j; + if (!use_bsearch) { + int j; - for (j = 0; j < n; j++) - { - if (BufTagMatchesRelFileLocator(&bufHdr->tag, &locators[j])) - { + for (j = 0; j < n; j++) { + if (BufTagMatchesRelFileLocator(&bufHdr->tag, &locators[j])) { rlocator = &locators[j]; break; } } - } - else - { + } else { RelFileLocator locator; locator = BufTagGetRelFileLocator(&bufHdr->tag); - rlocator = bsearch((const void *) &(locator), - locators, n, sizeof(RelFileLocator), + rlocator = bsearch((const void *)&(locator), locators, n, sizeof(RelFileLocator), rlocator_comparator); } @@ -7047,7 +6651,7 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) buf_state = LockBufHdr(bufHdr); if (BufTagMatchesRelFileLocator(&bufHdr->tag, rlocator)) - InvalidateBuffer(bufHdr); /* releases spinlock */ + InvalidateBuffer(bufHdr); /* releases spinlock */ else UnlockBufHdr(bufHdr, buf_state); } @@ -7066,20 +6670,18 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) * -------------------------------------------------------------------- */ static void -FindAndDropRelationBuffers(RelFileLocator rlocator, ForkNumber forkNum, - BlockNumber nForkBlock, +FindAndDropRelationBuffers(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber nForkBlock, BlockNumber firstDelBlock) { BlockNumber curBlock; - for (curBlock = firstDelBlock; curBlock < nForkBlock; curBlock++) - { - uint32 bufHash; /* hash value for tag */ - BufferTag bufTag; /* identity of requested block */ - LWLock *bufPartitionLock; /* buffer partition lock for it */ - int buf_id; + for (curBlock = firstDelBlock; curBlock < nForkBlock; curBlock++) { + uint32 bufHash; /* hash value for tag */ + BufferTag bufTag; /* identity of requested block */ + LWLock *bufPartitionLock; /* buffer partition lock for it */ + int buf_id; BufferDesc *bufHdr; - uint32 buf_state; + uint32 buf_state; /* create a tag so we can lookup the buffer */ InitBufferTag(&bufTag, &rlocator, forkNum, curBlock); @@ -7106,10 +6708,9 @@ FindAndDropRelationBuffers(RelFileLocator rlocator, ForkNumber forkNum, */ buf_state = LockBufHdr(bufHdr); - if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator) && - BufTagGetForkNum(&bufHdr->tag) == forkNum && - bufHdr->tag.blockNum >= firstDelBlock) - InvalidateBuffer(bufHdr); /* releases spinlock */ + if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator) + && BufTagGetForkNum(&bufHdr->tag) == forkNum && bufHdr->tag.blockNum >= firstDelBlock) + InvalidateBuffer(bufHdr); /* releases spinlock */ else UnlockBufHdr(bufHdr, buf_state); } @@ -7129,17 +6730,16 @@ FindAndDropRelationBuffers(RelFileLocator rlocator, ForkNumber forkNum, void DropDatabaseBuffers(Oid dbid) { - int i; + int i; /* * We needn't consider local buffers, since by assumption the target * database isn't our own. */ - for (i = 0; i < NBuffers; i++) - { + for (i = 0; i < NBuffers; i++) { BufferDesc *bufHdr = GetBufferDescriptor(i); - uint32 buf_state; + uint32 buf_state; /* * As in DropRelationBuffers, an unlocked precheck should be safe and @@ -7150,7 +6750,7 @@ DropDatabaseBuffers(Oid dbid) buf_state = LockBufHdr(bufHdr); if (bufHdr->tag.dbOid == dbid) - InvalidateBuffer(bufHdr); /* releases spinlock */ + InvalidateBuffer(bufHdr); /* releases spinlock */ else UnlockBufHdr(bufHdr, buf_state); } @@ -7167,22 +6767,20 @@ DropDatabaseBuffers(Oid dbid) void PrintBufferDescs(void) { - int i; + int i; - for (i = 0; i < NBuffers; ++i) - { + for (i = 0; i < NBuffers; ++i) { BufferDesc *buf = GetBufferDescriptor(i); - Buffer b = BufferDescriptorGetBuffer(buf); + Buffer b = BufferDescriptorGetBuffer(buf); /* theoretically we should lock the bufhdr here */ elog(LOG, "[%02d] (freeNext=%d, rel=%s, " "blockNum=%u, flags=0x%x, refcount=%u %d)", i, buf->freeNext, - relpathbackend(BufTagGetRelFileLocator(&buf->tag), - InvalidBackendId, BufTagGetForkNum(&buf->tag)), - buf->tag.blockNum, buf->flags, - buf->refcount, GetPrivateRefCount(b)); + relpathbackend(BufTagGetRelFileLocator(&buf->tag), InvalidBackendId, + BufTagGetForkNum(&buf->tag)), + buf->tag.blockNum, buf->flags, buf->refcount, GetPrivateRefCount(b)); } } #endif @@ -7191,24 +6789,20 @@ PrintBufferDescs(void) void PrintPinnedBufs(void) { - int i; + int i; - for (i = 0; i < NBuffers; ++i) - { + for (i = 0; i < NBuffers; ++i) { BufferDesc *buf = GetBufferDescriptor(i); - Buffer b = BufferDescriptorGetBuffer(buf); + Buffer b = BufferDescriptorGetBuffer(buf); - if (GetPrivateRefCount(b) > 0) - { + if (GetPrivateRefCount(b) > 0) { /* theoretically we should lock the bufhdr here */ elog(LOG, "[%02d] (freeNext=%d, rel=%s, " "blockNum=%u, flags=0x%x, refcount=%u %d)", i, buf->freeNext, - relpathperm(BufTagGetRelFileLocator(&buf->tag), - BufTagGetForkNum(&buf->tag)), - buf->tag.blockNum, buf->flags, - buf->refcount, GetPrivateRefCount(b)); + relpathperm(BufTagGetRelFileLocator(&buf->tag), BufTagGetForkNum(&buf->tag)), + buf->tag.blockNum, buf->flags, buf->refcount, GetPrivateRefCount(b)); } } } @@ -7235,29 +6829,26 @@ PrintPinnedBufs(void) void FlushRelationBuffers(Relation rel) { - int i; + int i; BufferDesc *bufHdr; - if (RelationUsesLocalBuffers(rel)) - { - for (i = 0; i < NLocBuffer; i++) - { - uint32 buf_state; - instr_time io_start; + if (RelationUsesLocalBuffers(rel)) { + for (i = 0; i < NLocBuffer; i++) { + uint32 buf_state; + instr_time io_start; bufHdr = GetLocalBufferDescriptor(i); - if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator) && - ((buf_state = pg_atomic_read_u32(&bufHdr->state)) & - (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY)) - { + if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator) + && ((buf_state = pg_atomic_read_u32(&bufHdr->state)) & (BM_VALID | BM_DIRTY)) + == (BM_VALID | BM_DIRTY)) { ErrorContextCallback errcallback; - Page localpage; + Page localpage; - localpage = (char *) LocalBufHdrGetBlock(bufHdr); + localpage = (char *)LocalBufHdrGetBlock(bufHdr); /* Setup error traceback support for ereport() */ errcallback.callback = local_buffer_write_error_callback; - errcallback.arg = (void *) bufHdr; + errcallback.arg = (void *)bufHdr; errcallback.previous = error_context_stack; error_context_stack = &errcallback; @@ -7265,14 +6856,10 @@ FlushRelationBuffers(Relation rel) io_start = pgstat_prepare_io_time(); - smgrwrite(RelationGetSmgr(rel), - BufTagGetForkNum(&bufHdr->tag), - bufHdr->tag.blockNum, - localpage, - false); + smgrwrite(RelationGetSmgr(rel), BufTagGetForkNum(&bufHdr->tag), + bufHdr->tag.blockNum, localpage, false); - pgstat_count_io_op_time(IOOBJECT_TEMP_RELATION, - IOCONTEXT_NORMAL, IOOP_WRITE, + pgstat_count_io_op_time(IOOBJECT_TEMP_RELATION, IOCONTEXT_NORMAL, IOOP_WRITE, io_start, 1); buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED); @@ -7291,9 +6878,8 @@ FlushRelationBuffers(Relation rel) /* Make sure we can handle the pin inside the loop */ ResourceOwnerEnlargeBuffers(CurrentResourceOwner); - for (i = 0; i < NBuffers; i++) - { - uint32 buf_state; + for (i = 0; i < NBuffers; i++) { + uint32 buf_state; bufHdr = GetBufferDescriptor(i); @@ -7307,16 +6893,14 @@ FlushRelationBuffers(Relation rel) ReservePrivateRefCountEntry(); buf_state = LockBufHdr(bufHdr); - if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator) && - (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY)) - { + if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator) + && (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY)) { PinBuffer_Locked(bufHdr); LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED); FlushBuffer(bufHdr, RelationGetSmgr(rel), IOOBJECT_RELATION, IOCONTEXT_NORMAL); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); UnpinBuffer(bufHdr); - } - else + } else UnlockBufHdr(bufHdr, buf_state); } } @@ -7333,9 +6917,9 @@ FlushRelationBuffers(Relation rel) void FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels) { - int i; + int i; SMgrSortArray *srels; - bool use_bsearch; + bool use_bsearch; if (nrels == 0) return; @@ -7343,8 +6927,7 @@ FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels) /* fill-in array for qsort */ srels = palloc(sizeof(SMgrSortArray) * nrels); - for (i = 0; i < nrels; i++) - { + for (i = 0; i < nrels; i++) { Assert(!RelFileLocatorBackendIsTemp(smgrs[i]->smgr_rlocator)); srels[i].rlocator = smgrs[i]->smgr_rlocator.locator; @@ -7364,37 +6947,30 @@ FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels) /* Make sure we can handle the pin inside the loop */ ResourceOwnerEnlargeBuffers(CurrentResourceOwner); - for (i = 0; i < NBuffers; i++) - { + for (i = 0; i < NBuffers; i++) { SMgrSortArray *srelent = NULL; BufferDesc *bufHdr = GetBufferDescriptor(i); - uint32 buf_state; + uint32 buf_state; /* * As in DropRelationBuffers, an unlocked precheck should be safe and * saves some cycles. */ - if (!use_bsearch) - { - int j; + if (!use_bsearch) { + int j; - for (j = 0; j < nrels; j++) - { - if (BufTagMatchesRelFileLocator(&bufHdr->tag, &srels[j].rlocator)) - { + for (j = 0; j < nrels; j++) { + if (BufTagMatchesRelFileLocator(&bufHdr->tag, &srels[j].rlocator)) { srelent = &srels[j]; break; } } - } - else - { + } else { RelFileLocator rlocator; rlocator = BufTagGetRelFileLocator(&bufHdr->tag); - srelent = bsearch((const void *) &(rlocator), - srels, nrels, sizeof(SMgrSortArray), + srelent = bsearch((const void *)&(rlocator), srels, nrels, sizeof(SMgrSortArray), rlocator_comparator); } @@ -7405,16 +6981,14 @@ FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels) ReservePrivateRefCountEntry(); buf_state = LockBufHdr(bufHdr); - if (BufTagMatchesRelFileLocator(&bufHdr->tag, &srelent->rlocator) && - (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY)) - { + if (BufTagMatchesRelFileLocator(&bufHdr->tag, &srelent->rlocator) + && (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY)) { PinBuffer_Locked(bufHdr); LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED); FlushBuffer(bufHdr, srelent->srel, IOOBJECT_RELATION, IOCONTEXT_NORMAL); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); UnpinBuffer(bufHdr); - } - else + } else UnlockBufHdr(bufHdr, buf_state); } @@ -7432,15 +7006,14 @@ FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels) * -------------------------------------------------------------------- */ static void -RelationCopyStorageUsingBuffer(RelFileLocator srclocator, - RelFileLocator dstlocator, +RelationCopyStorageUsingBuffer(RelFileLocator srclocator, RelFileLocator dstlocator, ForkNumber forkNum, bool permanent) { - Buffer srcBuf; - Buffer dstBuf; - Page srcPage; - Page dstPage; - bool use_wal; + Buffer srcBuf; + Buffer dstBuf; + Page srcPage; + Page dstPage; + bool use_wal; BlockNumber nblocks; BlockNumber blkno; PGIOAlignedBlock buf; @@ -7455,8 +7028,7 @@ RelationCopyStorageUsingBuffer(RelFileLocator srclocator, use_wal = XLogIsNeeded() && (permanent || forkNum == INIT_FORKNUM); /* Get number of blocks in the source relation. */ - nblocks = smgrnblocks(smgropen(srclocator, InvalidBackendId), - forkNum); + nblocks = smgrnblocks(smgropen(srclocator, InvalidBackendId), forkNum); /* Nothing to copy; just return. */ if (nblocks == 0) @@ -7467,28 +7039,24 @@ RelationCopyStorageUsingBuffer(RelFileLocator srclocator, * relation before starting to copy block by block. */ memset(buf.data, 0, BLCKSZ); - smgrextend(smgropen(dstlocator, InvalidBackendId), forkNum, nblocks - 1, - buf.data, true); + smgrextend(smgropen(dstlocator, InvalidBackendId), forkNum, nblocks - 1, buf.data, true); /* This is a bulk operation, so use buffer access strategies. */ bstrategy_src = GetAccessStrategy(BAS_BULKREAD); bstrategy_dst = GetAccessStrategy(BAS_BULKWRITE); /* Iterate over each block of the source relation file. */ - for (blkno = 0; blkno < nblocks; blkno++) - { + for (blkno = 0; blkno < nblocks; blkno++) { CHECK_FOR_INTERRUPTS(); /* Read block from source relation. */ - srcBuf = ReadBufferWithoutRelcache(srclocator, forkNum, blkno, - RBM_NORMAL, bstrategy_src, + srcBuf = ReadBufferWithoutRelcache(srclocator, forkNum, blkno, RBM_NORMAL, bstrategy_src, permanent); LockBuffer(srcBuf, BUFFER_LOCK_SHARE); srcPage = BufferGetPage(srcBuf); - dstBuf = ReadBufferWithoutRelcache(dstlocator, forkNum, blkno, - RBM_ZERO_AND_LOCK, bstrategy_dst, - permanent); + dstBuf = ReadBufferWithoutRelcache(dstlocator, forkNum, blkno, RBM_ZERO_AND_LOCK, + bstrategy_dst, permanent); dstPage = BufferGetPage(dstBuf); START_CRIT_SECTION(); @@ -7523,15 +7091,13 @@ RelationCopyStorageUsingBuffer(RelFileLocator srclocator, * -------------------------------------------------------------------- */ void -CreateAndCopyRelationData(RelFileLocator src_rlocator, - RelFileLocator dst_rlocator, bool permanent) +CreateAndCopyRelationData(RelFileLocator src_rlocator, RelFileLocator dst_rlocator, bool permanent) { RelFileLocatorBackend rlocator; - char relpersistence; + char relpersistence; /* Set the relpersistence. */ - relpersistence = permanent ? - RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED; + relpersistence = permanent ? RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED; /* * Create and copy all forks of the relation. During create database we @@ -7542,15 +7108,11 @@ CreateAndCopyRelationData(RelFileLocator src_rlocator, RelationCreateStorage(dst_rlocator, relpersistence, false); /* copy main fork. */ - RelationCopyStorageUsingBuffer(src_rlocator, dst_rlocator, MAIN_FORKNUM, - permanent); + RelationCopyStorageUsingBuffer(src_rlocator, dst_rlocator, MAIN_FORKNUM, permanent); /* copy those extra forks that exist */ - for (ForkNumber forkNum = MAIN_FORKNUM + 1; - forkNum <= MAX_FORKNUM; forkNum++) - { - if (smgrexists(smgropen(src_rlocator, InvalidBackendId), forkNum)) - { + for (ForkNumber forkNum = MAIN_FORKNUM + 1; forkNum <= MAX_FORKNUM; forkNum++) { + if (smgrexists(smgropen(src_rlocator, InvalidBackendId), forkNum)) { smgrcreate(smgropen(dst_rlocator, InvalidBackendId), forkNum, false); /* @@ -7561,8 +7123,7 @@ CreateAndCopyRelationData(RelFileLocator src_rlocator, log_smgrcreate(&dst_rlocator, forkNum); /* Copy a fork's data, block by block. */ - RelationCopyStorageUsingBuffer(src_rlocator, dst_rlocator, forkNum, - permanent); + RelationCopyStorageUsingBuffer(src_rlocator, dst_rlocator, forkNum, permanent); } } @@ -7594,15 +7155,14 @@ CreateAndCopyRelationData(RelFileLocator src_rlocator, void FlushDatabaseBuffers(Oid dbid) { - int i; + int i; BufferDesc *bufHdr; /* Make sure we can handle the pin inside the loop */ ResourceOwnerEnlargeBuffers(CurrentResourceOwner); - for (i = 0; i < NBuffers; i++) - { - uint32 buf_state; + for (i = 0; i < NBuffers; i++) { + uint32 buf_state; bufHdr = GetBufferDescriptor(i); @@ -7616,16 +7176,14 @@ FlushDatabaseBuffers(Oid dbid) ReservePrivateRefCountEntry(); buf_state = LockBufHdr(bufHdr); - if (bufHdr->tag.dbOid == dbid && - (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY)) - { + if (bufHdr->tag.dbOid == dbid + && (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY)) { PinBuffer_Locked(bufHdr); LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED); FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); UnpinBuffer(bufHdr); - } - else + } else UnlockBufHdr(bufHdr, buf_state); } } @@ -7693,8 +7251,7 @@ IncrBufferRefCount(Buffer buffer) ResourceOwnerEnlargeBuffers(CurrentResourceOwner); if (BufferIsLocal(buffer)) LocalRefCount[-buffer - 1]++; - else - { + else { PrivateRefCountEntry *ref; ref = GetPrivateRefCountEntry(buffer, true); @@ -7722,16 +7279,15 @@ void MarkBufferDirtyHint(Buffer buffer, bool buffer_std) { BufferDesc *bufHdr; - Page page = BufferGetPage(buffer); + Page page = BufferGetPage(buffer); #ifdef USE_PGRAC_CLUSTER - uint32 retained_state; + uint32 retained_state; #endif if (!BufferIsValid(buffer)) elog(ERROR, "bad buffer ID: %d", buffer); - if (BufferIsLocal(buffer)) - { + if (BufferIsLocal(buffer)) { MarkLocalBufferDirty(buffer); return; } @@ -7750,8 +7306,7 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) * dirty state and become eligible for stale output. */ retained_state = LockBufHdr(bufHdr); - if (cluster_bufmgr_pcm_x_retained_image_locked(bufHdr, retained_state)) - { + if (cluster_bufmgr_pcm_x_retained_image_locked(bufHdr, retained_state)) { UnlockBufHdr(bufHdr, retained_state); return; } @@ -7769,13 +7324,12 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) * is only intended to be used in cases where failing to write out the * data would be harmless anyway, it doesn't really matter. */ - if ((pg_atomic_read_u32(&bufHdr->state) & (BM_DIRTY | BM_JUST_DIRTIED)) != - (BM_DIRTY | BM_JUST_DIRTIED)) - { - XLogRecPtr lsn = InvalidXLogRecPtr; - bool dirtied = false; - bool delayChkptFlags = false; - uint32 buf_state; + if ((pg_atomic_read_u32(&bufHdr->state) & (BM_DIRTY | BM_JUST_DIRTIED)) + != (BM_DIRTY | BM_JUST_DIRTIED)) { + XLogRecPtr lsn = InvalidXLogRecPtr; + bool dirtied = false; + bool delayChkptFlags = false; + uint32 buf_state; /* * If we need to protect hint bit updates from torn writes, WAL-log a @@ -7786,9 +7340,7 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) * We don't check full_page_writes here because that logic is included * when we call XLogInsert() since the value changes dynamically. */ - if (XLogHintBitIsNeeded() && - (pg_atomic_read_u32(&bufHdr->state) & BM_PERMANENT)) - { + if (XLogHintBitIsNeeded() && (pg_atomic_read_u32(&bufHdr->state) & BM_PERMANENT)) { /* * If we must not write WAL, due to a relfilelocator-specific * condition or being in recovery, don't dirty the page. We can @@ -7797,8 +7349,8 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) * * See src/backend/storage/page/README for longer discussion. */ - if (RecoveryInProgress() || - RelFileLocatorSkippingWAL(BufTagGetRelFileLocator(&bufHdr->tag))) + if (RecoveryInProgress() + || RelFileLocatorSkippingWAL(BufTagGetRelFileLocator(&bufHdr->tag))) return; /* @@ -7834,9 +7386,8 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0); - if (!(buf_state & BM_DIRTY)) - { - dirtied = true; /* Means "will be dirtied by this action" */ + if (!(buf_state & BM_DIRTY)) { + dirtied = true; /* Means "will be dirtied by this action" */ /* * Set the page LSN if we wrote a backup block. We aren't supposed @@ -7861,8 +7412,7 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) if (delayChkptFlags) MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; - if (dirtied) - { + if (dirtied) { VacuumPageDirty++; pgBufferUsage.shared_blks_dirtied++; if (VacuumCostActive) @@ -7885,9 +7435,8 @@ UnlockBuffers(void) { BufferDesc *buf = PinCountWaitBuf; - if (buf) - { - uint32 buf_state; + if (buf) { + uint32 buf_state; buf_state = LockBufHdr(buf); @@ -7895,8 +7444,8 @@ UnlockBuffers(void) * Don't complain if flag bit not set; it could have been reset but we * got a cancel/die interrupt before getting the signal. */ - if ((buf_state & BM_PIN_COUNT_WAITER) != 0 && - buf->wait_backend_pgprocno == MyProc->pgprocno) + if ((buf_state & BM_PIN_COUNT_WAITER) != 0 + && buf->wait_backend_pgprocno == MyProc->pgprocno) buf_state &= ~BM_PIN_COUNT_WAITER; UnlockBufHdr(buf, buf_state); @@ -7921,16 +7470,16 @@ LockBuffer(Buffer buffer, int mode) { BufferDesc *buf; #ifdef USE_PGRAC_CLUSTER - bool pcm_acquired = false; + bool pcm_acquired = false; PcmLockMode pcm_mode = PCM_LOCK_MODE_N; - bool pcm_covered = false; /* took the cached-cover fast path */ - uint64 pcm_covered_gen = 0; /* ownership generation captured at cover */ - bool pcm_pending_set = false; /* we set GRANT_PENDING (W3), must clear */ + bool pcm_covered = false; /* took the cached-cover fast path */ + uint64 pcm_covered_gen = 0; /* ownership generation captured at cover */ + bool pcm_pending_set = false; /* we set GRANT_PENDING (W3), must clear */ MemoryContext pcm_error_context = CurrentMemoryContext; ClusterPcmOwnSnapshot pcm_pending_base; ClusterPcmOwnResult pcm_pending_result = CLUSTER_PCM_OWN_OK; - uint64 pcm_pending_token = 0; - uint64 pcm_committed_generation = 0; + uint64 pcm_pending_token = 0; + uint64 pcm_committed_generation = 0; ClusterPcmXHolderLedgerEntry *pcm_x_holder = NULL; ClusterPcmXWriterLedgerEntry *pcm_x_writer = NULL; bool pcm_x_writer_managed = false; @@ -7938,22 +7487,18 @@ LockBuffer(Buffer buffer, int mode) Assert(BufferIsPinned(buffer)); if (BufferIsLocal(buffer)) - return; /* local buffers need no lock */ + return; /* local buffers need no lock */ buf = GetBufferDescriptor(buffer - 1); - if (mode != BUFFER_LOCK_UNLOCK && - mode != BUFFER_LOCK_SHARE && - mode != BUFFER_LOCK_EXCLUSIVE) + if (mode != BUFFER_LOCK_UNLOCK && mode != BUFFER_LOCK_SHARE && mode != BUFFER_LOCK_EXCLUSIVE) elog(ERROR, "unrecognized buffer lock mode: %d", mode); #ifdef USE_PGRAC_CLUSTER - if (mode != BUFFER_LOCK_UNLOCK && - cluster_pcm_is_active() && - cluster_bufmgr_should_pcm_track(buf)) - { - uint8 pcm_initial_state = (uint8)PCM_STATE_N; - uint32 pcm_initial_flags = 0; + if (mode != BUFFER_LOCK_UNLOCK && cluster_pcm_is_active() + && cluster_bufmgr_should_pcm_track(buf)) { + uint8 pcm_initial_state = (uint8)PCM_STATE_N; + uint32 pcm_initial_flags = 0; pcm_mode = (mode == BUFFER_LOCK_SHARE) ? PCM_LOCK_MODE_S : PCM_LOCK_MODE_X; @@ -7963,10 +7508,10 @@ LockBuffer(Buffer buffer, int mode) * re-enqueues the same node X. The post-content-lock generation * check below remains the race-closing authority. */ if (pcm_mode == PCM_LOCK_MODE_X && cluster_gcs_block_local_cache) { - cluster_pcm_own_read(buf, &pcm_initial_state, &pcm_covered_gen, - &pcm_initial_flags); - pcm_covered = cluster_pcm_x_cached_cover_bypasses_queue(cluster_gcs_block_local_cache, - pcm_mode == PCM_LOCK_MODE_X, pcm_initial_state, pcm_initial_flags); + cluster_pcm_own_read(buf, &pcm_initial_state, &pcm_covered_gen, &pcm_initial_flags); + pcm_covered = cluster_pcm_x_cached_cover_bypasses_queue( + cluster_gcs_block_local_cache, pcm_mode == PCM_LOCK_MODE_X, pcm_initial_state, + pcm_initial_flags); } if (!pcm_covered) @@ -8076,8 +7621,7 @@ LockBuffer(Buffer buffer, int mode) } #endif - if (mode == BUFFER_LOCK_UNLOCK) - { + if (mode == BUFFER_LOCK_UNLOCK) { #ifdef USE_PGRAC_CLUSTER pcm_x_writer = cluster_bufmgr_pcm_x_writer_find(buf); pcm_x_writer_managed = pcm_x_writer != NULL; @@ -8090,9 +7634,7 @@ LockBuffer(Buffer buffer, int mode) cluster_bufmgr_pcm_x_writer_release(pcm_x_writer); cluster_bufmgr_pcm_x_holder_unregister(pcm_x_holder); #endif - } - else - { + } else { #ifdef USE_PGRAC_CLUSTER /* * GCS serve-stall round-6 (ownership P0) — cached-X BAST window. @@ -8137,17 +7679,15 @@ LockBuffer(Buffer buffer, int mode) * content lock and the downgrade path serializes under it, so no * further downgrade can intervene -- at most one fallback. */ - if (pcm_covered) - { - uint8 cur_state; - uint64 cur_gen; - uint32 cur_flags; + if (pcm_covered) { + uint8 cur_state; + uint64 cur_gen; + uint32 cur_flags; cluster_pcm_own_read(buf, &cur_state, &cur_gen, &cur_flags); if (cur_gen != pcm_covered_gen - || !cluster_pcm_mode_covers((PcmLockMode) cur_state, pcm_mode) - || (cur_flags & (PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING)) != 0) - { + || !cluster_pcm_mode_covers((PcmLockMode)cur_state, pcm_mode) + || (cur_flags & (PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING)) != 0) { cluster_pcm_note_writer_cover_stale_detected(); pcm_covered = false; LWLockRelease(BufferDescriptorGetContentLock(buf)); @@ -8171,7 +7711,7 @@ LockBuffer(Buffer buffer, int mode) } PG_CATCH(); { - ErrorData *original_error; + ErrorData *original_error; /* * Holder detach is itself allowed to encounter an LWLock ERROR. @@ -8207,7 +7747,7 @@ LockBuffer(Buffer buffer, int mode) */ else if (pcm_pending_set) cluster_pcm_own_abort_grant_after_error(buf, &pcm_pending_base, pcm_pending_token, - "LockBuffer content-lock acquire"); + "LockBuffer content-lock acquire"); ReThrowError(original_error); } PG_END_TRY(); @@ -8220,8 +7760,7 @@ LockBuffer(Buffer buffer, int mode) * prove the GRANT_PENDING consult parks it instead of acking the * grant away. */ - if (pcm_acquired && pcm_pending_set && pcm_mode == PCM_LOCK_MODE_X) - { + if (pcm_acquired && pcm_pending_set && pcm_mode == PCM_LOCK_MODE_X) { CLUSTER_INJECTION_POINT("cluster-pcm-grant-finalize-window"); /* @@ -8237,17 +7776,14 @@ LockBuffer(Buffer buffer, int mode) * W3 defect. */ CLUSTER_INJECTION_POINT("cluster-pcm-grant-finalize-deliver-invalidate"); - if (cluster_injection_should_skip( - "cluster-pcm-grant-finalize-deliver-invalidate")) - { + if (cluster_injection_should_skip("cluster-pcm-grant-finalize-deliver-invalidate")) { if (cluster_gcs_block_test_deliver_self_invalidate(buf->tag)) - elog(WARNING, - "cluster W3 delivery shim: synthetic INVALIDATE was ACKed instead of parked (GRANT_PENDING not honored)"); + elog(WARNING, "cluster W3 delivery shim: synthetic INVALIDATE was ACKed " + "instead of parked (GRANT_PENDING not honored)"); } } - if (pcm_acquired) - { + if (pcm_acquired) { /* * Finalize the grant under the header spinlock: set buffer_type + * pcm_state, bump the ownership generation, and clear @@ -8257,12 +7793,9 @@ LockBuffer(Buffer buffer, int mode) */ cluster_pcm_own_finish_grant_or_rollback( buf, &pcm_pending_base, pcm_pending_token, - (pcm_mode == PCM_LOCK_MODE_S) ? (uint8) PCM_STATE_S : (uint8) PCM_STATE_X, - pcm_mode, + (pcm_mode == PCM_LOCK_MODE_S) ? (uint8)PCM_STATE_S : (uint8)PCM_STATE_X, pcm_mode, &pcm_committed_generation); - } - else if (pcm_pending_set) - { + } else if (pcm_pending_set) { /* No durable grant (one-shot READ_IMAGE): clear the PENDING marker * we set before the acquire so it does not linger. */ cluster_pcm_own_abort_grant_or_error(buf, &pcm_pending_base, pcm_pending_token, @@ -8283,22 +7816,18 @@ LockBuffer(Buffer buffer, int mode) * LockBuffer re-acquires (cluster_pcm_mode_covers(N, ...) is false), * getting real X once the remote holder is terminal. */ - if (mode == BUFFER_LOCK_UNLOCK - && buf->pcm_state == (uint8) PCM_STATE_READ_IMAGE) - cluster_pcm_own_transition(buf, (uint8) PCM_STATE_N, 0, 0); + if (mode == BUFFER_LOCK_UNLOCK && buf->pcm_state == (uint8)PCM_STATE_READ_IMAGE) + cluster_pcm_own_transition(buf, (uint8)PCM_STATE_N, 0, 0); - if (mode == BUFFER_LOCK_UNLOCK && - cluster_pcm_is_active() && - cluster_bufmgr_should_pcm_track(buf) && - buf->pcm_state != (uint8) PCM_STATE_N) - { + if (mode == BUFFER_LOCK_UNLOCK && cluster_pcm_is_active() + && cluster_bufmgr_should_pcm_track(buf) && buf->pcm_state != (uint8)PCM_STATE_N) { /* PGRAC: spec-2.35 D4 (HC111 + HC112) — content-lock unlock path. * SCUR preserves cache residency (bit stays set so CF 2-way * forward can still target this node). XCUR delegates to the * eviction release (single-holder semantic preserved). Real * cache eviction is handled by the InvalidateBuffer / * InvalidateVictimBuffer / Drop*Buffers hook points (below). */ - PcmLockMode old_mode = (PcmLockMode) buf->pcm_state; + PcmLockMode old_mode = (PcmLockMode)buf->pcm_state; /* * PGRAC: spec-4.7a D2 — hold-until-revoked. With the node-level @@ -8342,11 +7871,10 @@ LockBufferForAuxiliaryPageInit(Buffer buffer, ClusterPcmDirectInitKind kind) return; buf = GetBufferDescriptor(buffer - 1); - if (cluster_pcm_is_active() && cluster_bufmgr_should_pcm_track(buf)) - { + if (cluster_pcm_is_active() && cluster_bufmgr_should_pcm_track(buf)) { ClusterPcmDirectInitProof direct_init_proof; - uint8 pcm_state; - uint32 flags; + uint8 pcm_state; + uint32 flags; /* * A concurrent initializer may already have established X while the @@ -8354,8 +7882,7 @@ LockBufferForAuxiliaryPageInit(Buffer buffer, ClusterPcmDirectInitKind kind) * path revalidates that grant after taking the content lock. */ cluster_pcm_own_read(buf, &pcm_state, NULL, &flags); - if (pcm_state == (uint8) PCM_STATE_X && flags == 0) - { + if (pcm_state == (uint8)PCM_STATE_X && flags == 0) { LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); return; } @@ -8404,23 +7931,21 @@ bool ConditionalLockBuffer(Buffer buffer) { BufferDesc *buf; - bool acquired; + bool acquired; #ifdef USE_PGRAC_CLUSTER - bool blocked; - uint32 buf_state; + bool blocked; + uint32 buf_state; #endif Assert(BufferIsPinned(buffer)); if (BufferIsLocal(buffer)) - return true; /* act as though we got it */ + return true; /* act as though we got it */ buf = GetBufferDescriptor(buffer - 1); - acquired = LWLockConditionalAcquire(BufferDescriptorGetContentLock(buf), - LW_EXCLUSIVE); + acquired = LWLockConditionalAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE); #ifdef USE_PGRAC_CLUSTER - if (acquired) - { + if (acquired) { /* This API bypasses LockBuffer/W1. Recreate its reservation boundary * after acquiring content authority: an already-running user wins and * serializes before revoke finish; a user arriving after REVOKING or @@ -8432,8 +7957,7 @@ ConditionalLockBuffer(Buffer buffer) cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state), buf->pcm_state, cluster_pcm_own_flags_get(buf->buf_id)); UnlockBufHdr(buf, buf_state); - if (blocked) - { + if (blocked) { LWLockRelease(BufferDescriptorGetContentLock(buf)); return false; } @@ -8451,17 +7975,12 @@ ConditionalLockBuffer(Buffer buffer) void CheckBufferIsPinnedOnce(Buffer buffer) { - if (BufferIsLocal(buffer)) - { + if (BufferIsLocal(buffer)) { if (LocalRefCount[-buffer - 1] != 1) - elog(ERROR, "incorrect local pin count: %d", - LocalRefCount[-buffer - 1]); - } - else - { + elog(ERROR, "incorrect local pin count: %d", LocalRefCount[-buffer - 1]); + } else { if (GetPrivateRefCount(buffer) != 1) - elog(ERROR, "incorrect local pin count: %d", - GetPrivateRefCount(buffer)); + elog(ERROR, "incorrect local pin count: %d", GetPrivateRefCount(buffer)); } } @@ -8491,8 +8010,8 @@ LockBufferForCleanup(Buffer buffer) { BufferDesc *bufHdr; TimestampTz waitStart = 0; - bool waiting = false; - bool logged_recovery_conflict = false; + bool waiting = false; + bool logged_recovery_conflict = false; Assert(BufferIsPinned(buffer)); Assert(PinCountWaitBuf == NULL); @@ -8505,17 +8024,15 @@ LockBufferForCleanup(Buffer buffer) bufHdr = GetBufferDescriptor(buffer - 1); - for (;;) - { - uint32 buf_state; + for (;;) { + uint32 buf_state; /* Try to acquire lock */ LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); buf_state = LockBufHdr(bufHdr); Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0); - if (BUF_STATE_GET_REFCOUNT(buf_state) == 1) - { + if (BUF_STATE_GET_REFCOUNT(buf_state) == 1) { /* Successfully acquired exclusive lock with pincount 1 */ UnlockBufHdr(bufHdr, buf_state); @@ -8525,12 +8042,10 @@ LockBufferForCleanup(Buffer buffer) * deadlock_timeout for it. */ if (logged_recovery_conflict) - LogRecoveryConflict(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, - waitStart, GetCurrentTimestamp(), - NULL, false); + LogRecoveryConflict(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, waitStart, + GetCurrentTimestamp(), NULL, false); - if (waiting) - { + if (waiting) { /* reset ps display to remove the suffix if we added one */ set_ps_display_remove_suffix(); waiting = false; @@ -8538,8 +8053,7 @@ LockBufferForCleanup(Buffer buffer) return; } /* Failed, so mark myself as waiting for pincount 1 */ - if (buf_state & BM_PIN_COUNT_WAITER) - { + if (buf_state & BM_PIN_COUNT_WAITER) { UnlockBufHdr(bufHdr, buf_state); LockBuffer(buffer, BUFFER_LOCK_UNLOCK); elog(ERROR, "multiple backends attempting to wait for pincount 1"); @@ -8551,10 +8065,8 @@ LockBufferForCleanup(Buffer buffer) LockBuffer(buffer, BUFFER_LOCK_UNLOCK); /* Wait to be signaled by UnpinBuffer() */ - if (InHotStandby) - { - if (!waiting) - { + if (InHotStandby) { + if (!waiting) { /* adjust the process title to indicate that it's waiting */ set_ps_display_suffix("waiting"); waiting = true; @@ -8568,15 +8080,12 @@ LockBufferForCleanup(Buffer buffer) * not started waiting yet in this case. So, the wait start * timestamp is set after this logic. */ - if (waitStart != 0 && !logged_recovery_conflict) - { + if (waitStart != 0 && !logged_recovery_conflict) { TimestampTz now = GetCurrentTimestamp(); - if (TimestampDifferenceExceeds(waitStart, now, - DeadlockTimeout)) - { - LogRecoveryConflict(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, - waitStart, now, NULL, true); + if (TimestampDifferenceExceeds(waitStart, now, DeadlockTimeout)) { + LogRecoveryConflict(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, waitStart, now, NULL, + true); logged_recovery_conflict = true; } } @@ -8594,8 +8103,7 @@ LockBufferForCleanup(Buffer buffer) ResolveRecoveryConflictWithBufferPin(); /* Reset the published bufid */ SetStartupBufferPinWaitBufId(-1); - } - else + } else ProcWaitForSignal(PG_WAIT_BUFFER_PIN); /* @@ -8607,8 +8115,8 @@ LockBufferForCleanup(Buffer buffer) * better be safe. */ buf_state = LockBufHdr(bufHdr); - if ((buf_state & BM_PIN_COUNT_WAITER) != 0 && - bufHdr->wait_backend_pgprocno == MyProc->pgprocno) + if ((buf_state & BM_PIN_COUNT_WAITER) != 0 + && bufHdr->wait_backend_pgprocno == MyProc->pgprocno) buf_state &= ~BM_PIN_COUNT_WAITER; UnlockBufHdr(bufHdr, buf_state); @@ -8624,7 +8132,7 @@ LockBufferForCleanup(Buffer buffer) bool HoldingBufferPinThatDelaysRecovery(void) { - int bufid = GetStartupBufferPinWaitBufId(); + int bufid = GetStartupBufferPinWaitBufId(); /* * If we get woken slowly then it's possible that the Startup process was @@ -8651,13 +8159,11 @@ bool ConditionalLockBufferForCleanup(Buffer buffer) { BufferDesc *bufHdr; - uint32 buf_state, - refcount; + uint32 buf_state, refcount; Assert(BufferIsValid(buffer)); - if (BufferIsLocal(buffer)) - { + if (BufferIsLocal(buffer)) { refcount = LocalRefCount[-buffer - 1]; /* There should be exactly one pin */ Assert(refcount > 0); @@ -8682,8 +8188,7 @@ ConditionalLockBufferForCleanup(Buffer buffer) refcount = BUF_STATE_GET_REFCOUNT(buf_state); Assert(refcount > 0); - if (refcount == 1) - { + if (refcount == 1) { /* Successfully acquired exclusive lock with pincount 1 */ UnlockBufHdr(bufHdr, buf_state); return true; @@ -8707,12 +8212,11 @@ bool IsBufferCleanupOK(Buffer buffer) { BufferDesc *bufHdr; - uint32 buf_state; + uint32 buf_state; Assert(BufferIsValid(buffer)); - if (BufferIsLocal(buffer)) - { + if (BufferIsLocal(buffer)) { /* There should be exactly one pin */ if (LocalRefCount[-buffer - 1] != 1) return false; @@ -8727,14 +8231,12 @@ IsBufferCleanupOK(Buffer buffer) bufHdr = GetBufferDescriptor(buffer - 1); /* caller must hold exclusive lock on buffer */ - Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr), - LW_EXCLUSIVE)); + Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr), LW_EXCLUSIVE)); buf_state = LockBufHdr(bufHdr); Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0); - if (BUF_STATE_GET_REFCOUNT(buf_state) == 1) - { + if (BUF_STATE_GET_REFCOUNT(buf_state) == 1) { /* pincount is OK. */ UnlockBufHdr(bufHdr, buf_state); return true; @@ -8763,9 +8265,8 @@ WaitIO(BufferDesc *buf) ConditionVariable *cv = BufferDescriptorGetIOCV(buf); ConditionVariablePrepareToSleep(cv); - for (;;) - { - uint32 buf_state; + for (;;) { + uint32 buf_state; /* * It may not be necessary to acquire the spinlock to check the flag @@ -8818,15 +8319,14 @@ WaitIO(BufferDesc *buf) static bool StartBufferIO(BufferDesc *buf, bool forInput) { - uint32 buf_state; + uint32 buf_state; #ifdef USE_PGRAC_CLUSTER - bool pi_implicit_discard = false; + bool pi_implicit_discard = false; #endif ResourceOwnerEnlargeBufferIOs(CurrentResourceOwner); - for (;;) - { + for (;;) { buf_state = LockBufHdr(buf); if (!(buf_state & BM_IO_IN_PROGRESS)) @@ -8837,8 +8337,7 @@ StartBufferIO(BufferDesc *buf, bool forInput) /* Once we get here, there is definitely no I/O active on this buffer */ - if (forInput ? (buf_state & BM_VALID) : !(buf_state & BM_DIRTY)) - { + if (forInput ? (buf_state & BM_VALID) : !(buf_state & BM_DIRTY)) { /* someone else already did the I/O */ UnlockBufHdr(buf, buf_state); return false; @@ -8846,9 +8345,8 @@ StartBufferIO(BufferDesc *buf, bool forInput) #ifdef USE_PGRAC_CLUSTER /* PGRAC: spec-6.12h D-h3a — see the function header note. */ - if (forInput && buf->buffer_type == (uint8) BUF_TYPE_PI) - { - buf->buffer_type = (uint8) BUF_TYPE_CURRENT; + if (forInput && buf->buffer_type == (uint8)BUF_TYPE_PI) { + buf->buffer_type = (uint8)BUF_TYPE_CURRENT; pi_implicit_discard = true; } #endif @@ -8861,8 +8359,7 @@ StartBufferIO(BufferDesc *buf, bool forInput) cluster_lever_h_note_pi_implicit_discard(); #endif - ResourceOwnerRememberBufferIO(CurrentResourceOwner, - BufferDescriptorGetBuffer(buf)); + ResourceOwnerRememberBufferIO(CurrentResourceOwner, BufferDescriptorGetBuffer(buf)); return true; } @@ -8886,7 +8383,7 @@ StartBufferIO(BufferDesc *buf, bool forInput) static void TerminateBufferIO(BufferDesc *buf, bool clear_dirty, uint32 set_flag_bits) { - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf); @@ -8899,8 +8396,7 @@ TerminateBufferIO(BufferDesc *buf, bool clear_dirty, uint32 set_flag_bits) buf_state |= set_flag_bits; UnlockBufHdr(buf, buf_state); - ResourceOwnerForgetBufferIO(CurrentResourceOwner, - BufferDescriptorGetBuffer(buf)); + ResourceOwnerForgetBufferIO(CurrentResourceOwner, BufferDescriptorGetBuffer(buf)); ConditionVariableBroadcast(BufferDescriptorGetIOCV(buf)); } @@ -8918,34 +8414,28 @@ void AbortBufferIO(Buffer buffer) { BufferDesc *buf_hdr = GetBufferDescriptor(buffer - 1); - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf_hdr); Assert(buf_state & (BM_IO_IN_PROGRESS | BM_TAG_VALID)); - if (!(buf_state & BM_VALID)) - { + if (!(buf_state & BM_VALID)) { Assert(!(buf_state & BM_DIRTY)); UnlockBufHdr(buf_hdr, buf_state); - } - else - { + } else { Assert(buf_state & BM_DIRTY); UnlockBufHdr(buf_hdr, buf_state); /* Issue notice if this is not the first failure... */ - if (buf_state & BM_IO_ERROR) - { + if (buf_state & BM_IO_ERROR) { /* Buffer is pinned, so we can read tag without spinlock */ - char *path; + char *path; path = relpathperm(BufTagGetRelFileLocator(&buf_hdr->tag), BufTagGetForkNum(&buf_hdr->tag)); - ereport(WARNING, - (errcode(ERRCODE_IO_ERROR), - errmsg("could not write block %u of %s", - buf_hdr->tag.blockNum, path), - errdetail("Multiple failures --- write error might be permanent."))); + ereport(WARNING, (errcode(ERRCODE_IO_ERROR), + errmsg("could not write block %u of %s", buf_hdr->tag.blockNum, path), + errdetail("Multiple failures --- write error might be permanent."))); pfree(path); } } @@ -8959,16 +8449,14 @@ AbortBufferIO(Buffer buffer) static void shared_buffer_write_error_callback(void *arg) { - BufferDesc *bufHdr = (BufferDesc *) arg; + BufferDesc *bufHdr = (BufferDesc *)arg; /* Buffer is pinned, so we can read the tag without locking the spinlock */ - if (bufHdr != NULL) - { - char *path = relpathperm(BufTagGetRelFileLocator(&bufHdr->tag), - BufTagGetForkNum(&bufHdr->tag)); + if (bufHdr != NULL) { + char *path + = relpathperm(BufTagGetRelFileLocator(&bufHdr->tag), BufTagGetForkNum(&bufHdr->tag)); - errcontext("writing block %u of relation %s", - bufHdr->tag.blockNum, path); + errcontext("writing block %u of relation %s", bufHdr->tag.blockNum, path); pfree(path); } } @@ -8979,16 +8467,13 @@ shared_buffer_write_error_callback(void *arg) static void local_buffer_write_error_callback(void *arg) { - BufferDesc *bufHdr = (BufferDesc *) arg; + BufferDesc *bufHdr = (BufferDesc *)arg; - if (bufHdr != NULL) - { - char *path = relpathbackend(BufTagGetRelFileLocator(&bufHdr->tag), - MyBackendId, - BufTagGetForkNum(&bufHdr->tag)); + if (bufHdr != NULL) { + char *path = relpathbackend(BufTagGetRelFileLocator(&bufHdr->tag), MyBackendId, + BufTagGetForkNum(&bufHdr->tag)); - errcontext("writing block %u of relation %s", - bufHdr->tag.blockNum, path); + errcontext("writing block %u of relation %s", bufHdr->tag.blockNum, path); pfree(path); } } @@ -8999,8 +8484,8 @@ local_buffer_write_error_callback(void *arg) static int rlocator_comparator(const void *p1, const void *p2) { - RelFileLocator n1 = *(const RelFileLocator *) p1; - RelFileLocator n2 = *(const RelFileLocator *) p2; + RelFileLocator n1 = *(const RelFileLocator *)p1; + RelFileLocator n2 = *(const RelFileLocator *)p2; if (n1.relNumber < n2.relNumber) return -1; @@ -9027,14 +8512,13 @@ uint32 LockBufHdr(BufferDesc *desc) { SpinDelayStatus delayStatus; - uint32 old_buf_state; + uint32 old_buf_state; Assert(!BufferIsLocal(BufferDescriptorGetBuffer(desc))); init_local_spin_delay(&delayStatus); - while (true) - { + while (true) { /* set BM_LOCKED flag */ old_buf_state = pg_atomic_fetch_or_u32(&desc->state, BM_LOCKED); /* if it wasn't set before we're OK */ @@ -9057,14 +8541,13 @@ static uint32 WaitBufHdrUnlocked(BufferDesc *buf) { SpinDelayStatus delayStatus; - uint32 buf_state; + uint32 buf_state; init_local_spin_delay(&delayStatus); buf_state = pg_atomic_read_u32(&buf->state); - while (buf_state & BM_LOCKED) - { + while (buf_state & BM_LOCKED) { perform_spin_delay(&delayStatus); buf_state = pg_atomic_read_u32(&buf->state); } @@ -9080,7 +8563,7 @@ WaitBufHdrUnlocked(BufferDesc *buf) static inline int buffertag_comparator(const BufferTag *ba, const BufferTag *bb) { - int ret; + int ret; RelFileLocator rlocatora; RelFileLocator rlocatorb; @@ -9145,8 +8628,8 @@ ckpt_buforder_comparator(const CkptSortItem *a, const CkptSortItem *b) static int ts_ckpt_progress_comparator(Datum a, Datum b, void *arg) { - CkptTsStatus *sa = (CkptTsStatus *) a; - CkptTsStatus *sb = (CkptTsStatus *) b; + CkptTsStatus *sa = (CkptTsStatus *)a; + CkptTsStatus *sb = (CkptTsStatus *)b; /* we want a min-heap, so return 1 for the a < b */ if (sa->progress < sb->progress) @@ -9178,8 +8661,7 @@ WritebackContextInit(WritebackContext *context, int *max_pending) * Add buffer to list of pending writeback requests. */ void -ScheduleBufferTagForWriteback(WritebackContext *wb_context, IOContext io_context, - BufferTag *tag) +ScheduleBufferTagForWriteback(WritebackContext *wb_context, IOContext io_context, BufferTag *tag) { PendingWriteback *pending; @@ -9190,8 +8672,7 @@ ScheduleBufferTagForWriteback(WritebackContext *wb_context, IOContext io_context * Add buffer to the pending writeback array, unless writeback control is * disabled. */ - if (*wb_context->max_pending > 0) - { + if (*wb_context->max_pending > 0) { Assert(*wb_context->max_pending <= WRITEBACK_MAX_PENDING_FLUSHES); pending = &wb_context->pending_writebacks[wb_context->nr_pending++]; @@ -9225,8 +8706,8 @@ ScheduleBufferTagForWriteback(WritebackContext *wb_context, IOContext io_context void IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context) { - instr_time io_start; - int i; + instr_time io_start; + int i; if (wb_context->nr_pending == 0) return; @@ -9235,8 +8716,7 @@ IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context) * Executing the writes in-order can make them a lot faster, and allows to * merge writeback requests to consecutive blocks into larger writebacks. */ - sort_pending_writebacks(wb_context->pending_writebacks, - wb_context->nr_pending); + sort_pending_writebacks(wb_context->pending_writebacks, wb_context->nr_pending); io_start = pgstat_prepare_io_time(); @@ -9245,15 +8725,14 @@ IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context) * through the, now sorted, array of pending flushes, and look forward to * find all neighbouring (or identical) writes. */ - for (i = 0; i < wb_context->nr_pending; i++) - { + for (i = 0; i < wb_context->nr_pending; i++) { PendingWriteback *cur; PendingWriteback *next; SMgrRelation reln; - int ahead; - BufferTag tag; + int ahead; + BufferTag tag; RelFileLocator currlocator; - Size nblocks = 1; + Size nblocks = 1; cur = &wb_context->pending_writebacks[i]; tag = cur->tag; @@ -9263,15 +8742,12 @@ IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context) * Peek ahead, into following writeback requests, to see if they can * be combined with the current one. */ - for (ahead = 0; i + ahead + 1 < wb_context->nr_pending; ahead++) - { - + for (ahead = 0; i + ahead + 1 < wb_context->nr_pending; ahead++) { next = &wb_context->pending_writebacks[i + ahead + 1]; /* different file, stop */ - if (!RelFileLocatorEquals(currlocator, - BufTagGetRelFileLocator(&next->tag)) || - BufTagGetForkNum(&cur->tag) != BufTagGetForkNum(&next->tag)) + if (!RelFileLocatorEquals(currlocator, BufTagGetRelFileLocator(&next->tag)) + || BufTagGetForkNum(&cur->tag) != BufTagGetForkNum(&next->tag)) break; /* ok, block queued twice, skip */ @@ -9297,8 +8773,8 @@ IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context) * Assume that writeback requests are only issued for buffers containing * blocks of permanent relations. */ - pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, - IOOP_WRITEBACK, io_start, wb_context->nr_pending); + pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, IOOP_WRITEBACK, io_start, + wb_context->nr_pending); wb_context->nr_pending = 0; } @@ -9315,9 +8791,7 @@ TestForOldSnapshot_impl(Snapshot snapshot, Relation relation) { if (RelationAllowsEarlyPruning(relation) && (snapshot)->whenTaken < GetOldSnapshotThresholdTimestamp()) - ereport(ERROR, - (errcode(ERRCODE_SNAPSHOT_TOO_OLD), - errmsg("snapshot too old"))); + ereport(ERROR, (errcode(ERRCODE_SNAPSHOT_TOO_OLD), errmsg("snapshot too old"))); } @@ -9391,7 +8865,7 @@ TestForOldSnapshot_impl(Snapshot snapshot, Relation relation) static XLogRecPtr cluster_gcs_clamp_ship_flush_lsn(XLogRecPtr page_lsn) { - XLogRecPtr local_insert; + XLogRecPtr local_insert; if (XLogRecPtrIsInvalid(page_lsn) || RecoveryInProgress()) return page_lsn; @@ -9420,41 +8894,35 @@ cluster_bufmgr_pin_for_gcs_locked(BufferDesc *buf, uint32 buf_state) static void cluster_bufmgr_unpin_for_gcs(BufferDesc *buf) { - uint32 buf_state; - uint32 old_buf_state; + uint32 buf_state; + uint32 old_buf_state; Assert(!LWLockHeldByMe(BufferDescriptorGetContentLock(buf))); VALGRIND_MAKE_MEM_NOACCESS(BufHdrGetBlock(buf), BLCKSZ); old_buf_state = pg_atomic_read_u32(&buf->state); - for (;;) - { + for (;;) { if (old_buf_state & BM_LOCKED) old_buf_state = WaitBufHdrUnlocked(buf); buf_state = old_buf_state; buf_state -= BUF_REFCOUNT_ONE; - if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state, - buf_state)) + if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state, buf_state)) break; } - if (buf_state & BM_PIN_COUNT_WAITER) - { + if (buf_state & BM_PIN_COUNT_WAITER) { buf_state = LockBufHdr(buf); - if ((buf_state & BM_PIN_COUNT_WAITER) && - BUF_STATE_GET_REFCOUNT(buf_state) == 1) - { - int wait_backend_pgprocno = buf->wait_backend_pgprocno; + if ((buf_state & BM_PIN_COUNT_WAITER) && BUF_STATE_GET_REFCOUNT(buf_state) == 1) { + int wait_backend_pgprocno = buf->wait_backend_pgprocno; buf_state &= ~BM_PIN_COUNT_WAITER; UnlockBufHdr(buf, buf_state); ProcSendSignal(wait_backend_pgprocno); - } - else + } else UnlockBufHdr(buf, buf_state); } } @@ -9473,21 +8941,20 @@ cluster_bufmgr_unpin_for_gcs(BufferDesc *buf) bool cluster_bufmgr_probe_block_for_gcs(BufferTag tag) { - uint32 hashcode = BufTableHashCode(&tag); - LWLock *partition_lock = BufMappingPartitionLock(hashcode); - int buf_id; - bool valid = false; + uint32 hashcode = BufTableHashCode(&tag); + LWLock *partition_lock = BufMappingPartitionLock(hashcode); + int buf_id; + bool valid = false; LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id >= 0) - { + if (buf_id >= 0) { BufferDesc *buf = GetBufferDescriptor(buf_id); - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf); valid = BufferTagsEqual(&buf->tag, &tag) - && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); UnlockBufHdr(buf, buf_state); } LWLockRelease(partition_lock); @@ -9516,12 +8983,12 @@ cluster_bufmgr_read_storage_scn_for_gcs(BufferTag tag, SCN *out_page_scn) reln = smgropen(BufTagGetRelFileLocator(&tag), InvalidBackendId); smgrread(reln, BufTagGetForkNum(&tag), tag.blockNum, scratch.data); - if (!PageIsVerifiedExtended((Page) scratch.data, tag.blockNum, + if (!PageIsVerifiedExtended((Page)scratch.data, tag.blockNum, PIV_LOG_WARNING | PIV_REPORT_STAT)) return false; if (out_page_scn != NULL) - *out_page_scn = ((PageHeader) scratch.data)->pd_block_scn; + *out_page_scn = ((PageHeader)scratch.data)->pd_block_scn; return true; } @@ -9544,16 +9011,16 @@ cluster_bufmgr_read_storage_scn_for_gcs(BufferTag tag, SCN *out_page_scn) bool cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char *dst) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - LWLock *content_lock; - XLogRecPtr first_lsn; - XLogRecPtr second_lsn; - int retries; - bool stable; - Page page; + LWLock *content_lock; + XLogRecPtr first_lsn; + XLogRecPtr second_lsn; + int retries; + bool stable; + Page page; Assert(dst != NULL); Assert(out_page_lsn != NULL); @@ -9563,8 +9030,7 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); return false; } @@ -9572,19 +9038,17 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char /* Partition lock keeps the buffer from being recycled before we raw-pin. */ { - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf); /* Re-verify tag under header lock to defend against tag-rewrite * races between the partition-lock-protected lookup and the pin. */ - if (!BufferTagsEqual(&buf->tag, &tag)) - { + if (!BufferTagsEqual(&buf->tag, &tag)) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; } - if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) - { + if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; @@ -9594,7 +9058,7 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char LWLockRelease(partition_lock); content_lock = BufferDescriptorGetContentLock(buf); - page = (Page) BufHdrGetBlock(buf); + page = (Page)BufHdrGetBlock(buf); /* * HC89: revalidation loop with single retry budget. retries = 0 — first @@ -9602,8 +9066,7 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char * fail-closed */ stable = false; - for (retries = 0; retries < 2; retries++) - { + for (retries = 0; retries < 2; retries++) { /* Read page_lsn under content_lock SHARED. */ LWLockAcquire(content_lock, LW_SHARED); first_lsn = PageGetLSN(page); @@ -9617,7 +9080,7 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char */ #ifdef USE_CLUSTER_UNIT if (cluster_gcs_block_test_xlog_flush_hook != NULL) - cluster_gcs_block_test_xlog_flush_hook((uint64) first_lsn); + cluster_gcs_block_test_xlog_flush_hook((uint64)first_lsn); #endif if (!XLogRecPtrIsInvalid(first_lsn)) XLogFlush(cluster_gcs_clamp_ship_flush_lsn(first_lsn)); @@ -9632,11 +9095,10 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char { uint32 buf_state = LockBufHdr(buf); bool current = BufferTagsEqual(&buf->tag, &tag) - && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); UnlockBufHdr(buf, buf_state); - if (!current) - { + if (!current) { LWLockRelease(content_lock); break; } @@ -9650,17 +9112,15 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char * consecutive LSN drift events. retries 0 + drift available means * second_lsn != first_lsn synthetically. */ - if (cluster_gcs_block_test_lsn_drift_hook != NULL) - { - int drift_remaining = cluster_gcs_block_test_lsn_drift_hook(); + if (cluster_gcs_block_test_lsn_drift_hook != NULL) { + int drift_remaining = cluster_gcs_block_test_lsn_drift_hook(); if (drift_remaining > retries) - second_lsn = first_lsn + 1; /* synthetic mismatch */ + second_lsn = first_lsn + 1; /* synthetic mismatch */ } #endif - if (BufferTagsEqual(&buf->tag, &tag) && first_lsn == second_lsn) - { + if (BufferTagsEqual(&buf->tag, &tag) && first_lsn == second_lsn) { memcpy(dst, page, BLCKSZ); *out_page_lsn = second_lsn; LWLockRelease(content_lock); @@ -9690,15 +9150,15 @@ bool cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page_lsn, void **out_page_addr, BufferDesc **out_buf) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - LWLock *content_lock; - XLogRecPtr first_lsn; - XLogRecPtr second_lsn; - int retries; - Page page; + LWLock *content_lock; + XLogRecPtr first_lsn; + XLogRecPtr second_lsn; + int retries; + Page page; Assert(out_page_lsn != NULL); Assert(out_page_addr != NULL); @@ -9713,20 +9173,18 @@ cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); return false; } buf = GetBufferDescriptor(buf_id); { - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf); if (!BufferTagsEqual(&buf->tag, &tag) - || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) - { + || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; @@ -9736,17 +9194,16 @@ cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page LWLockRelease(partition_lock); content_lock = BufferDescriptorGetContentLock(buf); - page = (Page) BufHdrGetBlock(buf); + page = (Page)BufHdrGetBlock(buf); - for (retries = 0; retries < 2; retries++) - { + for (retries = 0; retries < 2; retries++) { LWLockAcquire(content_lock, LW_SHARED); first_lsn = PageGetLSN(page); LWLockRelease(content_lock); #ifdef USE_CLUSTER_UNIT if (cluster_gcs_block_test_xlog_flush_hook != NULL) - cluster_gcs_block_test_xlog_flush_hook((uint64) first_lsn); + cluster_gcs_block_test_xlog_flush_hook((uint64)first_lsn); #endif if (!XLogRecPtrIsInvalid(first_lsn)) XLogFlush(cluster_gcs_clamp_ship_flush_lsn(first_lsn)); @@ -9755,11 +9212,10 @@ cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page { uint32 buf_state = LockBufHdr(buf); bool current = BufferTagsEqual(&buf->tag, &tag) - && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); UnlockBufHdr(buf, buf_state); - if (!current) - { + if (!current) { LWLockRelease(content_lock); break; } @@ -9767,17 +9223,15 @@ cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page second_lsn = PageGetLSN(page); #ifdef USE_CLUSTER_UNIT - if (cluster_gcs_block_test_lsn_drift_hook != NULL) - { - int drift_remaining = cluster_gcs_block_test_lsn_drift_hook(); + if (cluster_gcs_block_test_lsn_drift_hook != NULL) { + int drift_remaining = cluster_gcs_block_test_lsn_drift_hook(); if (drift_remaining > retries) second_lsn = first_lsn + 1; } #endif - if (BufferTagsEqual(&buf->tag, &tag) && first_lsn == second_lsn) - { + if (BufferTagsEqual(&buf->tag, &tag) && first_lsn == second_lsn) { *out_page_lsn = second_lsn; *out_page_addr = page; *out_buf = buf; @@ -9809,7 +9263,7 @@ bool cluster_bufmgr_prepare_direct_land_target_for_gcs(BufferDesc *buf, BufferTag tag, void **out_page_addr) { - uint32 buf_state; + uint32 buf_state; Assert(out_page_addr != NULL); @@ -9818,11 +9272,8 @@ cluster_bufmgr_prepare_direct_land_target_for_gcs(BufferDesc *buf, BufferTag tag return false; buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) - || (buf_state & BM_TAG_VALID) == 0 - || (buf_state & BM_VALID) != 0 - || (buf_state & BM_IO_IN_PROGRESS) != 0) - { + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_TAG_VALID) == 0 + || (buf_state & BM_VALID) != 0 || (buf_state & BM_IO_IN_PROGRESS) != 0) { UnlockBufHdr(buf, buf_state); return false; } @@ -9843,23 +9294,20 @@ cluster_bufmgr_prepare_direct_land_target_for_gcs(BufferDesc *buf, BufferTag tag * BM_VALID through TerminateBufferIO. On failure, leave the buffer invalid. */ void -cluster_bufmgr_finish_direct_land_target_for_gcs(BufferDesc *buf, bool valid, - XLogRecPtr page_lsn) +cluster_bufmgr_finish_direct_land_target_for_gcs(BufferDesc *buf, bool valid, XLogRecPtr page_lsn) { if (buf == NULL) return; - if (valid) - { - LWLock *content_lock = BufferDescriptorGetContentLock(buf); - Page page = (Page) BufHdrGetBlock(buf); + if (valid) { + LWLock *content_lock = BufferDescriptorGetContentLock(buf); + Page page = (Page)BufHdrGetBlock(buf); LWLockAcquire(content_lock, LW_EXCLUSIVE); PageSetLSN(page, page_lsn); LWLockRelease(content_lock); TerminateBufferIO(buf, false, BM_VALID); - } - else + } else TerminateBufferIO(buf, false, 0); } @@ -9894,29 +9342,27 @@ cluster_bufmgr_finish_direct_land_target_for_gcs(BufferDesc *buf, bool valid, bool cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - LWLock *content_lock; - uint32 buf_state; - bool dirty; + LWLock *content_lock; + uint32 buf_state; + bool dirty; hashcode = BufTableHashCode(&tag); partition_lock = BufMappingPartitionLock(hashcode); LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); return false; } buf = GetBufferDescriptor(buf_id); buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) - { + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; @@ -9926,8 +9372,7 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) content_lock = BufferDescriptorGetContentLock(buf); LWLockAcquire(content_lock, LW_EXCLUSIVE); - if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) - { + if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); return false; @@ -9939,10 +9384,8 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) * this in-memory copy for its commit stamp (the P0-2 dependency) — the * caller falls back to the one-shot read-image ship. */ - if (!BufferTagsEqual(&buf->tag, &tag) - || (PcmState) buf->pcm_state != PCM_STATE_X - || cluster_itl_page_has_active_slot((Page) BufHdrGetBlock(buf))) - { + if (!BufferTagsEqual(&buf->tag, &tag) || (PcmState)buf->pcm_state != PCM_STATE_X + || cluster_itl_page_has_active_slot((Page)BufHdrGetBlock(buf))) { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); return false; @@ -9955,9 +9398,7 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) if (dirty) FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); - if (!cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_X_TO_S_DOWNGRADE, - cluster_node_id)) - { + if (!cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_X_TO_S_DOWNGRADE, cluster_node_id)) { /* Master refused (state moved under us) — leave local X untouched. */ LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); @@ -9968,7 +9409,7 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) * ownership transition -- set pcm_state and bump the generation atomically * (header spinlock) so a cached-X writer racing the content-lock window * detects the revoke even across an X->S->X ABA. */ - cluster_pcm_own_transition(buf, (uint8) PCM_STATE_S, 0, 0); + cluster_pcm_own_transition(buf, (uint8)PCM_STATE_S, 0, 0); LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); @@ -10003,11 +9444,11 @@ Buffer cluster_bufmgr_lock_resident_for_stamp(RelFileLocator rlocator, ForkNumber forknum, BlockNumber blocknum) { - BufferTag tag; - uint32 hashcode; - uint32 buf_state; - LWLock *partition_lock; - int buf_id; + BufferTag tag; + uint32 hashcode; + uint32 buf_state; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; InitBufferTag(&tag, &rlocator, forknum, blocknum); @@ -10019,8 +9460,7 @@ cluster_bufmgr_lock_resident_for_stamp(RelFileLocator rlocator, ForkNumber forkn LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { /* Not resident -> transferred away (or never cached). Skip stamp. */ LWLockRelease(partition_lock); return InvalidBuffer; @@ -10055,11 +9495,9 @@ cluster_bufmgr_lock_resident_for_stamp(RelFileLocator rlocator, ForkNumber forkn * legal commit-stamp target. */ buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) - || (buf_state & BM_VALID) == 0 + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0 || cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state) - || (cluster_pcm_own_flags_get(buf->buf_id) & PCM_OWN_FLAG_REVOKING) != 0) - { + || (cluster_pcm_own_flags_get(buf->buf_id) & PCM_OWN_FLAG_REVOKING) != 0) { UnlockBufHdr(buf, buf_state); LWLockRelease(BufferDescriptorGetContentLock(buf)); ReleaseBuffer(BufferDescriptorGetBuffer(buf)); @@ -10115,13 +9553,13 @@ cluster_bufmgr_unlock_resident_stamp(Buffer buffer) bool cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - LWLock *content_lock; - uint32 buf_state; - bool dirty; + LWLock *content_lock; + uint32 buf_state; + bool dirty; if (master_node < 0 || master_node == cluster_node_id) return false; @@ -10131,16 +9569,14 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); return false; } buf = GetBufferDescriptor(buf_id); buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) - { + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; @@ -10150,18 +9586,15 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) content_lock = BufferDescriptorGetContentLock(buf); LWLockAcquire(content_lock, LW_EXCLUSIVE); - if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) - { + if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); return false; } /* Same re-verify as the local variant: tag, PCM X, quiescent. */ - if (!BufferTagsEqual(&buf->tag, &tag) - || (PcmState) buf->pcm_state != PCM_STATE_X - || cluster_itl_page_has_active_slot((Page) BufHdrGetBlock(buf))) - { + if (!BufferTagsEqual(&buf->tag, &tag) || (PcmState)buf->pcm_state != PCM_STATE_X + || cluster_itl_page_has_active_slot((Page)BufHdrGetBlock(buf))) { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); return false; @@ -10182,12 +9615,9 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) * recording X@us. Drives the renotify self-heal TAP leg (t/348 L8). */ CLUSTER_INJECTION_POINT("cluster-gcs-block-yield-notify-drop"); - if (cluster_injection_should_skip("cluster-gcs-block-yield-notify-drop")) - { + if (cluster_injection_should_skip("cluster-gcs-block-yield-notify-drop")) { /* simulated post-handoff loss: fall through to the S flip */ - } - else if (!cluster_gcs_send_transition_nowait(tag, PCM_TRANS_X_TO_S_DOWNGRADE, master_node)) - { + } else if (!cluster_gcs_send_transition_nowait(tag, PCM_TRANS_X_TO_S_DOWNGRADE, master_node)) { /* Notify not handed to transport — keep local X, master unchanged. */ LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); @@ -10198,7 +9628,7 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) * ownership transition -- set pcm_state and bump the generation atomically * (header spinlock) so a cached-X writer racing the content-lock window * detects the revoke even across an X->S->X ABA. */ - cluster_pcm_own_transition(buf, (uint8) PCM_STATE_S, 0, 0); + cluster_pcm_own_transition(buf, (uint8)PCM_STATE_S, 0, 0); LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); @@ -10223,12 +9653,12 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) bool cluster_bufmgr_renotify_s_for_gcs(BufferTag tag, int32 master_node) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - uint32 buf_state; - bool is_s; + uint32 buf_state; + bool is_s; if (master_node < 0 || master_node == cluster_node_id) return false; @@ -10238,8 +9668,7 @@ cluster_bufmgr_renotify_s_for_gcs(BufferTag tag, int32 master_node) LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); return false; } @@ -10247,7 +9676,7 @@ cluster_bufmgr_renotify_s_for_gcs(BufferTag tag, int32 master_node) buf_state = LockBufHdr(buf); is_s = BufferTagsEqual(&buf->tag, &tag) && (buf_state & BM_VALID) != 0 - && (PcmState) buf->pcm_state == PCM_STATE_S; + && (PcmState)buf->pcm_state == PCM_STATE_S; UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); @@ -10269,17 +9698,17 @@ bool cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_page_lsn, char *dst, ClusterSfDepVec *out_dep_vec) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - LWLock *content_lock; - XLogRecPtr first_lsn; - XLogRecPtr second_lsn; - int retries; - bool copied_stable; - bool stable; - Page page; + LWLock *content_lock; + XLogRecPtr first_lsn; + XLogRecPtr second_lsn; + int retries; + bool copied_stable; + bool stable; + Page page; if (dst == NULL || out_page_lsn == NULL || out_dep_vec == NULL) return false; @@ -10292,20 +9721,18 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); return false; } buf = GetBufferDescriptor(buf_id); { - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf); if (!BufferTagsEqual(&buf->tag, &tag) - || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) - { + || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; @@ -10315,20 +9742,18 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa LWLockRelease(partition_lock); content_lock = BufferDescriptorGetContentLock(buf); - page = (Page) BufHdrGetBlock(buf); + page = (Page)BufHdrGetBlock(buf); copied_stable = false; stable = false; - for (retries = 0; retries < 2; retries++) - { + for (retries = 0; retries < 2; retries++) { LWLockAcquire(content_lock, LW_SHARED); { uint32 buf_state = LockBufHdr(buf); bool current = BufferTagsEqual(&buf->tag, &tag) - && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); UnlockBufHdr(buf, buf_state); - if (!current) - { + if (!current) { LWLockRelease(content_lock); break; } @@ -10337,8 +9762,7 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa memcpy(dst, page, BLCKSZ); second_lsn = PageGetLSN(page); if (BufferTagsEqual(&buf->tag, &tag) && !XLogRecPtrIsInvalid(first_lsn) - && first_lsn == second_lsn) - { + && first_lsn == second_lsn) { *out_page_lsn = second_lsn; LWLockRelease(content_lock); copied_stable = true; @@ -10347,12 +9771,11 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa LWLockRelease(content_lock); } - if (copied_stable) - { + if (copied_stable) { ClusterSfDepVec existing_vec; - bool has_existing_deps; - bool add_local_dep; - uint32 buf_state; + bool has_existing_deps; + bool add_local_dep; + uint32 buf_state; /* * Keep lock ordering compatible with receiver install: @@ -10361,18 +9784,17 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa * the dep store after releasing content_lock, while the raw pin keeps * this descriptor/tag from being recycled. */ - has_existing_deps = - cluster_sf_dep_vec_for_ship(BufferDescriptorGetBuffer(buf), &existing_vec); + has_existing_deps + = cluster_sf_dep_vec_for_ship(BufferDescriptorGetBuffer(buf), &existing_vec); cluster_sf_dep_vec_reset(out_dep_vec); if (has_existing_deps) - (void) cluster_sf_dep_vec_union(out_dep_vec, &existing_vec); + (void)cluster_sf_dep_vec_union(out_dep_vec, &existing_vec); buf_state = LockBufHdr(buf); - add_local_dep = (buf_state & (BM_DIRTY | BM_JUST_DIRTIED)) != 0 - || !has_existing_deps; + add_local_dep = (buf_state & (BM_DIRTY | BM_JUST_DIRTIED)) != 0 || !has_existing_deps; UnlockBufHdr(buf, buf_state); if (add_local_dep) - (void) cluster_sf_dep_vec_set(out_dep_vec, cluster_node_id, *out_page_lsn); + (void)cluster_sf_dep_vec_set(out_dep_vec, cluster_node_id, *out_page_lsn); stable = !cluster_sf_dep_vec_is_empty(out_dep_vec); } @@ -10401,11 +9823,11 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa * authoritative lost-write check). No PCM state is mutated. */ int -cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, - ClusterGcsRedeclareCallback cb, void *arg) +cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, ClusterGcsRedeclareCallback cb, + void *arg) { - int i; - int end; + int i; + int end; Assert(cb != NULL); @@ -10415,23 +9837,20 @@ cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, if (end > NBuffers) end = NBuffers; - for (i = start_buf; i < end; i++) - { + for (i = start_buf; i < end; i++) { BufferDesc *buf = GetBufferDescriptor(i); - uint32 buf_state; - uint8 mode; - BufferTag tag; - LWLock *content_lock; - XLogRecPtr page_lsn; - SCN page_scn; /* spec-2.41 D3 — re-declare SCN carrier */ + uint32 buf_state; + uint8 mode; + BufferTag tag; + LWLock *content_lock; + XLogRecPtr page_lsn; + SCN page_scn; /* spec-2.41 D3 — re-declare SCN carrier */ buf_state = LockBufHdr(buf); mode = buf->pcm_state; - if ((buf_state & BM_VALID) == 0 - || (buf_state & BM_IO_IN_PROGRESS) != 0 - || (mode != (uint8) PCM_STATE_S && mode != (uint8) PCM_STATE_X) - || !cluster_bufmgr_should_pcm_track(buf)) - { + if ((buf_state & BM_VALID) == 0 || (buf_state & BM_IO_IN_PROGRESS) != 0 + || (mode != (uint8)PCM_STATE_S && mode != (uint8)PCM_STATE_X) + || !cluster_bufmgr_should_pcm_track(buf)) { UnlockBufHdr(buf, buf_state); continue; } @@ -10441,11 +9860,11 @@ cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, content_lock = BufferDescriptorGetContentLock(buf); LWLockAcquire(content_lock, LW_SHARED); - page_lsn = PageGetLSN((Page) BufHdrGetBlock(buf)); + page_lsn = PageGetLSN((Page)BufHdrGetBlock(buf)); /* PGRAC: spec-2.41 D3 — also read pd_block_scn so the re-declare carries * the cross-node version for the detector's SCN watermark (alongside * page_lsn for the redo-coverage required_lsn). */ - page_scn = ((PageHeader) BufHdrGetBlock(buf))->pd_block_scn; + page_scn = ((PageHeader)BufHdrGetBlock(buf))->pd_block_scn; LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); @@ -10485,18 +9904,18 @@ ClusterBufmgrGcsDropResult cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode, XLogRecPtr *out_page_lsn, SCN *out_page_scn) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - uint32 buf_state; - XLogRecPtr page_lsn = InvalidXLogRecPtr; - SCN page_scn = InvalidScn; /* PGRAC: spec-2.41 D3 — page version for the ACK SCN carrier */ - bool was_dirty = false; - uint8 saved_pcm_state; - uint64 staged_gen; /* PGRAC W2: ownership gen captured at stage-N */ + uint32 buf_state; + XLogRecPtr page_lsn = InvalidXLogRecPtr; + SCN page_scn = InvalidScn; /* PGRAC: spec-2.41 D3 — page version for the ACK SCN carrier */ + bool was_dirty = false; + uint8 saved_pcm_state; + uint64 staged_gen; /* PGRAC W2: ownership gen captured at stage-N */ - (void) expected_mode; + (void)expected_mode; if (out_page_lsn != NULL) *out_page_lsn = InvalidXLogRecPtr; @@ -10508,8 +9927,7 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); return CLUSTER_BUFMGR_GCS_DROP_NOT_RESIDENT; } @@ -10519,8 +9937,7 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode /* Re-verify tag under the header lock to defend against a tag-rewrite race * between the partition-lock lookup and the raw pin (copy_block_for_gcs * convention). */ - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) - { + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return CLUSTER_BUFMGR_GCS_DROP_NOT_RESIDENT; @@ -10533,8 +9950,7 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode * ClusterBufmgrGcsDropResult). Bail before the WAL flush — nothing is * dropped, so the HC123 flush-before-drop invariant is not owed yet. */ - if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) - { + if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return CLUSTER_BUFMGR_GCS_DROP_PINNED; @@ -10555,15 +9971,15 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode * cluster_bufmgr_copy_block_for_gcs and cluster_bufmgr_redeclare_scan_chunk. * The pin keeps the buffer identity stable, so the pre-pin tag match holds. */ - cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); /* raw-pin + unlock header */ + cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); /* raw-pin + unlock header */ LWLockRelease(partition_lock); { - LWLock *content_lock = BufferDescriptorGetContentLock(buf); - Page page = (Page) BufHdrGetBlock(buf); + LWLock *content_lock = BufferDescriptorGetContentLock(buf); + Page page = (Page)BufHdrGetBlock(buf); LWLockAcquire(content_lock, LW_SHARED); page_lsn = PageGetLSN(page); - page_scn = ((PageHeader) page)->pd_block_scn; + page_scn = ((PageHeader)page)->pd_block_scn; LWLockRelease(content_lock); } cluster_bufmgr_unpin_for_gcs(buf); @@ -10591,8 +10007,7 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode * cluster_bufmgr_drop_block_for_gcs_no_wire below). */ buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) - { + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) { UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_NOT_RESIDENT; } @@ -10602,15 +10017,14 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode * unpinned for the XLogFlush re-fails the drop (bounded contract; the * pcm_state is untouched so nothing observes a half-dropped state). */ - if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) - { + if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) { UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_PINNED; } saved_pcm_state = buf->pcm_state; - staged_gen = cluster_pcm_own_gen_get(buf->buf_id); /* PGRAC W2 */ - buf->pcm_state = (uint8) PCM_STATE_N; + staged_gen = cluster_pcm_own_gen_get(buf->buf_id); /* PGRAC W2 */ + buf->pcm_state = (uint8)PCM_STATE_N; /* PGRAC: spec-6.12h D-h1 — keep a Past Image instead of dropping. */ if (cluster_bufmgr_convert_to_pi_locked(buf, buf_state)) @@ -10623,8 +10037,8 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode * (it was cleared for the eviction hook) so the still-resident copy keeps * its true PCM state. */ - cluster_bufmgr_in_gcs_drop = true; /* gates the drop-prepin inject */ - if (!InvalidateBufferTry(buf)) /* releases the header spinlock */ + cluster_bufmgr_in_gcs_drop = true; /* gates the drop-prepin inject */ + if (!InvalidateBufferTry(buf)) /* releases the header spinlock */ { cluster_bufmgr_in_gcs_drop = false; @@ -10649,7 +10063,7 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode */ CLUSTER_INJECTION_POINT("cluster-pcm-restore-aba-force-round"); if (cluster_injection_should_skip("cluster-pcm-restore-aba-force-round")) - cluster_pcm_own_transition(buf, (uint8) PCM_STATE_N, 0, 0); + cluster_pcm_own_transition(buf, (uint8)PCM_STATE_N, 0, 0); buf_state = LockBufHdr(buf); @@ -10665,13 +10079,11 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode * moved, the block was legitimately re-owned and dropped in between, * so leave it at N. */ - if (BufferTagsEqual(&buf->tag, &tag) && - buf->pcm_state == (uint8) PCM_STATE_N && - cluster_pcm_own_gen_get(buf->buf_id) == staged_gen) + if (BufferTagsEqual(&buf->tag, &tag) && buf->pcm_state == (uint8)PCM_STATE_N + && cluster_pcm_own_gen_get(buf->buf_id) == staged_gen) buf->pcm_state = saved_pcm_state; - else if (BufferTagsEqual(&buf->tag, &tag) && - buf->pcm_state == (uint8) PCM_STATE_N && - cluster_pcm_own_gen_get(buf->buf_id) != staged_gen) + else if (BufferTagsEqual(&buf->tag, &tag) && buf->pcm_state == (uint8)PCM_STATE_N + && cluster_pcm_own_gen_get(buf->buf_id) != staged_gen) cluster_pcm_note_restore_aba_detected(); UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_PINNED; @@ -10707,11 +10119,11 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode SCN cluster_bufmgr_read_block_scn_for_gcs(BufferDesc *buf) { - LWLock *content_lock = BufferDescriptorGetContentLock(buf); - SCN scn; + LWLock *content_lock = BufferDescriptorGetContentLock(buf); + SCN scn; LWLockAcquire(content_lock, LW_SHARED); - scn = ((PageHeader) BufHdrGetBlock(buf))->pd_block_scn; + scn = ((PageHeader)BufHdrGetBlock(buf))->pd_block_scn; LWLockRelease(content_lock); return scn; } @@ -10732,7 +10144,7 @@ bool cluster_bufmgr_block_is_extension_for_gcs(BufferTag tag) { SMgrRelation reln = smgropen(BufTagGetRelFileLocator(&tag), InvalidBackendId); - ForkNumber fork = BufTagGetForkNum(&tag); + ForkNumber fork = BufTagGetForkNum(&tag); /* Drop any cached size so a concurrent extension is not missed. */ smgrrelease(reln); @@ -10743,12 +10155,12 @@ bool cluster_bufmgr_refresh_block_from_storage_for_gcs(BufferDesc *buf, SCN *out_page_scn) { PGAlignedBlock scratch; - BufferTag tag = buf->tag; + BufferTag tag = buf->tag; SMgrRelation reln; - LWLock *content_lock; - uint32 buf_state; - bool dirty; - bool write_permitted; + LWLock *content_lock; + uint32 buf_state; + bool dirty; + bool write_permitted; if (out_page_scn != NULL) *out_page_scn = InvalidScn; @@ -10763,12 +10175,13 @@ cluster_bufmgr_refresh_block_from_storage_for_gcs(BufferDesc *buf, SCN *out_page */ reln = smgropen(BufTagGetRelFileLocator(&tag), InvalidBackendId); smgrread(reln, BufTagGetForkNum(&tag), tag.blockNum, scratch.data); - if (!PageIsVerifiedExtended((Page) scratch.data, tag.blockNum, + if (!PageIsVerifiedExtended((Page)scratch.data, tag.blockNum, PIV_LOG_WARNING | PIV_REPORT_STAT)) - ereport(ERROR, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("invalid page in block %u of relation %u during GCS storage-fallback refresh", - tag.blockNum, tag.relNumber))); + ereport( + ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("invalid page in block %u of relation %u during GCS storage-fallback refresh", + tag.blockNum, tag.relNumber))); content_lock = BufferDescriptorGetContentLock(buf); LWLockAcquire(content_lock, LW_EXCLUSIVE); @@ -10776,16 +10189,15 @@ cluster_bufmgr_refresh_block_from_storage_for_gcs(BufferDesc *buf, SCN *out_page buf_state = LockBufHdr(buf); dirty = (buf_state & BM_DIRTY) != 0; UnlockBufHdr(buf, buf_state); - if (dirty || !write_permitted) - { + if (dirty || !write_permitted) { LWLockRelease(content_lock); return false; } - memcpy((char *) BufHdrGetBlock(buf), scratch.data, BLCKSZ); + memcpy((char *)BufHdrGetBlock(buf), scratch.data, BLCKSZ); LWLockRelease(content_lock); if (out_page_scn != NULL) - *out_page_scn = ((PageHeader) scratch.data)->pd_block_scn; + *out_page_scn = ((PageHeader)scratch.data)->pd_block_scn; return true; } @@ -10829,9 +10241,8 @@ static bool cluster_bufmgr_convert_to_pi_locked(BufferDesc *buf, uint32 buf_state) { if (!cluster_past_image) - return false; /* wave off: caller drops as today */ - if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) - { + return false; /* wave off: caller drops as today */ + if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) { /* Pinned: fail-safe fallback to the plain drop (single lock-free * atomic tick; same class as PG's own buf->state atomics under * this spinlock). */ @@ -10840,7 +10251,7 @@ cluster_bufmgr_convert_to_pi_locked(BufferDesc *buf, uint32 buf_state) } buf_state &= ~(BM_VALID | BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR); - buf->buffer_type = (uint8) BUF_TYPE_PI; + buf->buffer_type = (uint8)BUF_TYPE_PI; /* D-h3a ship-SCN boundary stamp — see the header note on why the * clock sample must sit inside this lock hold. An unarmed clock * stamps InvalidScn and the D-h3 gate fails closed to UNUSABLE. */ @@ -10866,11 +10277,11 @@ cluster_bufmgr_convert_to_pi_locked(BufferDesc *buf, uint32 buf_state) PcmLockMode cluster_bufmgr_block_pcm_state(BufferTag tag) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - uint32 buf_state; + uint32 buf_state; PcmLockMode mode = PCM_LOCK_MODE_N; hashcode = BufTableHashCode(&tag); @@ -10878,8 +10289,7 @@ cluster_bufmgr_block_pcm_state(BufferTag tag) LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); return PCM_LOCK_MODE_N; } @@ -10887,7 +10297,7 @@ cluster_bufmgr_block_pcm_state(BufferTag tag) buf_state = LockBufHdr(buf); if (BufferTagsEqual(&buf->tag, &tag) && (buf_state & BM_VALID) != 0) - mode = (PcmLockMode) buf->pcm_state; + mode = (PcmLockMode)buf->pcm_state; UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); @@ -10906,20 +10316,19 @@ cluster_bufmgr_block_pcm_state(BufferTag tag) bool cluster_bufmgr_block_grant_pending(BufferTag tag) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - uint32 buf_state; - bool pending = false; + uint32 buf_state; + bool pending = false; hashcode = BufTableHashCode(&tag); partition_lock = BufMappingPartitionLock(hashcode); LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); return false; } @@ -10968,15 +10377,15 @@ ClusterBufmgrGcsDropResult cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn, XLogRecPtr *out_page_lsn) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - uint32 buf_state; - XLogRecPtr page_lsn = InvalidXLogRecPtr; - bool was_dirty = false; - uint8 saved_pcm_state; - uint64 staged_gen; /* PGRAC W2: ownership gen captured at stage-N */ + uint32 buf_state; + XLogRecPtr page_lsn = InvalidXLogRecPtr; + bool was_dirty = false; + uint8 saved_pcm_state; + uint64 staged_gen; /* PGRAC W2: ownership gen captured at stage-N */ if (out_page_lsn != NULL) *out_page_lsn = InvalidXLogRecPtr; @@ -10986,16 +10395,14 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); return CLUSTER_BUFMGR_GCS_DROP_NOT_RESIDENT; } buf = GetBufferDescriptor(buf_id); buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) - { + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return CLUSTER_BUFMGR_GCS_DROP_NOT_RESIDENT; @@ -11007,15 +10414,14 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn * retryable deny instead of parking InvalidateBuffer's pin-wait loop in * the dispatch pump). */ - if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) - { + if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return CLUSTER_BUFMGR_GCS_DROP_PINNED; } was_dirty = (buf_state & BM_DIRTY) != 0; { - Page page = (Page) BufHdrGetBlock(buf); + Page page = (Page)BufHdrGetBlock(buf); page_lsn = PageGetLSN(page); } @@ -11033,8 +10439,7 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn * the current page (Rule 8.A — never grant a stale image over a * committed write). expected_lsn == Invalid skips the gate. */ - if (!XLogRecPtrIsInvalid(expected_lsn) && page_lsn != expected_lsn) - { + if (!XLogRecPtrIsInvalid(expected_lsn) && page_lsn != expected_lsn) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return CLUSTER_BUFMGR_GCS_DROP_STALE; @@ -11066,16 +10471,14 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn * one -- BLOCKER A). */ buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) - { + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) { UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_NOT_RESIDENT; } /* PGRAC: GCS serve-stall round-5 (A2) — a pin acquired during the * XLogFlush window re-fails the drop (nothing observed half-dropped). */ - if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) - { + if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) { UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_PINNED; } @@ -11086,16 +10489,15 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn * during the (blocking) XLogFlush window above. A LSN past expected_lsn * means the ship image is stale; refuse without touching pcm_state * (Rule 8.A). */ - if (!XLogRecPtrIsInvalid(expected_lsn) && - PageGetLSN((Page) BufHdrGetBlock(buf)) != expected_lsn) - { + if (!XLogRecPtrIsInvalid(expected_lsn) + && PageGetLSN((Page)BufHdrGetBlock(buf)) != expected_lsn) { UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_STALE; } saved_pcm_state = buf->pcm_state; - staged_gen = cluster_pcm_own_gen_get(buf->buf_id); /* PGRAC W2 */ - buf->pcm_state = (uint8) PCM_STATE_N; + staged_gen = cluster_pcm_own_gen_get(buf->buf_id); /* PGRAC W2 */ + buf->pcm_state = (uint8)PCM_STATE_N; /* * PGRAC: spec-6.12h D-h1 — keep a Past Image instead of dropping (the @@ -11106,8 +10508,8 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn /* PGRAC: GCS serve-stall round-5 (A2) — bounded drop; restore the * residency mode on a raced pin (mirrors the invalidate wrapper). */ - cluster_bufmgr_in_gcs_drop = true; /* gates the drop-prepin inject */ - if (!InvalidateBufferTry(buf)) /* releases the header spinlock */ + cluster_bufmgr_in_gcs_drop = true; /* gates the drop-prepin inject */ + if (!InvalidateBufferTry(buf)) /* releases the header spinlock */ { cluster_bufmgr_in_gcs_drop = false; @@ -11125,7 +10527,7 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn /* W2 RED force-round — see the twin arm above. */ CLUSTER_INJECTION_POINT("cluster-pcm-restore-aba-force-round"); if (cluster_injection_should_skip("cluster-pcm-restore-aba-force-round")) - cluster_pcm_own_transition(buf, (uint8) PCM_STATE_N, 0, 0); + cluster_pcm_own_transition(buf, (uint8)PCM_STATE_N, 0, 0); buf_state = LockBufHdr(buf); @@ -11136,13 +10538,11 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn * the InvalidateBuffer window does not get a stale pre-drop state * restored over it. */ - if (BufferTagsEqual(&buf->tag, &tag) && - buf->pcm_state == (uint8) PCM_STATE_N && - cluster_pcm_own_gen_get(buf->buf_id) == staged_gen) + if (BufferTagsEqual(&buf->tag, &tag) && buf->pcm_state == (uint8)PCM_STATE_N + && cluster_pcm_own_gen_get(buf->buf_id) == staged_gen) buf->pcm_state = saved_pcm_state; - else if (BufferTagsEqual(&buf->tag, &tag) && - buf->pcm_state == (uint8) PCM_STATE_N && - cluster_pcm_own_gen_get(buf->buf_id) != staged_gen) + else if (BufferTagsEqual(&buf->tag, &tag) && buf->pcm_state == (uint8)PCM_STATE_N + && cluster_pcm_own_gen_get(buf->buf_id) != staged_gen) cluster_pcm_note_restore_aba_detected(); UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_PINNED; @@ -11241,8 +10641,7 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, result = CLUSTER_PCM_OWN_STALE; else if ((buf_state & BM_IO_ERROR) != 0) result = CLUSTER_PCM_OWN_CORRUPT; - else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) - { + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) { /* * PGRAC: a dirty N page here is legitimate, not corruption evidence: * relation extension (PageInit + MarkBufferDirty) and recovery redo @@ -11254,14 +10653,13 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, * and report BUSY: one flush converges the state and the image pump * retries against a clean page. */ - PinBuffer_Locked(buf); /* consumes the buffer header lock */ + PinBuffer_Locked(buf); /* consumes the buffer header lock */ LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED); FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); LWLockRelease(BufferDescriptorGetContentLock(buf)); UnpinBuffer(buf); return CLUSTER_PCM_OWN_BUSY; - } - else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + } else if ((buf_state & BM_IO_IN_PROGRESS) != 0) result = CLUSTER_PCM_OWN_BUSY; else { result = cluster_pcm_own_reservation_begin_exact(buf->buf_id, expected_n->generation, @@ -11279,8 +10677,7 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, reln = smgropen(BufTagGetRelFileLocator(&tag), InvalidBackendId); smgrread(reln, BufTagGetForkNum(&tag), tag.blockNum, scratch.data); if (!PageIsVerifiedExtended((Page)scratch.data, tag.blockNum, - PIV_LOG_WARNING | PIV_REPORT_STAT)) - { + PIV_LOG_WARNING | PIV_REPORT_STAT)) { result = CLUSTER_PCM_OWN_CORRUPT; } @@ -11294,15 +10691,13 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, result = CLUSTER_PCM_OWN_STALE; else if ((buf_state & BM_IO_ERROR) != 0) result = CLUSTER_PCM_OWN_CORRUPT; - else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) - { + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) { /* PGRAC: REVOKING already blocks data writes, so dirt appearing * between the flush above and this recheck can only be an * idempotent hint-bit write. Retry via BUSY; the next pass * flushes it and converges. */ result = CLUSTER_PCM_OWN_BUSY; - } - else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + } else if ((buf_state & BM_IO_IN_PROGRESS) != 0) result = CLUSTER_PCM_OWN_BUSY; else { memcpy((char *)BufHdrGetBlock(buf), scratch.data, BLCKSZ); @@ -11359,35 +10754,32 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, * the final byte boundary is serialized by the content lock instead. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_begin_s_revoke(BufferDesc *buf, - const ClusterPcmOwnSnapshot *expected_s, - ClusterPcmOwnSnapshot *out_revoking) +cluster_bufmgr_pcm_own_begin_s_revoke(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_s, + ClusterPcmOwnSnapshot *out_revoking) { ClusterPcmOwnResult live_result; ClusterPcmOwnResult result; - uint64 live_token; - uint64 reservation_token = 0; - uint32 flags; - uint32 buf_state; + uint64 live_token; + uint64 reservation_token = 0; + uint32 flags; + uint32 buf_state; if (buf == NULL || expected_s == NULL || out_revoking == NULL) return CLUSTER_PCM_OWN_INVALID; memset(out_revoking, 0, sizeof(*out_revoking)); - if (expected_s->pcm_state != (uint8) PCM_STATE_S || expected_s->flags != 0) + if (expected_s->pcm_state != (uint8)PCM_STATE_S || expected_s->flags != 0) return CLUSTER_PCM_OWN_INVALID; if (ClusterPcmOwnArray == NULL) return CLUSTER_PCM_OWN_NOT_READY; buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &expected_s->tag) - || (buf_state & BM_VALID) == 0 + if (!BufferTagsEqual(&buf->tag, &expected_s->tag) || (buf_state & BM_VALID) == 0 || cluster_pcm_own_gen_get(buf->buf_id) != expected_s->generation - || buf->pcm_state != (uint8) PCM_STATE_S) + || buf->pcm_state != (uint8)PCM_STATE_S) result = CLUSTER_PCM_OWN_STALE; else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) result = CLUSTER_PCM_OWN_CORRUPT; - else - { + else { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -11395,11 +10787,9 @@ cluster_bufmgr_pcm_own_begin_s_revoke(BufferDesc *buf, result = live_result; else if (live_token != expected_s->reservation_token) result = CLUSTER_PCM_OWN_STALE; - else - { + else { result = cluster_pcm_own_reservation_begin_exact( - buf->buf_id, expected_s->generation, PCM_OWN_FLAG_REVOKING, - &reservation_token); + buf->buf_id, expected_s->generation, PCM_OWN_FLAG_REVOKING, &reservation_token); if (result == CLUSTER_PCM_OWN_OK) cluster_pcm_own_snapshot_locked(buf, out_revoking); } @@ -11411,17 +10801,17 @@ cluster_bufmgr_pcm_own_begin_s_revoke(BufferDesc *buf, /* Abort only the matching S-source staging reservation. */ ClusterPcmOwnResult cluster_bufmgr_pcm_own_abort_s_revoke(BufferDesc *buf, - const ClusterPcmOwnSnapshot *expected_revoking) + const ClusterPcmOwnSnapshot *expected_revoking) { ClusterPcmOwnResult live_result; ClusterPcmOwnResult result; - uint64 live_token; - uint32 flags; - uint32 buf_state; + uint64 live_token; + uint32 flags; + uint32 buf_state; if (buf == NULL || expected_revoking == NULL) return CLUSTER_PCM_OWN_INVALID; - if (expected_revoking->pcm_state != (uint8) PCM_STATE_S + if (expected_revoking->pcm_state != (uint8)PCM_STATE_S || expected_revoking->flags != PCM_OWN_FLAG_REVOKING || expected_revoking->reservation_token == 0) return CLUSTER_PCM_OWN_INVALID; @@ -11429,15 +10819,13 @@ cluster_bufmgr_pcm_own_abort_s_revoke(BufferDesc *buf, return CLUSTER_PCM_OWN_NOT_READY; buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &expected_revoking->tag) - || (buf_state & BM_VALID) == 0 + if (!BufferTagsEqual(&buf->tag, &expected_revoking->tag) || (buf_state & BM_VALID) == 0 || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation - || buf->pcm_state != (uint8) PCM_STATE_S) + || buf->pcm_state != (uint8)PCM_STATE_S) result = CLUSTER_PCM_OWN_STALE; else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) result = CLUSTER_PCM_OWN_CORRUPT; - else - { + else { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -11451,8 +10839,8 @@ cluster_bufmgr_pcm_own_abort_s_revoke(BufferDesc *buf, result = CLUSTER_PCM_OWN_STALE; else result = cluster_pcm_own_reservation_abort_exact( - buf->buf_id, expected_revoking->generation, - expected_revoking->reservation_token, PCM_OWN_FLAG_REVOKING); + buf->buf_id, expected_revoking->generation, expected_revoking->reservation_token, + PCM_OWN_FLAG_REVOKING); } UnlockBufHdr(buf, buf_state); return result; @@ -11466,35 +10854,32 @@ cluster_bufmgr_pcm_own_abort_s_revoke(BufferDesc *buf, * abort, and retained finish; callers never inspect the ownership sidecar. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_begin_x_revoke(BufferDesc *buf, - const ClusterPcmOwnSnapshot *expected_x, - ClusterPcmOwnSnapshot *out_revoking) +cluster_bufmgr_pcm_own_begin_x_revoke(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_x, + ClusterPcmOwnSnapshot *out_revoking) { ClusterPcmOwnResult live_result; ClusterPcmOwnResult result; - uint64 live_token; - uint64 reservation_token = 0; - uint32 flags; - uint32 buf_state; + uint64 live_token; + uint64 reservation_token = 0; + uint32 flags; + uint32 buf_state; if (buf == NULL || expected_x == NULL || out_revoking == NULL) return CLUSTER_PCM_OWN_INVALID; memset(out_revoking, 0, sizeof(*out_revoking)); - if (expected_x->pcm_state != (uint8) PCM_STATE_X || expected_x->flags != 0) + if (expected_x->pcm_state != (uint8)PCM_STATE_X || expected_x->flags != 0) return CLUSTER_PCM_OWN_INVALID; if (ClusterPcmOwnArray == NULL) return CLUSTER_PCM_OWN_NOT_READY; buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &expected_x->tag) - || (buf_state & BM_VALID) == 0 + if (!BufferTagsEqual(&buf->tag, &expected_x->tag) || (buf_state & BM_VALID) == 0 || cluster_pcm_own_gen_get(buf->buf_id) != expected_x->generation - || buf->pcm_state != (uint8) PCM_STATE_X) + || buf->pcm_state != (uint8)PCM_STATE_X) result = CLUSTER_PCM_OWN_STALE; else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) result = CLUSTER_PCM_OWN_CORRUPT; - else - { + else { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -11502,11 +10887,9 @@ cluster_bufmgr_pcm_own_begin_x_revoke(BufferDesc *buf, result = live_result; else if (live_token != expected_x->reservation_token) result = CLUSTER_PCM_OWN_STALE; - else - { + else { result = cluster_pcm_own_reservation_begin_exact( - buf->buf_id, expected_x->generation, PCM_OWN_FLAG_REVOKING, - &reservation_token); + buf->buf_id, expected_x->generation, PCM_OWN_FLAG_REVOKING, &reservation_token); if (result == CLUSTER_PCM_OWN_OK) cluster_pcm_own_snapshot_locked(buf, out_revoking); } @@ -11520,17 +10903,17 @@ cluster_bufmgr_pcm_own_begin_x_revoke(BufferDesc *buf, */ ClusterPcmOwnResult cluster_bufmgr_pcm_own_abort_x_revoke(BufferDesc *buf, - const ClusterPcmOwnSnapshot *expected_revoking) + const ClusterPcmOwnSnapshot *expected_revoking) { ClusterPcmOwnResult live_result; ClusterPcmOwnResult result; - uint64 live_token; - uint32 flags; - uint32 buf_state; + uint64 live_token; + uint32 flags; + uint32 buf_state; if (buf == NULL || expected_revoking == NULL) return CLUSTER_PCM_OWN_INVALID; - if (expected_revoking->pcm_state != (uint8) PCM_STATE_X + if (expected_revoking->pcm_state != (uint8)PCM_STATE_X || expected_revoking->flags != PCM_OWN_FLAG_REVOKING || expected_revoking->reservation_token == 0) return CLUSTER_PCM_OWN_INVALID; @@ -11538,15 +10921,13 @@ cluster_bufmgr_pcm_own_abort_x_revoke(BufferDesc *buf, return CLUSTER_PCM_OWN_NOT_READY; buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &expected_revoking->tag) - || (buf_state & BM_VALID) == 0 + if (!BufferTagsEqual(&buf->tag, &expected_revoking->tag) || (buf_state & BM_VALID) == 0 || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation - || buf->pcm_state != (uint8) PCM_STATE_X) + || buf->pcm_state != (uint8)PCM_STATE_X) result = CLUSTER_PCM_OWN_STALE; else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) result = CLUSTER_PCM_OWN_CORRUPT; - else - { + else { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -11560,8 +10941,8 @@ cluster_bufmgr_pcm_own_abort_x_revoke(BufferDesc *buf, result = CLUSTER_PCM_OWN_STALE; else result = cluster_pcm_own_reservation_abort_exact( - buf->buf_id, expected_revoking->generation, - expected_revoking->reservation_token, PCM_OWN_FLAG_REVOKING); + buf->buf_id, expected_revoking->generation, expected_revoking->reservation_token, + PCM_OWN_FLAG_REVOKING); } UnlockBufHdr(buf, buf_state); return result; @@ -11662,30 +11043,30 @@ cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(BufferDesc *buf, * REVOKING token prevents reuse until DRAIN. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_finish_revoke_retain( - BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, - XLogRecPtr expected_lsn, ClusterPcmOwnSnapshot *out_retained) +cluster_bufmgr_pcm_own_finish_revoke_retain(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected_revoking, + XLogRecPtr expected_lsn, + ClusterPcmOwnSnapshot *out_retained) { ClusterPcmOwnResult live_result; ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; - BufferTag tag; - LWLock *content_lock; - uint64 committed_generation; - uint64 live_token; - uint32 flags; - uint32 buf_state; - bool source_is_s; - bool source_is_x; + BufferTag tag; + LWLock *content_lock; + uint64 committed_generation; + uint64 live_token; + uint32 flags; + uint32 buf_state; + bool source_is_s; + bool source_is_x; ClusterPcmXRevokeFinishMode finish_mode; if (out_retained != NULL) memset(out_retained, 0, sizeof(*out_retained)); if (buf == NULL || expected_revoking == NULL || out_retained == NULL) return CLUSTER_PCM_OWN_INVALID; - source_is_s = expected_revoking->pcm_state == (uint8) PCM_STATE_S; - source_is_x = expected_revoking->pcm_state == (uint8) PCM_STATE_X; - if ((!source_is_s && !source_is_x) - || expected_revoking->flags != PCM_OWN_FLAG_REVOKING + source_is_s = expected_revoking->pcm_state == (uint8)PCM_STATE_S; + source_is_x = expected_revoking->pcm_state == (uint8)PCM_STATE_X; + if ((!source_is_s && !source_is_x) || expected_revoking->flags != PCM_OWN_FLAG_REVOKING || expected_revoking->reservation_token == 0) return CLUSTER_PCM_OWN_INVALID; if (ClusterPcmOwnArray == NULL) @@ -11703,8 +11084,7 @@ cluster_bufmgr_pcm_own_finish_revoke_retain( LWLockAcquire(content_lock, LW_EXCLUSIVE); buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) - || (buf_state & BM_VALID) == 0 + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0 || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation || buf->pcm_state != expected_revoking->pcm_state) result = CLUSTER_PCM_OWN_STALE; @@ -11712,8 +11092,7 @@ cluster_bufmgr_pcm_own_finish_revoke_retain( result = CLUSTER_PCM_OWN_CORRUPT; else if ((buf_state & BM_IO_IN_PROGRESS) != 0) result = CLUSTER_PCM_OWN_BUSY; - else - { + else { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -11725,21 +11104,18 @@ cluster_bufmgr_pcm_own_finish_revoke_retain( result = CLUSTER_PCM_OWN_BUSY; else if (live_token != expected_revoking->reservation_token) result = CLUSTER_PCM_OWN_STALE; - else if (PageGetLSN((Page) BufHdrGetBlock(buf)) != expected_lsn) + else if (PageGetLSN((Page)BufHdrGetBlock(buf)) != expected_lsn) result = CLUSTER_PCM_OWN_STALE; } - if (result == CLUSTER_PCM_OWN_OK) - { + if (result == CLUSTER_PCM_OWN_OK) { result = cluster_pcm_own_revoke_retain_commit_exact( - buf->buf_id, expected_revoking->generation, - expected_revoking->reservation_token, &committed_generation); - if (result == CLUSTER_PCM_OWN_OK) - { - buf->pcm_state = (uint8) PCM_STATE_N; - buf->buffer_type = (uint8) BUF_TYPE_PI; - buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED | - BM_CHECKPOINT_NEEDED | BM_IO_ERROR); + buf->buf_id, expected_revoking->generation, expected_revoking->reservation_token, + &committed_generation); + if (result == CLUSTER_PCM_OWN_OK) { + buf->pcm_state = (uint8)PCM_STATE_N; + buf->buffer_type = (uint8)BUF_TYPE_PI; + buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR); cluster_pcm_own_snapshot_locked(buf, out_retained); } } @@ -11753,22 +11129,21 @@ cluster_bufmgr_pcm_own_finish_revoke_retain( * generation+1 and current token bind a DRAIN to one image round; a delayed * DRAIN cannot clear REVOKING on a later transfer. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, - uint64 source_generation) +cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, uint64 source_generation) { ClusterPcmOwnResult live_result; ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; BufferDesc *buf; - BufferTag lookup_tag; - LWLock *content_lock; - LWLock *partition_lock; - uint64 committed_generation; - uint64 live_token; - uint64 retained_token = 0; - uint32 hashcode; - uint32 flags; - uint32 buf_state; - int buf_id; + BufferTag lookup_tag; + LWLock *content_lock; + LWLock *partition_lock; + uint64 committed_generation; + uint64 live_token; + uint64 retained_token = 0; + uint32 hashcode; + uint32 flags; + uint32 buf_state; + int buf_id; if (tag == NULL || source_generation == UINT64_MAX) return CLUSTER_PCM_OWN_INVALID; @@ -11780,8 +11155,7 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, partition_lock = BufMappingPartitionLock(hashcode); LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&lookup_tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); return CLUSTER_PCM_OWN_STALE; } @@ -11795,14 +11169,11 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, * ordinary content-lock callers). */ buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, tag) - || (buf_state & BM_VALID) == 0 - || buf->buffer_type != (uint8) BUF_TYPE_PI - || buf->pcm_state != (uint8) PCM_STATE_N + if (!BufferTagsEqual(&buf->tag, tag) || (buf_state & BM_VALID) == 0 + || buf->buffer_type != (uint8)BUF_TYPE_PI || buf->pcm_state != (uint8)PCM_STATE_N || cluster_pcm_own_gen_get(buf->buf_id) != committed_generation) result = CLUSTER_PCM_OWN_STALE; - else - { + else { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -11824,17 +11195,13 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, LWLockAcquire(content_lock, LW_EXCLUSIVE); buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, tag) - || (buf_state & BM_VALID) == 0 - || buf->buffer_type != (uint8) BUF_TYPE_PI - || buf->pcm_state != (uint8) PCM_STATE_N + if (!BufferTagsEqual(&buf->tag, tag) || (buf_state & BM_VALID) == 0 + || buf->buffer_type != (uint8)BUF_TYPE_PI || buf->pcm_state != (uint8)PCM_STATE_N || cluster_pcm_own_gen_get(buf->buf_id) != committed_generation) result = CLUSTER_PCM_OWN_STALE; - else if ((buf_state - & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR)) != 0) + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR)) != 0) result = CLUSTER_PCM_OWN_CORRUPT; - else - { + else { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -11847,8 +11214,8 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, else if (live_token != retained_token) result = CLUSTER_PCM_OWN_STALE; else { - result = cluster_pcm_own_revoke_retain_release_exact( - buf->buf_id, committed_generation, retained_token); + result = cluster_pcm_own_revoke_retain_release_exact(buf->buf_id, committed_generation, + retained_token); if (result == CLUSTER_PCM_OWN_OK) { /* * The exact token is the last stale-write fence. Release it @@ -11943,25 +11310,23 @@ cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, uint64 source_ bool cluster_bufmgr_block_is_pi(BufferTag tag) { - uint32 hash = BufTableHashCode(&tag); - LWLock *partition_lock = BufMappingPartitionLock(hash); - int buf_id; + uint32 hash = BufTableHashCode(&tag); + LWLock *partition_lock = BufMappingPartitionLock(hash); + int buf_id; BufferDesc *buf; - uint32 buf_state; - bool is_pi; + uint32 buf_state; + bool is_pi; LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hash); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); return false; } buf = GetBufferDescriptor(buf_id); buf_state = LockBufHdr(buf); - is_pi = BufferTagsEqual(&buf->tag, &tag) - && buf->buffer_type == (uint8) BUF_TYPE_PI - && (buf_state & BM_VALID) == 0; + is_pi = BufferTagsEqual(&buf->tag, &tag) && buf->buffer_type == (uint8)BUF_TYPE_PI + && (buf_state & BM_VALID) == 0; UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); @@ -11971,11 +11336,11 @@ cluster_bufmgr_block_is_pi(BufferTag tag) bool cluster_bufmgr_discard_pi_block(BufferTag tag) { - uint32 hash = BufTableHashCode(&tag); - LWLock *partition_lock = BufMappingPartitionLock(hash); - int buf_id; + uint32 hash = BufTableHashCode(&tag); + LWLock *partition_lock = BufMappingPartitionLock(hash); + int buf_id; BufferDesc *buf; - uint32 buf_state; + uint32 buf_state; LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hash); @@ -11985,11 +11350,8 @@ cluster_bufmgr_discard_pi_block(BufferTag tag) buf = GetBufferDescriptor(buf_id); buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) - || buf->buffer_type != (uint8) BUF_TYPE_PI - || (buf_state & BM_VALID) != 0 - || BUF_STATE_GET_REFCOUNT(buf_state) != 0) - { + if (!BufferTagsEqual(&buf->tag, &tag) || buf->buffer_type != (uint8)BUF_TYPE_PI + || (buf_state & BM_VALID) != 0 || BUF_STATE_GET_REFCOUNT(buf_state) != 0) { UnlockBufHdr(buf, buf_state); return false; } @@ -11998,14 +11360,14 @@ cluster_bufmgr_discard_pi_block(BufferTag tag) * A PI carries no residency claim (D-h1 set pcm_state N at conversion), * but clear it again under the lock so InvalidateBuffer's eviction hook * can never see a stale mode and emit a release wire from LMON. */ - buf->pcm_state = (uint8) PCM_STATE_N; + buf->pcm_state = (uint8)PCM_STATE_N; /* PGRAC: spec-6.12h D-h3a — hygiene: drop the shadow stamp with the PI. * Correctness never depends on this clear (the D-h3 consumer * re-validates the PI shape + tag under this same header lock, and * InvalidateBuffer below breaks the shape), but a directive-discarded * slot should not linger as a plausible-looking stamp. */ cluster_pi_shadow_clear(buf->buf_id); - InvalidateBuffer(buf); /* releases the header spinlock */ + InvalidateBuffer(buf); /* releases the header spinlock */ return true; } @@ -12034,57 +11396,50 @@ cluster_bufmgr_discard_pi_block(BufferTag tag) bool cluster_bufmgr_snapshot_pi_block(BufferTag tag, char *dst, SCN *out_ship_scn) { - uint32 hash = BufTableHashCode(&tag); - LWLock *partition_lock = BufMappingPartitionLock(hash); - int buf_id; + uint32 hash = BufTableHashCode(&tag); + LWLock *partition_lock = BufMappingPartitionLock(hash); + int buf_id; BufferDesc *buf; - uint32 buf_state; - SCN ship_scn; - bool intact; + uint32 buf_state; + SCN ship_scn; + bool intact; *out_ship_scn = InvalidScn; LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hash); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); return false; } buf = GetBufferDescriptor(buf_id); buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) - || buf->buffer_type != (uint8) BUF_TYPE_PI - || (buf_state & BM_VALID) != 0 - || (buf_state & BM_TAG_VALID) == 0 - || (buf_state & BM_IO_IN_PROGRESS) != 0) - { + if (!BufferTagsEqual(&buf->tag, &tag) || buf->buffer_type != (uint8)BUF_TYPE_PI + || (buf_state & BM_VALID) != 0 || (buf_state & BM_TAG_VALID) == 0 + || (buf_state & BM_IO_IN_PROGRESS) != 0) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; } ship_scn = cluster_pi_shadow_read(buf->buf_id); - if (!SCN_VALID(ship_scn)) - { + if (!SCN_VALID(ship_scn)) { /* Unstamped PI (clock unarmed at conversion): the recovery boundary * is unprovable, so the PI is useless as a base (gate UNUSABLE). */ UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; } - cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); /* raw-pin + unlock header */ + cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); /* raw-pin + unlock header */ LWLockRelease(partition_lock); - memcpy(dst, (char *) BufHdrGetBlock(buf), BLCKSZ); + memcpy(dst, (char *)BufHdrGetBlock(buf), BLCKSZ); buf_state = LockBufHdr(buf); /* SCN_CMP_OK: identity recheck of the same slot (same-stamp equality), * not a Lamport-order comparison. */ - intact = BufferTagsEqual(&buf->tag, &tag) - && buf->buffer_type == (uint8) BUF_TYPE_PI - && (buf_state & BM_VALID) == 0 - && cluster_pi_shadow_read(buf->buf_id) == ship_scn; + intact = BufferTagsEqual(&buf->tag, &tag) && buf->buffer_type == (uint8)BUF_TYPE_PI + && (buf_state & BM_VALID) == 0 && cluster_pi_shadow_read(buf->buf_id) == ship_scn; UnlockBufHdr(buf, buf_state); cluster_bufmgr_unpin_for_gcs(buf); @@ -12115,7 +11470,7 @@ cluster_bufmgr_flush_seq_page_to_storage(Buffer buffer) { Assert(BufferIsValid(buffer)); if (BufferIsLocal(buffer)) - return; /* temp/local buffers are never CF-shared */ + return; /* temp/local buffers are never CF-shared */ FlushOneBuffer(buffer); } @@ -12167,8 +11522,8 @@ cluster_bufmgr_flush_seq_page_to_storage(Buffer buffer) uint32 cluster_bufmgr_flush_and_release_x_for_leave(void) { - uint32 flushed = 0; - int i; + uint32 flushed = 0; + int i; /* CL-I9: must run with a real ResourceOwner (backend/checkpointer). */ Assert(CurrentResourceOwner != NULL); @@ -12179,22 +11534,20 @@ cluster_bufmgr_flush_and_release_x_for_leave(void) */ ResourceOwnerEnlargeBuffers(CurrentResourceOwner); - for (i = 0; i < NBuffers; i++) - { + for (i = 0; i < NBuffers; i++) { BufferDesc *bufHdr = GetBufferDescriptor(i); - uint32 buf_state; + uint32 buf_state; ReservePrivateRefCountEntry(); buf_state = LockBufHdr(bufHdr); - if ((buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY) && - (PcmState) bufHdr->pcm_state == PCM_STATE_X) - { + if ((buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY) + && (PcmState)bufHdr->pcm_state == PCM_STATE_X) { ClusterPcmOwnResult bump_result; - uint32 observed_flags = 0; - uint64 observed_generation = 0; + uint32 observed_flags = 0; + uint64 observed_generation = 0; - PinBuffer_Locked(bufHdr); /* releases the header spinlock */ + PinBuffer_Locked(bufHdr); /* releases the header spinlock */ LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED); FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); @@ -12207,22 +11560,20 @@ cluster_bufmgr_flush_and_release_x_for_leave(void) buf_state = LockBufHdr(bufHdr); /* PGRAC ownership-gen: bump under the held spinlock (X -> N release * after the block is durable on shared storage). */ - bump_result = cluster_pcm_own_bump_locked(bufHdr, 0, 0, - &observed_generation, &observed_flags); - if (bump_result != CLUSTER_PCM_OWN_OK) - { + bump_result + = cluster_pcm_own_bump_locked(bufHdr, 0, 0, &observed_generation, &observed_flags); + if (bump_result != CLUSTER_PCM_OWN_OK) { UnlockBufHdr(bufHdr, buf_state); UnpinBuffer(bufHdr); cluster_pcm_own_report_bump_failure(bufHdr, bump_result, observed_generation, - observed_flags, "clean-leave X release"); + observed_flags, "clean-leave X release"); } - bufHdr->pcm_state = (uint8) PCM_STATE_N; + bufHdr->pcm_state = (uint8)PCM_STATE_N; UnlockBufHdr(bufHdr, buf_state); UnpinBuffer(bufHdr); flushed++; - } - else + } else UnlockBufHdr(bufHdr, buf_state); } @@ -12264,19 +11615,18 @@ bool cluster_bufmgr_block_write_permitted(Buffer buffer) { BufferDesc *buf; - PcmState state; - uint32 buf_state; - uint32 own_flags; + PcmState state; + uint32 buf_state; + uint32 own_flags; if (BufferIsLocal(buffer)) - return true; /* temp / local buffers are never CF-shared */ + return true; /* temp / local buffers are never CF-shared */ buf = GetBufferDescriptor(buffer - 1); buf_state = LockBufHdr(buf); - state = (PcmState) buf->pcm_state; + state = (PcmState)buf->pcm_state; own_flags = cluster_pcm_own_flags_get(buf->buf_id); if (cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state) - || (own_flags & PCM_OWN_FLAG_REVOKING) != 0) - { + || (own_flags & PCM_OWN_FLAG_REVOKING) != 0) { UnlockBufHdr(buf, buf_state); return false; } @@ -12284,8 +11634,8 @@ cluster_bufmgr_block_write_permitted(Buffer buffer) if (state == PCM_STATE_READ_IMAGE) return false; if (cluster_read_scache && state == PCM_STATE_S) - return false; /* spec-6.12a: S is a revoked-write state */ + return false; /* spec-6.12a: S is a revoked-write state */ return true; } -#endif /* USE_PGRAC_CLUSTER */ +#endif /* USE_PGRAC_CLUSTER */ diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index 08ceb43971..53f3a8ac24 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -3672,6 +3672,9 @@ extern bool cluster_gcs_pcm_x_stage_frame(uint8 msg_type, int32 dest_node_id, co extern PcmXQueueResult cluster_gcs_pcm_x_acquire_writer(BufferDesc *buf, PcmXLocalWriterClaim *claim_out, bool *claim_handed_off); +/* Diagnostic: source line of this backend's most recent non-OK + * acquire-writer exit (0 when it never failed). */ +extern int cluster_gcs_pcm_x_requester_last_fail_line(void); extern PcmXQueueResult cluster_gcs_pcm_x_writer_claim_release_and_wake_exact(const PcmXLocalWriterClaim *claim); /* Error/owner-exit cleanup adapter: never propagates ERROR. A caught ERROR diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h index 6d853ab688..c2fbd809eb 100644 --- a/src/include/cluster/cluster_pcm_x_convert.h +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -1652,6 +1652,9 @@ extern PcmXQueueResult cluster_pcm_x_local_probe_freeze_snapshot_exact( * ForEachLWLockHeldByMe callback, then rejects any tag whose local PROBE * barrier is closed. BUSY/CORRUPT are closed decisions, never "inactive". */ extern PcmXQueueResult cluster_pcm_x_nested_wait_guard_before_block(void); +/* Diagnostic: the held content-lock tag and result of this backend's most + * recent nested-guard refusal; false when the guard never refused. */ +extern bool cluster_pcm_x_nested_wait_guard_last_block(BufferTag *tag_out, int *result_out); extern PcmXQueueResult cluster_pcm_x_local_holder_snapshot_revalidate(const BufferTag *tag, const PcmXLocalHolderSnapshot *snapshot); diff --git a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl index f912f5a7d5..be688942b4 100644 --- a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl +++ b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl @@ -429,7 +429,7 @@ sub write_file { my @samples; - for my $offset (8, 12) + for my $offset (1, 3, 8) { my $probe_at = $quad->node0->safe_psql('postgres', "SELECT GREATEST(0.0, EXTRACT(EPOCH FROM " @@ -483,20 +483,24 @@ sub write_file } } - # During a pure stall the t+8 -> t+12 per-node counter movement names the - # spinning arm: whatever still ticks is the live retry loop, whatever is - # frozen is wedged. - for my $i (0 .. 3) + # Early samples catch the terminal handoff mid-flight; the late pair + # separates the live retry loop (still ticking) from the wedged stages + # (frozen). + for my $pair ([1, 3], [3, 8]) { - next unless $samples[8][$i] && $samples[12][$i]; - my @moved; - for my $key (@pcm_x_pcm_keys) + my ($from, $to) = @{$pair}; + for my $i (0 .. 3) { - my $d = $samples[12][$i]{$key} - $samples[8][$i]{$key}; - push @moved, "$key:+$d" if $d != 0; + next unless $samples[$from][$i] && $samples[$to][$i]; + my @moved; + for my $key (@pcm_x_pcm_keys) + { + my $d = $samples[$to][$i]{$key} - $samples[$from][$i]{$key}; + push @moved, "$key:+$d" if $d != 0; + } + diag("L3 mid-leg node$i t+$from..t+$to moved: " + . (@moved ? join(' ', @moved) : '(all frozen)')); } - diag("L3 mid-leg node$i t+8..t+12 moved: " - . (@moved ? join(' ', @moved) : '(all frozen)')); } } From 63f1cb2aca65011ea6d43a49960c4fa36963961f Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 05:46:52 -0700 Subject: [PATCH 11/56] style(cluster): undo accidental clang-format of PG core bufmgr.c bufmgr.c keeps upstream pgindent formatting and is outside the cluster clang-format scope; restore it wholesale and re-apply only the writer failure-report evidence hunk. --- src/backend/storage/buffer/bufmgr.c | 3646 ++++++++++++++++----------- 1 file changed, 2154 insertions(+), 1492 deletions(-) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 9b18a8dc2d..3a06cd8541 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -34,9 +34,9 @@ #include #include "access/tableam.h" -#include "access/xlog.h" /* PGRAC (spec-6.14 D8): GetXLogInsertRecPtr */ +#include "access/xlog.h" /* PGRAC (spec-6.14 D8): GetXLogInsertRecPtr */ #ifdef USE_PGRAC_CLUSTER -#include "access/transam.h" /* FirstNormalObjectId */ +#include "access/transam.h" /* FirstNormalObjectId */ #endif #include "access/xloginsert.h" #include "access/xlogutils.h" @@ -54,26 +54,26 @@ #include "postmaster/bgwriter.h" #include "storage/buf_internals.h" #ifdef USE_PGRAC_CLUSTER -#include "cluster/cluster_catalog_stats.h" /* spec-6.14 D10b — catalog buf hit/miss */ -#include "cluster/cluster_mode.h" /* PGRAC (spec-6.14 D8): storage-mode gate */ +#include "cluster/cluster_catalog_stats.h" /* spec-6.14 D10b — catalog buf hit/miss */ +#include "cluster/cluster_mode.h" /* PGRAC (spec-6.14 D8): storage-mode gate */ #include "cluster/cluster_pcm_direct_init.h" #include "cluster/cluster_gcs_block.h" #include "cluster/cluster_pcm_lock.h" -#include "cluster/cluster_grd.h" /* existing block-path fail-closed counter */ -#include "cluster/cluster_guc.h" /* spec-4.7a D2 — cluster_gcs_block_local_cache */ +#include "cluster/cluster_grd.h" /* existing block-path fail-closed counter */ +#include "cluster/cluster_guc.h" /* spec-4.7a D2 — cluster_gcs_block_local_cache */ #include "cluster/cluster_block_recovery.h" /* spec-4.10 D1 — online block recovery */ #include "cluster/cluster_conf.h" /* spec-5.7 HW — cluster_conf_node_count */ #include "cluster/cluster_hw.h" /* spec-5.7 HW — relation-extend authority */ #include "cluster/cluster_xnode_lever.h" /* spec-6.12h — PI keep counter */ -#include "cluster/cluster_pi_shadow.h" /* spec-6.12h D-h3a — PI ship-SCN stamp */ -#include "cluster/cluster_hw_lease.h" /* spec-6.12d — per-node HW space leases */ -#include "cluster/cluster_extend_gate.h" /* spec-5.7 §3.1d — liveness engage gate */ +#include "cluster/cluster_pi_shadow.h" /* spec-6.12h D-h3a — PI ship-SCN stamp */ +#include "cluster/cluster_hw_lease.h" /* spec-6.12d — per-node HW space leases */ +#include "cluster/cluster_extend_gate.h" /* spec-5.7 §3.1d — liveness engage gate */ #include "cluster/cluster_epoch.h" -#include "cluster/cluster_sf_dep.h" /* spec-6.2 Smart Fusion DBWR brake */ -#include "cluster/cluster_xnode_profile.h" /* spec-5.59 D3/D4 — read probe + relkind hint */ -#include "cluster/cluster_itl.h" /* spec-6.12a — quiescent check for X->S downgrade */ -#include "cluster/cluster_inject.h" /* GCS-race round-4c P1 — yield-notify-drop point */ -#include "cluster/cluster_pcm_own.h" /* ownership-generation wave — per-buffer gen + flags */ +#include "cluster/cluster_sf_dep.h" /* spec-6.2 Smart Fusion DBWR brake */ +#include "cluster/cluster_xnode_profile.h" /* spec-5.59 D3/D4 — read probe + relkind hint */ +#include "cluster/cluster_itl.h" /* spec-6.12a — quiescent check for X->S downgrade */ +#include "cluster/cluster_inject.h" /* GCS-race round-4c P1 — yield-notify-drop point */ +#include "cluster/cluster_pcm_own.h" /* ownership-generation wave — per-buffer gen + flags */ #include "cluster/cluster_pcm_x_bufmgr.h" /* spec-2.36a C1 opaque reservation API */ #include "cluster/cluster_pcm_x_convert.h" @@ -103,17 +103,18 @@ extern bool ignore_checksum_failure; /* Note: these two macros only work on shared buffers, not local ones! */ -#define BufHdrGetBlock(bufHdr) ((Block)(BufferBlocks + ((Size)(bufHdr)->buf_id) * BLCKSZ)) -#define BufferGetLSN(bufHdr) (PageGetLSN(BufHdrGetBlock(bufHdr))) +#define BufHdrGetBlock(bufHdr) ((Block) (BufferBlocks + ((Size) (bufHdr)->buf_id) * BLCKSZ)) +#define BufferGetLSN(bufHdr) (PageGetLSN(BufHdrGetBlock(bufHdr))) /* Note: this macro only works on local buffers, not shared ones! */ -#define LocalBufHdrGetBlock(bufHdr) LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)] +#define LocalBufHdrGetBlock(bufHdr) \ + LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)] /* Bits in SyncOneBuffer's return value */ -#define BUF_WRITTEN 0x01 -#define BUF_REUSABLE 0x02 +#define BUF_WRITTEN 0x01 +#define BUF_REUSABLE 0x02 -#define RELS_BSEARCH_THRESHOLD 20 +#define RELS_BSEARCH_THRESHOLD 20 #ifdef USE_PGRAC_CLUSTER /* @@ -134,7 +135,7 @@ extern bool ignore_checksum_failure; static inline bool cluster_bufmgr_reln_pcm_tracked(RelFileNumber relnum) { - return cluster_shared_catalog || relnum >= (RelFileNumber)FirstNormalObjectId; + return cluster_shared_catalog || relnum >= (RelFileNumber) FirstNormalObjectId; } static inline bool @@ -172,33 +173,33 @@ cluster_pcm_own_bump_failure(BufferDesc *buf, uint64 generation, uint32 *out_fla } static void -cluster_pcm_own_report_bump_failure(BufferDesc *buf, ClusterPcmOwnResult result, uint64 generation, - uint32 flags, const char *context) +cluster_pcm_own_report_bump_failure(BufferDesc *buf, ClusterPcmOwnResult result, + uint64 generation, uint32 flags, const char *context) { if (result == CLUSTER_PCM_OWN_BUSY) ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), errmsg("cluster PCM ownership transition conflicts with an active reservation"), - errdetail("context=%s buffer=%d generation=%llu flags=0x%x", context, buf->buf_id, - (unsigned long long)generation, flags))); + errdetail("context=%s buffer=%d generation=%llu flags=0x%x", context, + buf->buf_id, (unsigned long long) generation, flags))); if (result == CLUSTER_PCM_OWN_EXHAUSTED) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("cluster PCM ownership generation exhausted for buffer %d", buf->buf_id), errdetail("context=%s generation=%llu flags=0x%x", context, - (unsigned long long)generation, flags))); + (unsigned long long) generation, flags))); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("cluster PCM ownership generation bump failed for buffer %d", buf->buf_id), errdetail("context=%s generation=%llu flags=0x%x result=%d", context, - (unsigned long long)generation, flags, (int)result))); + (unsigned long long) generation, flags, (int) result))); } -static inline ClusterPcmOwnResult cluster_pcm_own_bump_locked(BufferDesc *buf, uint32 set_flags, - uint32 clear_flags, - uint64 *out_generation, - uint32 *out_flags); +static inline ClusterPcmOwnResult cluster_pcm_own_bump_locked(BufferDesc *buf, + uint32 set_flags, uint32 clear_flags, + uint64 *out_generation, + uint32 *out_flags); /* * PGRAC ownership-generation wave — coherent @@ -226,12 +227,12 @@ cluster_pcm_own_transition(BufferDesc *buf, uint8 new_pcm_state, uint32 set_flag uint64 new_generation; buf_state = LockBufHdr(buf); - bump_result = cluster_pcm_own_bump_locked(buf, set_flags, clear_flags, &new_generation, - &observed_flags); + bump_result = cluster_pcm_own_bump_locked(buf, set_flags, clear_flags, + &new_generation, &observed_flags); if (bump_result != CLUSTER_PCM_OWN_OK) { UnlockBufHdr(buf, buf_state); - cluster_pcm_own_report_bump_failure(buf, bump_result, new_generation, observed_flags, - "ownership transition"); + cluster_pcm_own_report_bump_failure(buf, bump_result, new_generation, + observed_flags, "ownership transition"); } buf->pcm_state = new_pcm_state; UnlockBufHdr(buf, buf_state); @@ -284,7 +285,8 @@ cluster_pcm_own_snapshot_locked(BufferDesc *buf, ClusterPcmOwnSnapshot *out) static inline bool cluster_bufmgr_pcm_x_retained_image_locked(BufferDesc *buf, uint32 buf_state) { - return buf != NULL && buf->buffer_type == (uint8)BUF_TYPE_PI && (buf_state & BM_VALID) != 0; + return buf != NULL && buf->buffer_type == (uint8) BUF_TYPE_PI + && (buf_state & BM_VALID) != 0; } /* A resident mapping is GCS-shippable only while its node owns matching PCM @@ -294,12 +296,13 @@ static inline bool cluster_bufmgr_pcm_current_image_locked(BufferDesc *buf, uint32 buf_state) { return buf != NULL - && cluster_pcm_x_current_image_shape(buf->pcm_state, buf->buffer_type, - (buf_state & BM_VALID) != 0); + && cluster_pcm_x_current_image_shape(buf->pcm_state, buf->buffer_type, + (buf_state & BM_VALID) != 0); } static inline bool -cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(BufferDesc *buf, uint32 buf_state) +cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(BufferDesc *buf, + uint32 buf_state) { uint32 flags; @@ -312,13 +315,14 @@ cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(BufferDesc *buf, uint32 * material. Its monotonic token may legitimately still be zero before * the first reservation, so flags -- not token value -- are the active * lifecycle authority. Every live or malformed shape remains blocked. */ - return buf->pcm_state != (uint8)PCM_STATE_N - || (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR)) != 0 - || flags != 0; + return buf->pcm_state != (uint8) PCM_STATE_N + || (buf_state + & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR)) != 0 + || flags != 0; } -static inline bool cluster_pcm_own_snapshot_matches_locked(BufferDesc *buf, - const ClusterPcmOwnSnapshot *expected); +static inline bool cluster_pcm_own_snapshot_matches_locked( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected); static inline void cluster_pcm_own_eviction_capture_locked(BufferDesc *buf, ClusterPcmOwnEvictionCapture *out) @@ -327,7 +331,8 @@ cluster_pcm_own_eviction_capture_locked(BufferDesc *buf, ClusterPcmOwnEvictionCa } static ClusterPcmOwnResult -cluster_pcm_own_eviction_commit_locked(BufferDesc *buf, const ClusterPcmOwnEvictionCapture *capture, +cluster_pcm_own_eviction_commit_locked(BufferDesc *buf, + const ClusterPcmOwnEvictionCapture *capture, uint64 *out_generation, uint32 *out_flags) { ClusterPcmOwnResult result; @@ -335,7 +340,8 @@ cluster_pcm_own_eviction_commit_locked(BufferDesc *buf, const ClusterPcmOwnEvict if (!cluster_pcm_own_snapshot_matches_locked(buf, capture)) return CLUSTER_PCM_OWN_STALE; if (!cluster_pcm_own_eviction_reuse_allowed(capture)) { - result = cluster_pcm_own_classify_live_flags(capture->flags, capture->reservation_token); + result = cluster_pcm_own_classify_live_flags(capture->flags, + capture->reservation_token); return result != CLUSTER_PCM_OWN_OK ? result : CLUSTER_PCM_OWN_EXHAUSTED; } @@ -360,7 +366,7 @@ cluster_pcm_own_snapshot_matches_locked(BufferDesc *buf, const ClusterPcmOwnSnap ClusterPcmOwnResult cluster_bufmgr_pcm_own_snapshot(BufferDesc *buf, ClusterPcmOwnSnapshot *out_snapshot) { - uint32 buf_state; + uint32 buf_state; if (buf == NULL || out_snapshot == NULL) return CLUSTER_PCM_OWN_INVALID; @@ -376,17 +382,17 @@ cluster_bufmgr_pcm_own_snapshot(BufferDesc *buf, ClusterPcmOwnSnapshot *out_snap bool cluster_bufmgr_pcm_x_content_write_permitted(BufferDesc *buf) { - bool permitted; - uint32 buf_state; - uint32 flags; + bool permitted; + uint32 buf_state; + uint32 flags; if (buf == NULL) return false; buf_state = LockBufHdr(buf); flags = cluster_pcm_own_flags_get(buf->buf_id); permitted = (flags & PCM_OWN_FLAG_REVOKING) == 0 - && (!cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state) - || flags == PCM_OWN_FLAG_GRANT_PENDING); + && (!cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state) + || flags == PCM_OWN_FLAG_GRANT_PENDING); UnlockBufHdr(buf, buf_state); return permitted; } @@ -396,11 +402,11 @@ cluster_bufmgr_pcm_own_snapshot_by_tag(const BufferTag *tag, int *out_buffer_id, ClusterPcmOwnSnapshot *out_snapshot) { BufferDesc *buf; - BufferTag lookup_tag; - LWLock *partition_lock; - uint32 hashcode; - uint32 buf_state; - int buf_id; + BufferTag lookup_tag; + LWLock *partition_lock; + uint32 hashcode; + uint32 buf_state; + int buf_id; ClusterPcmOwnResult result; if (out_buffer_id != NULL) @@ -417,7 +423,8 @@ cluster_bufmgr_pcm_own_snapshot_by_tag(const BufferTag *tag, int *out_buffer_id, partition_lock = BufMappingPartitionLock(hashcode); LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&lookup_tag, hashcode); - if (buf_id < 0) { + if (buf_id < 0) + { LWLockRelease(partition_lock); return CLUSTER_PCM_OWN_STALE; } @@ -425,7 +432,8 @@ cluster_bufmgr_pcm_own_snapshot_by_tag(const BufferTag *tag, int *out_buffer_id, buf_state = LockBufHdr(buf); if (!BufferTagsEqual(&buf->tag, tag) || (buf_state & BM_VALID) == 0) result = CLUSTER_PCM_OWN_STALE; - else { + else + { cluster_pcm_own_snapshot_locked(buf, out_snapshot); *out_buffer_id = buf_id; result = CLUSTER_PCM_OWN_OK; @@ -446,29 +454,31 @@ cluster_bufmgr_pcm_own_snapshot_by_tag(const BufferTag *tag, int *out_buffer_id, * present. Queue JOIN/WAIT never sets a reservation flag here. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_finish_s_release_to_n(BufferDesc *buf, - const ClusterPcmOwnSnapshot *expected_s, - ClusterPcmOwnSnapshot *out_n_snapshot) +cluster_bufmgr_pcm_own_finish_s_release_to_n( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_s, + ClusterPcmOwnSnapshot *out_n_snapshot) { ClusterPcmOwnResult result; - uint32 buf_state; + uint32 buf_state; if (buf == NULL || expected_s == NULL || out_n_snapshot == NULL) return CLUSTER_PCM_OWN_INVALID; memset(out_n_snapshot, 0, sizeof(*out_n_snapshot)); if (ClusterPcmOwnArray == NULL) return CLUSTER_PCM_OWN_NOT_READY; - if (expected_s->pcm_state != (uint8)PCM_STATE_S || expected_s->flags != 0) + if (expected_s->pcm_state != (uint8) PCM_STATE_S || expected_s->flags != 0) return CLUSTER_PCM_OWN_STALE; buf_state = LockBufHdr(buf); if (!cluster_pcm_own_snapshot_matches_locked(buf, expected_s)) result = CLUSTER_PCM_OWN_STALE; - else { + else + { result = cluster_pcm_own_bump_locked(buf, 0, 0, NULL, NULL); - if (result == CLUSTER_PCM_OWN_OK) { - buf->pcm_state = (uint8)PCM_STATE_N; - buf->buffer_type = (uint8)BUF_TYPE_CURRENT; + if (result == CLUSTER_PCM_OWN_OK) + { + buf->pcm_state = (uint8) PCM_STATE_N; + buf->buffer_type = (uint8) BUF_TYPE_CURRENT; cluster_pcm_own_snapshot_locked(buf, out_n_snapshot); } } @@ -490,7 +500,8 @@ cluster_bufmgr_pcm_own_finish_s_release_to_n(BufferDesc *buf, * pin-only readers may consume bytes without that content-lock gate. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, XLogRecPtr *out_page_lsn, +cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, + XLogRecPtr *out_page_lsn, uint64 *out_page_scn) { BufferTag lookup_tag; @@ -526,7 +537,8 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, XLogRecPtr partition_lock = BufMappingPartitionLock(hashcode); LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&lookup_tag, hashcode); - if (buf_id < 0) { + if (buf_id < 0) + { LWLockRelease(partition_lock); return CLUSTER_PCM_OWN_STALE; } @@ -534,14 +546,15 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, XLogRecPtr buf_state = LockBufHdr(buf); if (!BufferTagsEqual(&buf->tag, tag) || (buf_state & BM_VALID) == 0) result = CLUSTER_PCM_OWN_STALE; - else { + else + { shared_refcount = BUF_STATE_GET_REFCOUNT(buf_state); token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, token); if (live_result == CLUSTER_PCM_OWN_CORRUPT) result = live_result; - else if (buf->pcm_state != (uint8)PCM_STATE_S) + else if (buf->pcm_state != (uint8) PCM_STATE_S) result = CLUSTER_PCM_OWN_STALE; else if (flags != 0) result = CLUSTER_PCM_OWN_BUSY; @@ -554,7 +567,8 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, XLogRecPtr else if (cluster_pcm_x_revoke_finish_mode(tag, shared_refcount) != CLUSTER_PCM_X_REVOKE_FINISH_RETAIN) result = CLUSTER_PCM_OWN_BUSY; - else { + else + { cluster_pcm_own_snapshot_locked(buf, &expected_s); cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); buf_state = 0; @@ -577,10 +591,11 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, XLogRecPtr || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state) || (buf_state & BM_IO_IN_PROGRESS) != 0) result = CLUSTER_PCM_OWN_STALE; - else { - page = (Page)BufHdrGetBlock(buf); + else + { + page = (Page) BufHdrGetBlock(buf); page_lsn = PageGetLSN(page); - page_scn = (uint64)((PageHeader)page)->pd_block_scn; + page_scn = (uint64) ((PageHeader) page)->pd_block_scn; dirty = (buf_state & BM_DIRTY) != 0; } UnlockBufHdr(buf, buf_state); @@ -588,24 +603,29 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, XLogRecPtr if (result == CLUSTER_PCM_OWN_OK && dirty && !XLogRecPtrIsInvalid(page_lsn)) XLogFlush(cluster_gcs_clamp_ship_flush_lsn(page_lsn)); - if (result == CLUSTER_PCM_OWN_OK) { + if (result == CLUSTER_PCM_OWN_OK) + { buf_state = LockBufHdr(buf); - page = (Page)BufHdrGetBlock(buf); + page = (Page) BufHdrGetBlock(buf); if (!cluster_pcm_own_snapshot_matches_locked(buf, &expected_s) || (buf_state & BM_VALID) == 0 || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state) - || (buf_state & BM_IO_IN_PROGRESS) != 0 || PageGetLSN(page) != page_lsn) + || (buf_state & BM_IO_IN_PROGRESS) != 0 + || PageGetLSN(page) != page_lsn) result = CLUSTER_PCM_OWN_STALE; - else { + else + { result = cluster_pcm_own_bump_locked(buf, 0, 0, NULL, NULL); - if (result == CLUSTER_PCM_OWN_OK) { - buf->pcm_state = (uint8)PCM_STATE_N; + if (result == CLUSTER_PCM_OWN_OK) + { + buf->pcm_state = (uint8) PCM_STATE_N; /* PI+BM_VALID is the established never-write/never-serve shape. * Unlike a source retain it has no live reservation flag (the * monotonic token remains idle): the next exact GRANT_PENDING * install may overwrite it, then republishes CURRENT. */ - buf->buffer_type = (uint8)BUF_TYPE_PI; - buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR); + buf->buffer_type = (uint8) BUF_TYPE_PI; + buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED + | BM_IO_ERROR); } } UnlockBufHdr(buf, buf_state); @@ -624,7 +644,8 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, XLogRecPtr PG_END_TRY(); cluster_bufmgr_unpin_for_gcs(buf); - if (result == CLUSTER_PCM_OWN_OK) { + if (result == CLUSTER_PCM_OWN_OK) + { *out_page_lsn = page_lsn; *out_page_scn = page_scn; } @@ -634,9 +655,8 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, XLogRecPtr /* Make freshly installed bytes visible only while the same queue * reservation still owns both the descriptor and content EXCLUSIVE. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_publish_installed_x_image(BufferDesc *buf, - const ClusterPcmOwnSnapshot *expected, - uint64 reservation_token) +cluster_bufmgr_pcm_own_publish_installed_x_image( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected, uint64 reservation_token) { ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; uint32 buf_state; @@ -647,7 +667,7 @@ cluster_bufmgr_pcm_own_publish_installed_x_image(BufferDesc *buf, return CLUSTER_PCM_OWN_INVALID; if (ClusterPcmOwnArray == NULL) return CLUSTER_PCM_OWN_NOT_READY; - if (expected->pcm_state != (uint8)PCM_STATE_N || expected->flags != 0 + if (expected->pcm_state != (uint8) PCM_STATE_N || expected->flags != 0 || expected->reservation_token == UINT64_MAX || reservation_token != expected->reservation_token + 1) return CLUSTER_PCM_OWN_STALE; @@ -657,11 +677,12 @@ cluster_bufmgr_pcm_own_publish_installed_x_image(BufferDesc *buf, || cluster_pcm_own_gen_get(buf->buf_id) != expected->generation || cluster_pcm_own_reservation_token_get(buf->buf_id) != reservation_token || cluster_pcm_own_flags_get(buf->buf_id) != PCM_OWN_FLAG_GRANT_PENDING - || buf->pcm_state != (uint8)PCM_STATE_N) + || buf->pcm_state != (uint8) PCM_STATE_N) result = CLUSTER_PCM_OWN_STALE; - else { + else + { buf_state |= BM_VALID; - buf->buffer_type = (uint8)BUF_TYPE_CURRENT; + buf->buffer_type = (uint8) BUF_TYPE_CURRENT; } UnlockBufHdr(buf, buf_state); return result; @@ -964,7 +985,8 @@ cluster_pcm_own_abort_grant_after_master_rollback(BufferDesc *buf, return result; } -pg_attribute_noreturn() static void cluster_pcm_own_rollback_grant_after_error_and_rethrow( +pg_attribute_noreturn() static void +cluster_pcm_own_rollback_grant_after_error_and_rethrow( BufferDesc *buf, const ClusterPcmOwnSnapshot *base, uint64 reservation_token, PcmLockMode acquired_mode, bool has_reservation, ErrorData *original_error, MemoryContext caller_context) @@ -1003,15 +1025,15 @@ pg_attribute_noreturn() static void cluster_pcm_own_rollback_grant_after_error_a if (master_released) { if (has_reservation) { - rollback_result - = cluster_pcm_own_abort_grant_after_master_rollback(buf, base, reservation_token); + rollback_result = cluster_pcm_own_abort_grant_after_master_rollback( + buf, base, reservation_token); if (rollback_result != CLUSTER_PCM_OWN_OK) elog(LOG, "failed to converge local cluster PCM state after content-lock error " "master rollback: buffer=%d token=%llu result=%d; reservation remains " "fail-closed", - buf != NULL ? buf->buf_id : -1, (unsigned long long)reservation_token, - (int)rollback_result); + buf != NULL ? buf->buf_id : -1, + (unsigned long long)reservation_token, (int)rollback_result); } else elog(LOG, "cluster PCM content-lock error rolled back a master grant without local " @@ -1105,7 +1127,7 @@ static bool cluster_bufmgr_in_gcs_drop = false; static inline void cluster_pcm_own_read(BufferDesc *buf, uint8 *out_state, uint64 *out_gen, uint32 *out_flags) { - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf); if (out_state != NULL) @@ -1121,7 +1143,8 @@ cluster_pcm_own_read(BufferDesc *buf, uint8 *out_state, uint64 *out_gen, uint32 * bounded by PG's own held-LWLock ceiling and stores the complete exact * handle returned by PCM-X; error cleanup never reconstructs identity from a * possibly changed BufferDesc ownership mirror. */ -typedef enum ClusterPcmXHolderLedgerPhase { +typedef enum ClusterPcmXHolderLedgerPhase +{ PCM_X_HOLDER_LEDGER_UNUSED = 0, PCM_X_HOLDER_LEDGER_ACQUIRING, PCM_X_HOLDER_LEDGER_ACTIVE, @@ -1129,14 +1152,16 @@ typedef enum ClusterPcmXHolderLedgerPhase { PCM_X_HOLDER_LEDGER_DEFERRED } ClusterPcmXHolderLedgerPhase; -typedef struct ClusterPcmXHolderLedgerEntry { +typedef struct ClusterPcmXHolderLedgerEntry +{ ClusterPcmXHolderLedgerPhase phase; - int32 buffer_id; - LWLock *content_lock; + int32 buffer_id; + LWLock *content_lock; PcmXLocalHolderHandle handle; } ClusterPcmXHolderLedgerEntry; -static ClusterPcmXHolderLedgerEntry cluster_bufmgr_pcm_x_holder_ledger[LWLOCK_MAX_HELD_BY_PROC]; +static ClusterPcmXHolderLedgerEntry + cluster_bufmgr_pcm_x_holder_ledger[LWLOCK_MAX_HELD_BY_PROC]; static uint64 cluster_bufmgr_pcm_x_holder_identity = 0; StaticAssertDecl(lengthof(cluster_bufmgr_pcm_x_holder_ledger) == LWLOCK_MAX_HELD_BY_PROC, @@ -1183,9 +1208,10 @@ static void cluster_bufmgr_pcm_direct_init_snapshot_locked(BufferDesc *buf, uint ClusterPcmDirectInitSnapshot *out); static void -cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, uint32 wait_index) +cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, + uint32 wait_index) { - long delay_ms; + long delay_ms; PcmXQueueResult guard_result; /* @@ -1193,11 +1219,10 @@ cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, ui * lock is held would make RETIRE depend on the holder it is draining. */ if (content_lock == NULL || LWLockHeldByMe(content_lock)) - ereport( - ERROR, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("cannot wait for a cluster PCM-X holder gate while holding content authority"), - errdetail("buffer=%d wait_index=%u", buffer_id, wait_index))); + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cannot wait for a cluster PCM-X holder gate while holding content authority"), + errdetail("buffer=%d wait_index=%u", buffer_id, wait_index))); /* A different held content lock may itself be the holder that a frozen * PROBE is draining. Reuse the protocol's exact held-lock snapshot so a * safe lock-coupling path can still wait, but never sleep across a closed @@ -1205,23 +1230,24 @@ cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, ui */ guard_result = cluster_pcm_x_nested_wait_guard_before_block(); if (guard_result != PCM_X_QUEUE_OK) - cluster_bufmgr_pcm_x_holder_report_failure(guard_result, GetBufferDescriptor(buffer_id), - "retry wait nested guard"); + cluster_bufmgr_pcm_x_holder_report_failure( + guard_result, GetBufferDescriptor(buffer_id), "retry wait nested guard"); delay_ms = cluster_pcm_x_holder_retry_delay_ms(wait_index); CHECK_FOR_INTERRUPTS(); - (void)WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, delay_ms, - WAIT_EVENT_PCM_BLOCK_CONVERT_WAIT); + (void) WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, delay_ms, + WAIT_EVENT_PCM_BLOCK_CONVERT_WAIT); CHECK_FOR_INTERRUPTS(); } static ClusterPcmXHolderLedgerEntry * cluster_bufmgr_pcm_x_holder_find(BufferDesc *buf) { - int i; + int i; if (buf == NULL) return NULL; - for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) { + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) + { ClusterPcmXHolderLedgerEntry *entry = &cluster_bufmgr_pcm_x_holder_ledger[i]; if (entry->phase != PCM_X_HOLDER_LEDGER_UNUSED && entry->buffer_id == buf->buf_id) @@ -1233,7 +1259,7 @@ cluster_bufmgr_pcm_x_holder_find(BufferDesc *buf) static ClusterPcmXHolderLedgerEntry * cluster_bufmgr_pcm_x_holder_free_entry(void) { - int i; + int i; for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) if (cluster_bufmgr_pcm_x_holder_ledger[i].phase == PCM_X_HOLDER_LEDGER_UNUSED) @@ -1242,10 +1268,11 @@ cluster_bufmgr_pcm_x_holder_free_entry(void) } static bool -cluster_bufmgr_pcm_x_holder_entry_exact(const ClusterPcmXHolderLedgerEntry *entry, BufferDesc *buf) +cluster_bufmgr_pcm_x_holder_entry_exact(const ClusterPcmXHolderLedgerEntry *entry, + BufferDesc *buf) { PcmXRuntimeSnapshot runtime; - uint64 cluster_epoch; + uint64 cluster_epoch; if (entry == NULL || buf == NULL) return false; @@ -1256,7 +1283,8 @@ cluster_bufmgr_pcm_x_holder_entry_exact(const ClusterPcmXHolderLedgerEntry *entr return false; if (cluster_node_id < 0 || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT || entry->handle.key.identity.node_id != cluster_node_id || MyProc == NULL - || MyProc->pgprocno < 0 || entry->handle.key.identity.procno != (uint32)MyProc->pgprocno + || MyProc->pgprocno < 0 + || entry->handle.key.identity.procno != (uint32) MyProc->pgprocno || entry->handle.key.identity.request_id == 0 || entry->handle.key.identity.wait_seq != entry->handle.key.identity.request_id || entry->handle.tag_slot.slot_index == PCM_X_INVALID_SLOT_INDEX @@ -1272,20 +1300,22 @@ cluster_bufmgr_pcm_x_holder_entry_exact(const ClusterPcmXHolderLedgerEntry *entr static void cluster_bufmgr_pcm_x_holder_report_failure(PcmXQueueResult result, BufferDesc *buf, - const char *operation) + const char *operation) { if (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY || result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BARRIER_CLOSED || result == PCM_X_QUEUE_NO_CAPACITY) - ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), - errmsg("cluster PCM-X holder operation is not ready"), - errdetail("operation=%s buffer=%d result=%d", operation, - buf != NULL ? buf->buf_id : -1, (int)result))); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_IN_USE), + errmsg("cluster PCM-X holder operation is not ready"), + errdetail("operation=%s buffer=%d result=%d", operation, + buf != NULL ? buf->buf_id : -1, (int) result))); ereport(ERROR, - (errcode(ERRCODE_DATA_CORRUPTED), errmsg("cluster PCM-X holder operation failed"), + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cluster PCM-X holder operation failed"), errdetail("operation=%s buffer=%d result=%d", operation, - buf != NULL ? buf->buf_id : -1, (int)result))); + buf != NULL ? buf->buf_id : -1, (int) result))); } static void @@ -1311,36 +1341,37 @@ cluster_bufmgr_pcm_x_holder_defer_fail_closed(ClusterPcmXHolderLedgerEntry *entr static void cluster_bufmgr_pcm_x_holder_drain_deferred_nowait(void) { - int i; + int i; - for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) { + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) + { ClusterPcmXHolderLedgerEntry *entry = &cluster_bufmgr_pcm_x_holder_ledger[i]; PcmXQueueResult result; if (entry->phase != PCM_X_HOLDER_LEDGER_DEFERRED) continue; - if (entry->content_lock == NULL) { + if (entry->content_lock == NULL) + { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); - elog(LOG, - "could not drain deferred cluster PCM-X holder with no content lock: buffer=%d", + elog(LOG, "could not drain deferred cluster PCM-X holder with no content lock: buffer=%d", entry->buffer_id); continue; } - if (LWLockHeldByMe(entry->content_lock)) { - elog(LOG, - "could not drain deferred cluster PCM-X holder while content lock is held: " - "buffer=%d", + if (LWLockHeldByMe(entry->content_lock)) + { + elog(LOG, "could not drain deferred cluster PCM-X holder while content lock is held: buffer=%d", entry->buffer_id); continue; } result = cluster_pcm_x_local_holder_exceptional_detach_exact(&entry->handle, - entry->content_lock); + entry->content_lock); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_NOT_FOUND) cluster_bufmgr_pcm_x_holder_clear(entry); - else if (result != PCM_X_QUEUE_GATE_RETRY && result != PCM_X_QUEUE_BUSY) { + else if (result != PCM_X_QUEUE_GATE_RETRY && result != PCM_X_QUEUE_BUSY) + { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); elog(LOG, "could not drain deferred exact cluster PCM-X holder: buffer=%d result=%d", - entry->buffer_id, (int)result); + entry->buffer_id, (int) result); } } } @@ -1353,42 +1384,51 @@ static void cluster_bufmgr_pcm_x_holder_drain_deferred(ClusterPcmXHolderLedgerEntry *entry) { PcmXRuntimeSnapshot runtime; - uint32 wait_index = 0; + uint32 wait_index = 0; if (entry == NULL || entry->phase != PCM_X_HOLDER_LEDGER_DEFERRED) return; - for (;;) { + for (;;) + { PcmXQueueResult result; ClusterPcmXHolderRetryAction action; - if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)) { + if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)) + { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, - GetBufferDescriptor(entry->buffer_id), - "deferred detach lock order"); + GetBufferDescriptor(entry->buffer_id), + "deferred detach lock order"); } result = cluster_pcm_x_local_holder_exceptional_detach_exact(&entry->handle, - entry->content_lock); + entry->content_lock); action = cluster_pcm_x_holder_unregister_retry_action(result, 0); - if (action == CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE) { + if (action == CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE) + { cluster_bufmgr_pcm_x_holder_clear(entry); return; } - if (action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT) { + if (action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT) + { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); - cluster_bufmgr_pcm_x_holder_report_failure( - result, GetBufferDescriptor(entry->buffer_id), "deferred detach"); + cluster_bufmgr_pcm_x_holder_report_failure(result, + GetBufferDescriptor(entry->buffer_id), + "deferred detach"); } - cluster_bufmgr_pcm_x_holder_retry_wait(entry->content_lock, entry->buffer_id, - wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); + cluster_bufmgr_pcm_x_holder_retry_wait( + entry->content_lock, entry->buffer_id, + wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); wait_index++; - if (wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS == 0) { + if (wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS == 0) + { runtime = cluster_pcm_x_runtime_snapshot(); - if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) { + if (runtime.state != PCM_X_RUNTIME_ACTIVE + || runtime.master_session_incarnation == 0) + { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); - cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_NOT_READY, - GetBufferDescriptor(entry->buffer_id), - "deferred detach runtime"); + cluster_bufmgr_pcm_x_holder_report_failure( + PCM_X_QUEUE_NOT_READY, GetBufferDescriptor(entry->buffer_id), + "deferred detach runtime"); } } } @@ -1414,16 +1454,19 @@ cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf) cluster_bufmgr_pcm_x_holder_drain_deferred_nowait(); entry = cluster_bufmgr_pcm_x_holder_find(buf); - if (entry != NULL && entry->phase == PCM_X_HOLDER_LEDGER_DEFERRED) { + if (entry != NULL && entry->phase == PCM_X_HOLDER_LEDGER_DEFERRED) + { cluster_bufmgr_pcm_x_holder_drain_deferred(entry); entry = NULL; } - if (entry != NULL) { + if (entry != NULL) + { if (entry->phase != PCM_X_HOLDER_LEDGER_ACQUIRING - || !cluster_bufmgr_pcm_x_holder_entry_exact(entry, buf)) { + || !cluster_bufmgr_pcm_x_holder_entry_exact(entry, buf)) + { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, buf, - "reuse existing holder ledger"); + "reuse existing holder ledger"); } if (LWLockHeldByMe(entry->content_lock)) ereport(ERROR, @@ -1463,15 +1506,18 @@ cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf) errmsg("cannot register a cluster PCM-X holder for a pre-held content lock"), errdetail("buffer=%d", buf->buf_id))); entry = cluster_bufmgr_pcm_x_holder_free_entry(); - if (entry == NULL) { - int i; + if (entry == NULL) + { + int i; /* * Deferred exact handles do not consume the ledger forever. If every * slot is occupied, wait for one safe detach before declaring a real * held-LWLock bound violation. */ - for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) { - ClusterPcmXHolderLedgerEntry *candidate = &cluster_bufmgr_pcm_x_holder_ledger[i]; + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) + { + ClusterPcmXHolderLedgerEntry *candidate + = &cluster_bufmgr_pcm_x_holder_ledger[i]; if (candidate->phase != PCM_X_HOLDER_LEDGER_DEFERRED) continue; @@ -1481,9 +1527,10 @@ cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf) } } if (entry == NULL) - ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg("cluster PCM-X holder ledger is full"), - errdetail("maximum entries=%d", LWLOCK_MAX_HELD_BY_PROC))); + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("cluster PCM-X holder ledger is full"), + errdetail("maximum entries=%d", LWLOCK_MAX_HELD_BY_PROC))); cluster_pcm_own_read(buf, NULL, &own_generation, NULL); cluster_epoch = cluster_epoch_get_current(); @@ -1515,7 +1562,8 @@ cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf) key.identity.base_own_generation = own_generation; } key.buffer_id = buf->buf_id; - for (;;) { + for (;;) + { ClusterPcmXHolderRetryAction action; PcmXRuntimeSnapshot current_runtime; @@ -1547,7 +1595,8 @@ cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf) if (action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT) cluster_bufmgr_pcm_x_holder_report_failure(result, buf, "register"); cluster_bufmgr_pcm_x_holder_retry_wait( - content_lock, buf->buf_id, wait_index++ % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); + content_lock, buf->buf_id, + wait_index++ % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); } return entry; @@ -1561,12 +1610,12 @@ cluster_bufmgr_pcm_x_holder_activate(ClusterPcmXHolderLedgerEntry *entry) if (entry == NULL) return; if (entry->phase != PCM_X_HOLDER_LEDGER_ACQUIRING) - cluster_bufmgr_pcm_x_holder_report_failure( - PCM_X_QUEUE_BAD_STATE, GetBufferDescriptor(entry->buffer_id), "activate phase"); + cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, + GetBufferDescriptor(entry->buffer_id), "activate phase"); result = cluster_pcm_x_local_holder_activate_exact(&entry->handle); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) - cluster_bufmgr_pcm_x_holder_report_failure(result, GetBufferDescriptor(entry->buffer_id), - "activate"); + cluster_bufmgr_pcm_x_holder_report_failure(result, + GetBufferDescriptor(entry->buffer_id), "activate"); entry->phase = PCM_X_HOLDER_LEDGER_ACTIVE; } @@ -1578,12 +1627,13 @@ cluster_bufmgr_pcm_x_holder_mark_releasing(ClusterPcmXHolderLedgerEntry *entry) if (entry == NULL) return; if (entry->phase != PCM_X_HOLDER_LEDGER_ACTIVE) - cluster_bufmgr_pcm_x_holder_report_failure( - PCM_X_QUEUE_BAD_STATE, GetBufferDescriptor(entry->buffer_id), "mark releasing phase"); + cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, + GetBufferDescriptor(entry->buffer_id), + "mark releasing phase"); result = cluster_pcm_x_local_holder_mark_releasing_exact(&entry->handle); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) - cluster_bufmgr_pcm_x_holder_report_failure(result, GetBufferDescriptor(entry->buffer_id), - "mark releasing"); + cluster_bufmgr_pcm_x_holder_report_failure(result, + GetBufferDescriptor(entry->buffer_id), "mark releasing"); entry->phase = PCM_X_HOLDER_LEDGER_RELEASING; } @@ -1591,35 +1641,42 @@ static void cluster_bufmgr_pcm_x_holder_unregister(ClusterPcmXHolderLedgerEntry *entry) { PcmXQueueResult result; - uint32 waits_used = 0; + uint32 waits_used = 0; if (entry == NULL) return; if (entry->phase != PCM_X_HOLDER_LEDGER_RELEASING) - cluster_bufmgr_pcm_x_holder_report_failure( - PCM_X_QUEUE_BAD_STATE, GetBufferDescriptor(entry->buffer_id), "unregister phase"); + cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, + GetBufferDescriptor(entry->buffer_id), + "unregister phase"); if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)) - cluster_bufmgr_pcm_x_holder_report_failure( - PCM_X_QUEUE_BAD_STATE, GetBufferDescriptor(entry->buffer_id), "unregister lock order"); - for (;;) { + cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, + GetBufferDescriptor(entry->buffer_id), + "unregister lock order"); + for (;;) + { ClusterPcmXHolderRetryAction action; result = cluster_pcm_x_local_holder_unregister_exact(&entry->handle); action = cluster_pcm_x_holder_unregister_retry_action(result, waits_used); - if (action == CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE) { + if (action == CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE) + { cluster_bufmgr_pcm_x_holder_clear(entry); return; } - if (action == CLUSTER_PCM_X_HOLDER_RETRY_DEFER) { + if (action == CLUSTER_PCM_X_HOLDER_RETRY_DEFER) + { entry->phase = PCM_X_HOLDER_LEDGER_DEFERRED; return; } - if (action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT) { + if (action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT) + { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); cluster_bufmgr_pcm_x_holder_report_failure( result, GetBufferDescriptor(entry->buffer_id), "unregister"); } - cluster_bufmgr_pcm_x_holder_retry_wait(entry->content_lock, entry->buffer_id, waits_used++); + cluster_bufmgr_pcm_x_holder_retry_wait(entry->content_lock, entry->buffer_id, + waits_used++); } } @@ -1630,7 +1687,8 @@ cluster_bufmgr_pcm_x_holder_abort_acquiring(ClusterPcmXHolderLedgerEntry *entry) if (entry == NULL) return; - if (entry->content_lock == NULL) { + if (entry->content_lock == NULL) + { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); elog(LOG, "could not abort cluster PCM-X ACQUIRING holder with no content lock: buffer=%d", entry->buffer_id); @@ -1643,45 +1701,46 @@ cluster_bufmgr_pcm_x_holder_abort_acquiring(ClusterPcmXHolderLedgerEntry *entry) cluster_bufmgr_pcm_x_holder_clear(entry); else if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BUSY) entry->phase = PCM_X_HOLDER_LEDGER_DEFERRED; - else { + else + { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); elog(LOG, "could not abort exact cluster PCM-X ACQUIRING holder: buffer=%d result=%d", - entry->buffer_id, (int)result); + entry->buffer_id, (int) result); } } static void cluster_bufmgr_pcm_x_holder_exception_cleanup_all(void) { - int i; + int i; - for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) { + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) + { ClusterPcmXHolderLedgerEntry *entry = &cluster_bufmgr_pcm_x_holder_ledger[i]; PcmXQueueResult result; if (entry->phase == PCM_X_HOLDER_LEDGER_UNUSED) continue; - if (entry->content_lock == NULL) { + if (entry->content_lock == NULL) + { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); - elog( - LOG, - "could not detach cluster PCM-X holder after error with no content lock: buffer=%d", - entry->buffer_id); + elog(LOG, "could not detach cluster PCM-X holder after error with no content lock: buffer=%d", + entry->buffer_id); continue; } if (LWLockHeldByMe(entry->content_lock)) continue; result = cluster_pcm_x_local_holder_exceptional_detach_exact(&entry->handle, - entry->content_lock); + entry->content_lock); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_NOT_FOUND) cluster_bufmgr_pcm_x_holder_clear(entry); else if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BUSY) entry->phase = PCM_X_HOLDER_LEDGER_DEFERRED; - else { + else + { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); - elog(LOG, - "could not detach exact cluster PCM-X holder after error: buffer=%d result=%d", - entry->buffer_id, (int)result); + elog(LOG, "could not detach exact cluster PCM-X holder after error: buffer=%d result=%d", + entry->buffer_id, (int) result); } } } @@ -1689,7 +1748,8 @@ cluster_bufmgr_pcm_x_holder_exception_cleanup_all(void) static void cluster_bufmgr_pcm_x_holder_reset(void) { - MemSet(cluster_bufmgr_pcm_x_holder_ledger, 0, sizeof(cluster_bufmgr_pcm_x_holder_ledger)); + MemSet(cluster_bufmgr_pcm_x_holder_ledger, 0, + sizeof(cluster_bufmgr_pcm_x_holder_ledger)); cluster_bufmgr_pcm_x_holder_identity = 0; } @@ -1752,9 +1812,9 @@ static void cluster_bufmgr_pcm_x_writer_report_failure(PcmXQueueResult result, BufferDesc *buf, const char *operation) { - BufferTag guard_tag; - int guard_result = 0; - char guard_detail[96]; + BufferTag guard_tag; + int guard_result = 0; + char guard_detail[96]; /* Name the exact requester escape arm and, for a nested-guard refusal, * the held content-lock tag that closed the guard. */ @@ -2192,7 +2252,8 @@ static inline int32 GetPrivateRefCount(Buffer buffer); * serialized by the BufferDesc header lock. The private refcount table is * process-local and cannot change concurrently inside this backend. */ static void -cluster_bufmgr_pcm_direct_init_snapshot_locked(BufferDesc *buf, uint32 buf_state, bool page_is_new, +cluster_bufmgr_pcm_direct_init_snapshot_locked(BufferDesc *buf, uint32 buf_state, + bool page_is_new, ClusterPcmDirectInitSnapshot *out) { memset(out, 0, sizeof(*out)); @@ -2208,34 +2269,37 @@ cluster_bufmgr_pcm_direct_init_snapshot_locked(BufferDesc *buf, uint32 buf_state out->page_is_new = page_is_new; } -pg_attribute_noreturn() static void cluster_bufmgr_pcm_direct_init_report_failure( - BufferDesc *buf, ClusterPcmOwnResult result, const ClusterPcmDirectInitSnapshot *observed, - const char *context) +pg_attribute_noreturn() static void +cluster_bufmgr_pcm_direct_init_report_failure(BufferDesc *buf, ClusterPcmOwnResult result, + const ClusterPcmDirectInitSnapshot *observed, + const char *context) { - uint64 generation = observed != NULL ? observed->generation : 0; - uint32 flags = observed != NULL ? observed->flags : 0; + uint64 generation = observed != NULL ? observed->generation : 0; + uint32 flags = observed != NULL ? observed->flags : 0; cluster_grd_inc_block_path_failclosed(); - if (result == CLUSTER_PCM_OWN_BUSY || result == CLUSTER_PCM_OWN_CORRUPT - || result == CLUSTER_PCM_OWN_EXHAUSTED || result == CLUSTER_PCM_OWN_NOT_READY) + if (result == CLUSTER_PCM_OWN_BUSY || result == CLUSTER_PCM_OWN_CORRUPT || + result == CLUSTER_PCM_OWN_EXHAUSTED || result == CLUSTER_PCM_OWN_NOT_READY) cluster_pcm_own_report_bump_failure(buf, result, generation, flags, context); - ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("cluster PCM direct initialization lacks an exact live proof"), - errdetail("context=%s buffer=%d result=%d generation=%llu flags=0x%x", context, - buf != NULL ? buf->buf_id : -1, (int)result, - (unsigned long long)generation, flags))); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cluster PCM direct initialization lacks an exact live proof"), + errdetail("context=%s buffer=%d result=%d generation=%llu flags=0x%x", + context, buf != NULL ? buf->buf_id : -1, (int) result, + (unsigned long long) generation, flags))); } -pg_attribute_noreturn() static void cluster_bufmgr_pcm_direct_init_no_grant_failclosed( - BufferDesc *buf, ClusterPcmDirectInitKind kind) +pg_attribute_noreturn() static void +cluster_bufmgr_pcm_direct_init_no_grant_failclosed(BufferDesc *buf, + ClusterPcmDirectInitKind kind) { cluster_grd_inc_block_path_failclosed(); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("cluster PCM direct initialization did not obtain X ownership"), errdetail("buffer=%d operation=%d returned a one-shot image without a durable grant", - buf != NULL ? buf->buf_id : -1, (int)kind))); + buf != NULL ? buf->buf_id : -1, (int) kind))); } /* Arm only at a source-authorized operation site. This does not reserve the @@ -2247,20 +2311,21 @@ cluster_bufmgr_pcm_arm_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind kin { ClusterPcmDirectInitSnapshot observed; ClusterPcmOwnResult result; - uint32 buf_state; + uint32 buf_state; memset(proof, 0, sizeof(*proof)); if (!cluster_pcm_is_active() || !cluster_bufmgr_should_pcm_track(buf)) return; buf_state = LockBufHdr(buf); - cluster_bufmgr_pcm_direct_init_snapshot_locked(buf, buf_state, - PageIsNew((Page)BufHdrGetBlock(buf)), &observed); + cluster_bufmgr_pcm_direct_init_snapshot_locked( + buf, buf_state, PageIsNew((Page) BufHdrGetBlock(buf)), &observed); result = cluster_pcm_direct_init_proof_arm(kind, &observed, proof); UnlockBufHdr(buf, buf_state); if (result != CLUSTER_PCM_OWN_OK) - cluster_bufmgr_pcm_direct_init_report_failure(buf, result, &observed, "direct-init arm"); + cluster_bufmgr_pcm_direct_init_report_failure(buf, result, &observed, + "direct-init arm"); } /* @@ -2277,25 +2342,26 @@ cluster_bufmgr_pcm_gate_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind ki ClusterPcmDirectInitSnapshot observed; ClusterPcmOwnSnapshot pending_base; ClusterPcmOwnResult pending_result; - bool grant_acquired = false; - uint32 buf_state; - uint64 pending_token = 0; - uint64 committed_generation; + bool grant_acquired = false; + uint32 buf_state; + uint64 pending_token = 0; + uint64 committed_generation; if (!cluster_pcm_is_active() || !cluster_bufmgr_should_pcm_track(buf)) return; if (proof == NULL) cluster_bufmgr_pcm_direct_init_report_failure(buf, CLUSTER_PCM_OWN_INVALID, NULL, - "direct-init missing proof"); + "direct-init missing proof"); buf_state = LockBufHdr(buf); - cluster_bufmgr_pcm_direct_init_snapshot_locked(buf, buf_state, - PageIsNew((Page)BufHdrGetBlock(buf)), &observed); + cluster_bufmgr_pcm_direct_init_snapshot_locked( + buf, buf_state, PageIsNew((Page) BufHdrGetBlock(buf)), &observed); pending_result = cluster_pcm_direct_init_proof_consume(kind, &observed, proof); if (pending_result == CLUSTER_PCM_OWN_OK) pending_result = cluster_pcm_own_reservation_begin_exact( buf->buf_id, observed.generation, PCM_OWN_FLAG_GRANT_PENDING, &pending_token); - if (pending_result == CLUSTER_PCM_OWN_OK) { + if (pending_result == CLUSTER_PCM_OWN_OK) + { memset(&pending_base, 0, sizeof(pending_base)); pending_base.tag = observed.tag; pending_base.generation = observed.generation; @@ -2307,7 +2373,7 @@ cluster_bufmgr_pcm_gate_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind ki if (pending_result != CLUSTER_PCM_OWN_OK) cluster_bufmgr_pcm_direct_init_report_failure(buf, pending_result, &observed, - "direct-init consume"); + "direct-init consume"); PG_TRY(); { @@ -2323,11 +2389,12 @@ cluster_bufmgr_pcm_gate_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind ki if (grant_acquired) cluster_pcm_own_finish_grant_or_rollback(buf, &pending_base, pending_token, - (uint8)PCM_STATE_X, PCM_LOCK_MODE_X, - &committed_generation); - else { + (uint8) PCM_STATE_X, PCM_LOCK_MODE_X, + &committed_generation); + else + { cluster_pcm_own_abort_grant_or_error(buf, &pending_base, pending_token, - "direct-init read-image"); + "direct-init read-image"); cluster_bufmgr_pcm_direct_init_no_grant_failclosed(buf, kind); } } @@ -2340,11 +2407,12 @@ cluster_bufmgr_pcm_gate_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind ki * being dropped. For the relations with size below this threshold, we find * the buffers by doing lookups in BufMapping table. */ -#define BUF_DROP_FULL_SCAN_THRESHOLD (uint64)(NBuffers / 32) +#define BUF_DROP_FULL_SCAN_THRESHOLD (uint64) (NBuffers / 32) -typedef struct PrivateRefCountEntry { - Buffer buffer; - int32 refcount; +typedef struct PrivateRefCountEntry +{ + Buffer buffer; + int32 refcount; } PrivateRefCountEntry; /* 64 bytes, about the size of a cache line on common systems */ @@ -2354,9 +2422,10 @@ typedef struct PrivateRefCountEntry { * Status of buffers to checkpoint for a particular tablespace, used * internally in BufferSync. */ -typedef struct CkptTsStatus { +typedef struct CkptTsStatus +{ /* oid of the tablespace */ - Oid tsId; + Oid tsId; /* * Checkpoint progress for this tablespace. To make progress comparable @@ -2365,16 +2434,16 @@ typedef struct CkptTsStatus { * page checkpointed in this tablespace increments this space's progress * by progress_slice. */ - float8 progress; - float8 progress_slice; + float8 progress; + float8 progress_slice; /* number of to-be checkpointed pages in this tablespace */ - int num_to_scan; + int num_to_scan; /* already processed pages in this tablespace */ - int num_scanned; + int num_scanned; /* current offset in CkptBufferIds for this tablespace */ - int index; + int index; } CkptTsStatus; /* @@ -2384,16 +2453,17 @@ typedef struct CkptTsStatus { * DropRelationsAllBuffers. Pointer to this struct and RelFileLocator must be * compatible. */ -typedef struct SMgrSortArray { - RelFileLocator rlocator; /* This must be the first member */ +typedef struct SMgrSortArray +{ + RelFileLocator rlocator; /* This must be the first member */ SMgrRelation srel; } SMgrSortArray; /* GUC variables */ -bool zero_damaged_pages = false; -int bgwriter_lru_maxpages = 100; -double bgwriter_lru_multiplier = 2.0; -bool track_io_timing = false; +bool zero_damaged_pages = false; +int bgwriter_lru_maxpages = 100; +double bgwriter_lru_multiplier = 2.0; +bool track_io_timing = false; /* * How many buffers PrefetchBuffer callers should try to stay ahead of their @@ -2401,22 +2471,22 @@ bool track_io_timing = false; * for buffers not belonging to tablespaces that have their * effective_io_concurrency parameter set. */ -int effective_io_concurrency = DEFAULT_EFFECTIVE_IO_CONCURRENCY; +int effective_io_concurrency = DEFAULT_EFFECTIVE_IO_CONCURRENCY; /* * Like effective_io_concurrency, but used by maintenance code paths that might * benefit from a higher setting because they work on behalf of many sessions. * Overridden by the tablespace setting of the same name. */ -int maintenance_io_concurrency = DEFAULT_MAINTENANCE_IO_CONCURRENCY; +int maintenance_io_concurrency = DEFAULT_MAINTENANCE_IO_CONCURRENCY; /* * GUC variables about triggering kernel writeback for buffers written; OS * dependent defaults are set via the GUC mechanism. */ -int checkpoint_flush_after = DEFAULT_CHECKPOINT_FLUSH_AFTER; -int bgwriter_flush_after = DEFAULT_BGWRITER_FLUSH_AFTER; -int backend_flush_after = DEFAULT_BACKEND_FLUSH_AFTER; +int checkpoint_flush_after = DEFAULT_CHECKPOINT_FLUSH_AFTER; +int bgwriter_flush_after = DEFAULT_BGWRITER_FLUSH_AFTER; +int backend_flush_after = DEFAULT_BACKEND_FLUSH_AFTER; /* local state for LockBufferForCleanup */ static BufferDesc *PinCountWaitBuf = NULL; @@ -2480,14 +2550,16 @@ ReservePrivateRefCountEntry(void) * majority of cases. */ { - int i; + int i; - for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++) { + for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++) + { PrivateRefCountEntry *res; res = &PrivateRefCountArray[i]; - if (res->buffer == InvalidBuffer) { + if (res->buffer == InvalidBuffer) + { ReservedRefCountEntry = res; return; } @@ -2504,17 +2576,19 @@ ReservePrivateRefCountEntry(void) * hashtable. Use that slot. */ PrivateRefCountEntry *hashent; - bool found; + bool found; /* select victim slot */ - ReservedRefCountEntry - = &PrivateRefCountArray[PrivateRefCountClock++ % REFCOUNT_ARRAY_ENTRIES]; + ReservedRefCountEntry = + &PrivateRefCountArray[PrivateRefCountClock++ % REFCOUNT_ARRAY_ENTRIES]; /* Better be used, otherwise we shouldn't get here. */ Assert(ReservedRefCountEntry->buffer != InvalidBuffer); /* enter victim array entry into hashtable */ - hashent = hash_search(PrivateRefCountHash, &(ReservedRefCountEntry->buffer), HASH_ENTER, + hashent = hash_search(PrivateRefCountHash, + &(ReservedRefCountEntry->buffer), + HASH_ENTER, &found); Assert(!found); hashent->refcount = ReservedRefCountEntry->refcount; @@ -2560,7 +2634,7 @@ static PrivateRefCountEntry * GetPrivateRefCountEntry(Buffer buffer, bool do_move) { PrivateRefCountEntry *res; - int i; + int i; Assert(BufferIsValid(buffer)); Assert(!BufferIsLocal(buffer)); @@ -2569,7 +2643,8 @@ GetPrivateRefCountEntry(Buffer buffer, bool do_move) * First search for references in the array, that'll be sufficient in the * majority of cases. */ - for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++) { + for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++) + { res = &PrivateRefCountArray[i]; if (res->buffer == buffer) @@ -2590,12 +2665,15 @@ GetPrivateRefCountEntry(Buffer buffer, bool do_move) if (res == NULL) return NULL; - else if (!do_move) { + else if (!do_move) + { /* caller doesn't want us to move the hash entry into the array */ return res; - } else { + } + else + { /* move buffer from hashtable into the free array slot */ - bool found; + bool found; PrivateRefCountEntry *free; /* Ensure there's a free array slot */ @@ -2654,7 +2732,9 @@ ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref) { Assert(ref->refcount == 0); - if (ref >= &PrivateRefCountArray[0] && ref < &PrivateRefCountArray[REFCOUNT_ARRAY_ENTRIES]) { + if (ref >= &PrivateRefCountArray[0] && + ref < &PrivateRefCountArray[REFCOUNT_ARRAY_ENTRIES]) + { ref->buffer = InvalidBuffer; /* @@ -2663,9 +2743,11 @@ ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref) * entries. */ ReservedRefCountEntry = ref; - } else { - bool found; - Buffer buffer = ref->buffer; + } + else + { + bool found; + Buffer buffer = ref->buffer; hash_search(PrivateRefCountHash, &buffer, HASH_REMOVE, &found); Assert(found); @@ -2681,23 +2763,38 @@ ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref) * NOTE: what we check here is that *this* backend holds a pin on * the buffer. We do not care whether some other backend does. */ -#define BufferIsPinned(bufnum) \ - (!BufferIsValid(bufnum) ? false \ - : BufferIsLocal(bufnum) ? (LocalRefCount[-(bufnum) - 1] > 0) \ - : (GetPrivateRefCount(bufnum) > 0)) - - -static Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, - BlockNumber blockNum, ReadBufferMode mode, - BufferAccessStrategy strategy, bool *hit); -static BlockNumber ExtendBufferedRelCommon(BufferManagerRelation bmr, ForkNumber fork, - BufferAccessStrategy strategy, uint32 flags, - uint32 extend_by, BlockNumber extend_upto, - Buffer *buffers, uint32 *extended_by); -static BlockNumber ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, - BufferAccessStrategy strategy, uint32 flags, - uint32 extend_by, BlockNumber extend_upto, - Buffer *buffers, uint32 *extended_by); +#define BufferIsPinned(bufnum) \ +( \ + !BufferIsValid(bufnum) ? \ + false \ + : \ + BufferIsLocal(bufnum) ? \ + (LocalRefCount[-(bufnum) - 1] > 0) \ + : \ + (GetPrivateRefCount(bufnum) > 0) \ +) + + +static Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, + ForkNumber forkNum, BlockNumber blockNum, + ReadBufferMode mode, BufferAccessStrategy strategy, + bool *hit); +static BlockNumber ExtendBufferedRelCommon(BufferManagerRelation bmr, + ForkNumber fork, + BufferAccessStrategy strategy, + uint32 flags, + uint32 extend_by, + BlockNumber extend_upto, + Buffer *buffers, + uint32 *extended_by); +static BlockNumber ExtendBufferedRelShared(BufferManagerRelation bmr, + ForkNumber fork, + BufferAccessStrategy strategy, + uint32 flags, + uint32 extend_by, + BlockNumber extend_upto, + Buffer *buffers, + uint32 *extended_by); static bool PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy); static void PinBuffer_Locked(BufferDesc *buf); static void UnpinBuffer(BufferDesc *buf); @@ -2715,10 +2812,12 @@ extern void cluster_gcs_block_pi_write_note(BufferTag tag, SCN page_scn); extern bool cluster_gcs_block_test_deliver_self_invalidate(BufferTag tag); #endif static uint32 WaitBufHdrUnlocked(BufferDesc *buf); -static int SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context); +static int SyncOneBuffer(int buf_id, bool skip_recently_used, + WritebackContext *wb_context); static void WaitIO(BufferDesc *buf); static bool StartBufferIO(BufferDesc *buf, bool forInput); -static void TerminateBufferIO(BufferDesc *buf, bool clear_dirty, uint32 set_flag_bits); +static void TerminateBufferIO(BufferDesc *buf, bool clear_dirty, + uint32 set_flag_bits); static void shared_buffer_write_error_callback(void *arg); static void local_buffer_write_error_callback(void *arg); @@ -2727,46 +2826,56 @@ static void local_buffer_write_error_callback(void *arg); static bool InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, LWLock *oldPartitionLock, uint32 buf_state); static void InvalidateBufferCommitTailLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, - LWLock *oldPartitionLock, uint32 buf_state, - uint8 old_pcm_mode, bool release_pcm_holder); + LWLock *oldPartitionLock, uint32 buf_state, + uint8 old_pcm_mode, bool release_pcm_holder); static bool InvalidateBufferTry(BufferDesc *buf); -static BufferDesc *BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, - BlockNumber blockNum, BufferAccessStrategy strategy, bool *foundPtr, - IOContext io_context); +static BufferDesc *BufferAlloc(SMgrRelation smgr, + char relpersistence, + ForkNumber forkNum, + BlockNumber blockNum, + BufferAccessStrategy strategy, + bool *foundPtr, IOContext io_context); static Buffer GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context); -static void FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, - IOContext io_context); -static void FindAndDropRelationBuffers(RelFileLocator rlocator, ForkNumber forkNum, - BlockNumber nForkBlock, BlockNumber firstDelBlock); -static void RelationCopyStorageUsingBuffer(RelFileLocator srclocator, RelFileLocator dstlocator, +static void FlushBuffer(BufferDesc *buf, SMgrRelation reln, + IOObject io_object, IOContext io_context); +static void FindAndDropRelationBuffers(RelFileLocator rlocator, + ForkNumber forkNum, + BlockNumber nForkBlock, + BlockNumber firstDelBlock); +static void RelationCopyStorageUsingBuffer(RelFileLocator srclocator, + RelFileLocator dstlocator, ForkNumber forkNum, bool permanent); static void AtProcExit_Buffers(int code, Datum arg); static void CheckForBufferLeaks(void); #ifdef USE_ASSERT_CHECKING -static void AssertNotCatalogBufferLock(LWLock *lock, LWLockMode mode, void *unused_context); +static void AssertNotCatalogBufferLock(LWLock *lock, LWLockMode mode, + void *unused_context); #endif -static int rlocator_comparator(const void *p1, const void *p2); +static int rlocator_comparator(const void *p1, const void *p2); static inline int buffertag_comparator(const BufferTag *ba, const BufferTag *bb); static inline int ckpt_buforder_comparator(const CkptSortItem *a, const CkptSortItem *b); -static int ts_ckpt_progress_comparator(Datum a, Datum b, void *arg); +static int ts_ckpt_progress_comparator(Datum a, Datum b, void *arg); /* * Implementation of PrefetchBuffer() for shared buffers. */ PrefetchBufferResult -PrefetchSharedBuffer(SMgrRelation smgr_reln, ForkNumber forkNum, BlockNumber blockNum) +PrefetchSharedBuffer(SMgrRelation smgr_reln, + ForkNumber forkNum, + BlockNumber blockNum) { - PrefetchBufferResult result = { InvalidBuffer, false }; - BufferTag newTag; /* identity of requested block */ - uint32 newHash; /* hash value for newTag */ - LWLock *newPartitionLock; /* buffer partition lock for it */ - int buf_id; + PrefetchBufferResult result = {InvalidBuffer, false}; + BufferTag newTag; /* identity of requested block */ + uint32 newHash; /* hash value for newTag */ + LWLock *newPartitionLock; /* buffer partition lock for it */ + int buf_id; Assert(BlockNumberIsValid(blockNum)); /* create a tag so we can lookup the buffer */ - InitBufferTag(&newTag, &smgr_reln->smgr_rlocator.locator, forkNum, blockNum); + InitBufferTag(&newTag, &smgr_reln->smgr_rlocator.locator, + forkNum, blockNum); /* determine its hash code and partition lock ID */ newHash = BufTableHashCode(&newTag); @@ -2778,17 +2887,22 @@ PrefetchSharedBuffer(SMgrRelation smgr_reln, ForkNumber forkNum, BlockNumber blo LWLockRelease(newPartitionLock); /* If not in buffers, initiate prefetch */ - if (buf_id < 0) { + if (buf_id < 0) + { #ifdef USE_PREFETCH /* * Try to initiate an asynchronous read. This returns false in * recovery if the relation file doesn't exist. */ - if ((io_direct_flags & IO_DIRECT_DATA) == 0 && smgrprefetch(smgr_reln, forkNum, blockNum)) { + if ((io_direct_flags & IO_DIRECT_DATA) == 0 && + smgrprefetch(smgr_reln, forkNum, blockNum)) + { result.initiated_io = true; } -#endif /* USE_PREFETCH */ - } else { +#endif /* USE_PREFETCH */ + } + else + { /* * Report the buffer it was in at that time. The caller may be able * to avoid a buffer table lookup, but it's not pinned and it must be @@ -2842,15 +2956,19 @@ PrefetchBuffer(Relation reln, ForkNumber forkNum, BlockNumber blockNum) Assert(RelationIsValid(reln)); Assert(BlockNumberIsValid(blockNum)); - if (RelationUsesLocalBuffers(reln)) { + if (RelationUsesLocalBuffers(reln)) + { /* see comments in ReadBufferExtended */ if (RELATION_IS_OTHER_TEMP(reln)) - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot access temporary tables of other sessions"))); + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot access temporary tables of other sessions"))); /* pass it off to localbuf.c */ return PrefetchLocalBuffer(RelationGetSmgr(reln), forkNum, blockNum); - } else { + } + else + { /* pass it to the shared buffer version */ return PrefetchSharedBuffer(RelationGetSmgr(reln), forkNum, blockNum); } @@ -2868,9 +2986,9 @@ ReadRecentBuffer(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockN Buffer recent_buffer) { BufferDesc *bufHdr; - BufferTag tag; - uint32 buf_state; - bool have_private_ref; + BufferTag tag; + uint32 buf_state; + bool have_private_ref; Assert(BufferIsValid(recent_buffer)); @@ -2878,21 +2996,25 @@ ReadRecentBuffer(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockN ReservePrivateRefCountEntry(); InitBufferTag(&tag, &rlocator, forkNum, blockNum); - if (BufferIsLocal(recent_buffer)) { - int b = -recent_buffer - 1; + if (BufferIsLocal(recent_buffer)) + { + int b = -recent_buffer - 1; bufHdr = GetLocalBufferDescriptor(b); buf_state = pg_atomic_read_u32(&bufHdr->state); /* Is it still valid and holding the right tag? */ - if ((buf_state & BM_VALID) && BufferTagsEqual(&tag, &bufHdr->tag)) { + if ((buf_state & BM_VALID) && BufferTagsEqual(&tag, &bufHdr->tag)) + { PinLocalBuffer(bufHdr, true); pgBufferUsage.local_blks_hit++; return true; } - } else { + } + else + { bufHdr = GetBufferDescriptor(recent_buffer - 1); have_private_ref = GetPrivateRefCount(recent_buffer) > 0; @@ -2906,16 +3028,17 @@ ReadRecentBuffer(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockN else buf_state = LockBufHdr(bufHdr); - if ((buf_state & BM_VALID) && BufferTagsEqual(&tag, &bufHdr->tag)) { + if ((buf_state & BM_VALID) && BufferTagsEqual(&tag, &bufHdr->tag)) + { /* * It's now safe to pin the buffer. We can't pin first and ask * questions later, because it might confuse code paths like * InvalidateBuffer() if we pinned a random non-matching buffer. */ if (have_private_ref) - PinBuffer(bufHdr, NULL); /* bump pin count */ + PinBuffer(bufHdr, NULL); /* bump pin count */ else - PinBuffer_Locked(bufHdr); /* pin for first time */ + PinBuffer_Locked(bufHdr); /* pin for first time */ pgBufferUsage.shared_blks_hit++; @@ -2982,11 +3105,11 @@ ReadBuffer(Relation reln, BlockNumber blockNum) * See buffer/README for details. */ Buffer -ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode, - BufferAccessStrategy strategy) +ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, + ReadBufferMode mode, BufferAccessStrategy strategy) { - bool hit; - Buffer buf; + bool hit; + Buffer buf; /* * Reject attempts to read non-local temporary relations; we would be @@ -2994,8 +3117,9 @@ ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, Read * session's local buffers. */ if (RELATION_IS_OTHER_TEMP(reln)) - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot access temporary tables of other sessions"))); + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot access temporary tables of other sessions"))); #ifdef USE_PGRAC_CLUSTER @@ -3006,12 +3130,15 @@ ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, Read * hint by exact BufferTag match. Profiling-only, GUC-gated inside the * setter, no behavior change. */ - if (unlikely(cluster_xnode_profile_enabled) && blockNum != P_NEW - && !RelationUsesLocalBuffers(reln)) { - BufferTag xp_tag; + if (unlikely(cluster_xnode_profile_enabled) && + blockNum != P_NEW && + !RelationUsesLocalBuffers(reln)) + { + BufferTag xp_tag; InitBufferTag(&xp_tag, &reln->rd_locator, forkNum, blockNum); - cluster_xp_relkind_hint_set(&xp_tag, reln->rd_rel->relkind == RELKIND_INDEX); + cluster_xp_relkind_hint_set(&xp_tag, + reln->rd_rel->relkind == RELKIND_INDEX); } #endif @@ -3020,8 +3147,8 @@ ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, Read * miss. */ pgstat_count_buffer_read(reln); - buf = ReadBuffer_common(RelationGetSmgr(reln), reln->rd_rel->relpersistence, forkNum, blockNum, - mode, strategy, &hit); + buf = ReadBuffer_common(RelationGetSmgr(reln), reln->rd_rel->relpersistence, + forkNum, blockNum, mode, strategy, &hit); if (hit) pgstat_count_buffer_hit(reln); return buf; @@ -3039,28 +3166,33 @@ ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, Read * BackendId). */ Buffer -ReadBufferWithoutRelcache(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockNum, - ReadBufferMode mode, BufferAccessStrategy strategy, bool permanent) +ReadBufferWithoutRelcache(RelFileLocator rlocator, ForkNumber forkNum, + BlockNumber blockNum, ReadBufferMode mode, + BufferAccessStrategy strategy, bool permanent) { - bool hit; + bool hit; SMgrRelation smgr = smgropen(rlocator, InvalidBackendId); - return ReadBuffer_common(smgr, permanent ? RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED, - forkNum, blockNum, mode, strategy, &hit); + return ReadBuffer_common(smgr, permanent ? RELPERSISTENCE_PERMANENT : + RELPERSISTENCE_UNLOGGED, forkNum, blockNum, + mode, strategy, &hit); } /* * Convenience wrapper around ExtendBufferedRelBy() extending by one block. */ Buffer -ExtendBufferedRel(BufferManagerRelation bmr, ForkNumber forkNum, BufferAccessStrategy strategy, +ExtendBufferedRel(BufferManagerRelation bmr, + ForkNumber forkNum, + BufferAccessStrategy strategy, uint32 flags) { - Buffer buf; - uint32 extend_by = 1; + Buffer buf; + uint32 extend_by = 1; - ExtendBufferedRelBy(bmr, forkNum, strategy, flags, extend_by, &buf, &extend_by); + ExtendBufferedRelBy(bmr, forkNum, strategy, flags, extend_by, + &buf, &extend_by); return buf; } @@ -3083,19 +3215,26 @@ ExtendBufferedRel(BufferManagerRelation bmr, ForkNumber forkNum, BufferAccessStr * be empty. */ BlockNumber -ExtendBufferedRelBy(BufferManagerRelation bmr, ForkNumber fork, BufferAccessStrategy strategy, - uint32 flags, uint32 extend_by, Buffer *buffers, uint32 *extended_by) +ExtendBufferedRelBy(BufferManagerRelation bmr, + ForkNumber fork, + BufferAccessStrategy strategy, + uint32 flags, + uint32 extend_by, + Buffer *buffers, + uint32 *extended_by) { Assert((bmr.rel != NULL) != (bmr.smgr != NULL)); Assert(bmr.smgr == NULL || bmr.relpersistence != 0); Assert(extend_by > 0); - if (bmr.smgr == NULL) { + if (bmr.smgr == NULL) + { bmr.smgr = RelationGetSmgr(bmr.rel); bmr.relpersistence = bmr.rel->rd_rel->relpersistence; } - return ExtendBufferedRelCommon(bmr, fork, strategy, flags, extend_by, InvalidBlockNumber, + return ExtendBufferedRelCommon(bmr, fork, strategy, flags, + extend_by, InvalidBlockNumber, buffers, extended_by); } @@ -3108,19 +3247,24 @@ ExtendBufferedRelBy(BufferManagerRelation bmr, ForkNumber fork, BufferAccessStra * crash recovery). */ Buffer -ExtendBufferedRelTo(BufferManagerRelation bmr, ForkNumber fork, BufferAccessStrategy strategy, - uint32 flags, BlockNumber extend_to, ReadBufferMode mode) +ExtendBufferedRelTo(BufferManagerRelation bmr, + ForkNumber fork, + BufferAccessStrategy strategy, + uint32 flags, + BlockNumber extend_to, + ReadBufferMode mode) { BlockNumber current_size; - uint32 extended_by = 0; - Buffer buffer = InvalidBuffer; - Buffer buffers[64]; + uint32 extended_by = 0; + Buffer buffer = InvalidBuffer; + Buffer buffers[64]; Assert((bmr.rel != NULL) != (bmr.smgr != NULL)); Assert(bmr.smgr == NULL || bmr.relpersistence != 0); Assert(extend_to != InvalidBlockNumber && extend_to > 0); - if (bmr.smgr == NULL) { + if (bmr.smgr == NULL) + { bmr.smgr = RelationGetSmgr(bmr.rel); bmr.relpersistence = bmr.rel->rd_rel->relpersistence; } @@ -3130,10 +3274,11 @@ ExtendBufferedRelTo(BufferManagerRelation bmr, ForkNumber fork, BufferAccessStra * smgr_cached_nblocks[fork] is positive then it must exist, no need for * an smgrexists call. */ - if ((flags & EB_CREATE_FORK_IF_NEEDED) - && (bmr.smgr->smgr_cached_nblocks[fork] == 0 - || bmr.smgr->smgr_cached_nblocks[fork] == InvalidBlockNumber) - && !smgrexists(bmr.smgr, fork)) { + if ((flags & EB_CREATE_FORK_IF_NEEDED) && + (bmr.smgr->smgr_cached_nblocks[fork] == 0 || + bmr.smgr->smgr_cached_nblocks[fork] == InvalidBlockNumber) && + !smgrexists(bmr.smgr, fork)) + { LockRelationForExtension(bmr.rel, ExclusiveLock); /* could have been closed while waiting for lock */ @@ -3169,20 +3314,23 @@ ExtendBufferedRelTo(BufferManagerRelation bmr, ForkNumber fork, BufferAccessStra if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) flags |= EB_LOCK_TARGET; - while (current_size < extend_to) { - uint32 num_pages = lengthof(buffers); + while (current_size < extend_to) + { + uint32 num_pages = lengthof(buffers); BlockNumber first_block; - if ((uint64)current_size + num_pages > extend_to) + if ((uint64) current_size + num_pages > extend_to) num_pages = extend_to - current_size; - first_block = ExtendBufferedRelCommon(bmr, fork, strategy, flags, num_pages, extend_to, + first_block = ExtendBufferedRelCommon(bmr, fork, strategy, flags, + num_pages, extend_to, buffers, &extended_by); current_size = first_block + extended_by; Assert(num_pages != 0 || current_size >= extend_to); - for (int i = 0; i < extended_by; i++) { + for (int i = 0; i < extended_by; i++) + { if (first_block + i != extend_to - 1) ReleaseBuffer(buffers[i]); else @@ -3196,12 +3344,14 @@ ExtendBufferedRelTo(BufferManagerRelation bmr, ForkNumber fork, BufferAccessStra * * XXX: Should we control this via a flag? */ - if (buffer == InvalidBuffer) { - bool hit; + if (buffer == InvalidBuffer) + { + bool hit; Assert(extended_by == 0); - buffer = ReadBuffer_common(bmr.smgr, bmr.relpersistence, fork, extend_to - 1, mode, - strategy, &hit); + buffer = ReadBuffer_common(bmr.smgr, bmr.relpersistence, + fork, extend_to - 1, mode, strategy, + &hit); } return buffer; @@ -3213,15 +3363,16 @@ ExtendBufferedRelTo(BufferManagerRelation bmr, ForkNumber fork, BufferAccessStra * *hit is set to true if the request was satisfied from shared buffer cache. */ static Buffer -ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, - ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit) +ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, + BlockNumber blockNum, ReadBufferMode mode, + BufferAccessStrategy strategy, bool *hit) { BufferDesc *bufHdr; - Block bufBlock; - bool found; - IOContext io_context; - IOObject io_object; - bool isLocalBuf = SmgrIsTemp(smgr); + Block bufBlock; + bool found; + IOContext io_context; + IOObject io_object; + bool isLocalBuf = SmgrIsTemp(smgr); #ifdef USE_PGRAC_CLUSTER ClusterPcmDirectInitProof direct_init_proof; #endif @@ -3233,8 +3384,9 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, Bl * instead, as acquiring the extension lock inside ExtendBufferedRel() * scales a lot better. */ - if (unlikely(blockNum == P_NEW)) { - uint32 flags = EB_SKIP_EXTENSION_LOCK; + if (unlikely(blockNum == P_NEW)) + { + uint32 flags = EB_SKIP_EXTENSION_LOCK; /* * Since no-one else can be looking at the page contents yet, there is @@ -3244,17 +3396,21 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, Bl if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) flags |= EB_LOCK_FIRST; - return ExtendBufferedRel(BMR_SMGR(smgr, relpersistence), forkNum, strategy, flags); + return ExtendBufferedRel(BMR_SMGR(smgr, relpersistence), + forkNum, strategy, flags); } /* Make sure we will have room to remember the buffer pin */ ResourceOwnerEnlargeBuffers(CurrentResourceOwner); - TRACE_POSTGRESQL_BUFFER_READ_START( - forkNum, blockNum, smgr->smgr_rlocator.locator.spcOid, smgr->smgr_rlocator.locator.dbOid, - smgr->smgr_rlocator.locator.relNumber, smgr->smgr_rlocator.backend); + TRACE_POSTGRESQL_BUFFER_READ_START(forkNum, blockNum, + smgr->smgr_rlocator.locator.spcOid, + smgr->smgr_rlocator.locator.dbOid, + smgr->smgr_rlocator.locator.relNumber, + smgr->smgr_rlocator.backend); - if (isLocalBuf) { + if (isLocalBuf) + { /* * We do not use a BufferAccessStrategy for I/O of temporary tables. * However, in some cases, the "strategy" may not be NULL, so we can't @@ -3266,19 +3422,24 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, Bl bufHdr = LocalBufferAlloc(smgr, forkNum, blockNum, &found); if (found) pgBufferUsage.local_blks_hit++; - else if (mode == RBM_NORMAL || mode == RBM_NORMAL_NO_LOG || mode == RBM_ZERO_ON_ERROR) + else if (mode == RBM_NORMAL || mode == RBM_NORMAL_NO_LOG || + mode == RBM_ZERO_ON_ERROR) pgBufferUsage.local_blks_read++; - } else { + } + else + { /* * lookup the buffer. IO_IN_PROGRESS is set if the requested block is * not currently in memory. */ io_context = IOContextForStrategy(strategy); io_object = IOOBJECT_RELATION; - bufHdr = BufferAlloc(smgr, relpersistence, forkNum, blockNum, strategy, &found, io_context); + bufHdr = BufferAlloc(smgr, relpersistence, forkNum, blockNum, + strategy, &found, io_context); if (found) pgBufferUsage.shared_blks_hit++; - else if (mode == RBM_NORMAL || mode == RBM_NORMAL_NO_LOG || mode == RBM_ZERO_ON_ERROR) + else if (mode == RBM_NORMAL || mode == RBM_NORMAL_NO_LOG || + mode == RBM_ZERO_ON_ERROR) pgBufferUsage.shared_blks_read++; #ifdef USE_PGRAC_CLUSTER @@ -3290,8 +3451,10 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, Bl * transfer -- the cache-locality signal the A-L9 observability leg * watches. */ - if (cluster_shared_catalog - && smgr->smgr_rlocator.locator.relNumber < (RelFileNumber)FirstNormalObjectId) { + if (cluster_shared_catalog && + smgr->smgr_rlocator.locator.relNumber < + (RelFileNumber) FirstNormalObjectId) + { if (found) cluster_catalog_stats_buf_hit_inc(); else @@ -3303,7 +3466,8 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, Bl /* At this point we do NOT hold any locks. */ /* if it was already in the buffer pool, we're done */ - if (found) { + if (found) + { /* Just need to update stats before we exit */ *hit = true; VacuumPageHit++; @@ -3312,16 +3476,19 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, Bl if (VacuumCostActive) VacuumCostBalance += VacuumCostPageHit; - TRACE_POSTGRESQL_BUFFER_READ_DONE(forkNum, blockNum, smgr->smgr_rlocator.locator.spcOid, + TRACE_POSTGRESQL_BUFFER_READ_DONE(forkNum, blockNum, + smgr->smgr_rlocator.locator.spcOid, smgr->smgr_rlocator.locator.dbOid, smgr->smgr_rlocator.locator.relNumber, - smgr->smgr_rlocator.backend, found); + smgr->smgr_rlocator.backend, + found); /* * In RBM_ZERO_AND_LOCK mode the caller expects the page to be locked * on return. */ - if (!isLocalBuf) { + if (!isLocalBuf) + { if (mode == RBM_ZERO_AND_LOCK) LockBuffer(BufferDescriptorGetBuffer(bufHdr), BUFFER_LOCK_EXCLUSIVE); else if (mode == RBM_ZERO_AND_CLEANUP_LOCK) @@ -3336,7 +3503,7 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, Bl * page but its contents are not yet valid. IO_IN_PROGRESS is set for it, * if it's a shared buffer. */ - Assert(!(pg_atomic_read_u32(&bufHdr->state) & BM_VALID)); /* spinlock not needed */ + Assert(!(pg_atomic_read_u32(&bufHdr->state) & BM_VALID)); /* spinlock not needed */ bufBlock = isLocalBuf ? LocalBufHdrGetBlock(bufHdr) : BufHdrGetBlock(bufHdr); @@ -3344,24 +3511,28 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, Bl * Read in the page, unless the caller intends to overwrite it and just * wants us to allocate a buffer. */ - if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) { - MemSet((char *)bufBlock, 0, BLCKSZ); + if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) + { + MemSet((char *) bufBlock, 0, BLCKSZ); #ifdef USE_PGRAC_CLUSTER if (!isLocalBuf) - cluster_bufmgr_pcm_arm_direct_init(bufHdr, CLUSTER_PCM_DIRECT_INIT_READ_MISS, - &direct_init_proof); + cluster_bufmgr_pcm_arm_direct_init( + bufHdr, CLUSTER_PCM_DIRECT_INIT_READ_MISS, &direct_init_proof); #endif - } else { - instr_time io_start = pgstat_prepare_io_time(); - bool verified; + } + else + { + instr_time io_start = pgstat_prepare_io_time(); + bool verified; smgrread(smgr, forkNum, blockNum, bufBlock); - pgstat_count_io_op_time(io_object, io_context, IOOP_READ, io_start, 1); + pgstat_count_io_op_time(io_object, io_context, + IOOP_READ, io_start, 1); /* check for garbage data */ - verified - = PageIsVerifiedExtended((Page)bufBlock, blockNum, PIV_LOG_WARNING | PIV_REPORT_STAT); + verified = PageIsVerifiedExtended((Page) bufBlock, blockNum, + PIV_LOG_WARNING | PIV_REPORT_STAT); #ifdef USE_PGRAC_CLUSTER @@ -3379,27 +3550,34 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, Bl * ignore_checksum_failure's "return the page as-is" remains the * fallback (verified stays true). */ - if (!verified) { + if (!verified) + { if (relpersistence == RELPERSISTENCE_PERMANENT && !isLocalBuf - && cluster_block_recovery_on_read(smgr, forkNum, blockNum, (char *)bufBlock)) + && cluster_block_recovery_on_read(smgr, forkNum, blockNum, (char *) bufBlock)) verified = true; - } else if (relpersistence == RELPERSISTENCE_PERMANENT && !isLocalBuf - && ignore_checksum_failure && cluster_online_block_recovery - && cluster_block_recovery_checksum_mismatch((char *)bufBlock, blockNum)) { - (void)cluster_block_recovery_on_read(smgr, forkNum, blockNum, (char *)bufBlock); + } + else if (relpersistence == RELPERSISTENCE_PERMANENT && !isLocalBuf + && ignore_checksum_failure && cluster_online_block_recovery + && cluster_block_recovery_checksum_mismatch((char *) bufBlock, blockNum)) + { + (void) cluster_block_recovery_on_read(smgr, forkNum, blockNum, (char *) bufBlock); } #endif - if (!verified) { - if (mode == RBM_ZERO_ON_ERROR || zero_damaged_pages) { + if (!verified) + { + if (mode == RBM_ZERO_ON_ERROR || zero_damaged_pages) + { ereport(WARNING, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid page in block %u of relation %s; zeroing out page", - blockNum, relpath(smgr->smgr_rlocator, forkNum)))); - MemSet((char *)bufBlock, 0, BLCKSZ); + blockNum, + relpath(smgr->smgr_rlocator, forkNum)))); + MemSet((char *) bufBlock, 0, BLCKSZ); } #ifdef USE_PGRAC_CLUSTER - else if (cluster_online_block_recovery) { + else if (cluster_online_block_recovery) + { /* * Online recovery on but the block could not be rebuilt. * PANIC escalation (operator opt-in) is restricted to the @@ -3407,14 +3585,14 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, Bl * reconstructible from scratch, so an unrebuildable one * should fail the query (ERROR), not crash the instance. */ - int elevel = (cluster_block_recovery_on_unrecoverable == CLUSTER_BLKREC_ACTION_PANIC - && forkNum == MAIN_FORKNUM) - ? PANIC - : ERROR; + int elevel = (cluster_block_recovery_on_unrecoverable == CLUSTER_BLKREC_ACTION_PANIC + && forkNum == MAIN_FORKNUM) + ? PANIC : ERROR; ereport(elevel, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("invalid page in block %u of relation %s", blockNum, + errmsg("invalid page in block %u of relation %s", + blockNum, relpath(smgr->smgr_rlocator, forkNum)), errhint("online block recovery could not rebuild this block from WAL " "(no full-page-image base in retained WAL, an unsupported " @@ -3422,9 +3600,11 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, Bl } #endif else - ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("invalid page in block %u of relation %s", blockNum, - relpath(smgr->smgr_rlocator, forkNum)))); + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("invalid page in block %u of relation %s", + blockNum, + relpath(smgr->smgr_rlocator, forkNum)))); } } @@ -3439,21 +3619,26 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, Bl * that we cannot use LockBuffer() or LockBufferForCleanup() here, because * they assert that the buffer is already valid.) */ - if ((mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) && !isLocalBuf) { + if ((mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) && + !isLocalBuf) + { #ifdef USE_PGRAC_CLUSTER - cluster_bufmgr_pcm_gate_direct_init(bufHdr, CLUSTER_PCM_DIRECT_INIT_READ_MISS, - &direct_init_proof); + cluster_bufmgr_pcm_gate_direct_init( + bufHdr, CLUSTER_PCM_DIRECT_INIT_READ_MISS, &direct_init_proof); #endif LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_EXCLUSIVE); } - if (isLocalBuf) { + if (isLocalBuf) + { /* Only need to adjust flags */ - uint32 buf_state = pg_atomic_read_u32(&bufHdr->state); + uint32 buf_state = pg_atomic_read_u32(&bufHdr->state); buf_state |= BM_VALID; pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state); - } else { + } + else + { /* Set BM_VALID, terminate IO, and wake up any waiters */ TerminateBufferIO(bufHdr, false, BM_VALID); } @@ -3462,9 +3647,12 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, Bl if (VacuumCostActive) VacuumCostBalance += VacuumCostPageMiss; - TRACE_POSTGRESQL_BUFFER_READ_DONE( - forkNum, blockNum, smgr->smgr_rlocator.locator.spcOid, smgr->smgr_rlocator.locator.dbOid, - smgr->smgr_rlocator.locator.relNumber, smgr->smgr_rlocator.backend, found); + TRACE_POSTGRESQL_BUFFER_READ_DONE(forkNum, blockNum, + smgr->smgr_rlocator.locator.spcOid, + smgr->smgr_rlocator.locator.dbOid, + smgr->smgr_rlocator.locator.relNumber, + smgr->smgr_rlocator.backend, + found); return BufferDescriptorGetBuffer(bufHdr); } @@ -3493,16 +3681,18 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, Bl * No locks are held either at entry or exit. */ static BufferDesc * -BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, - BufferAccessStrategy strategy, bool *foundPtr, IOContext io_context) -{ - BufferTag newTag; /* identity of requested block */ - uint32 newHash; /* hash value for newTag */ - LWLock *newPartitionLock; /* buffer partition lock for it */ - int existing_buf_id; - Buffer victim_buffer; +BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, + BlockNumber blockNum, + BufferAccessStrategy strategy, + bool *foundPtr, IOContext io_context) +{ + BufferTag newTag; /* identity of requested block */ + uint32 newHash; /* hash value for newTag */ + LWLock *newPartitionLock; /* buffer partition lock for it */ + int existing_buf_id; + Buffer victim_buffer; BufferDesc *victim_buf_hdr; - uint32 victim_buf_state; + uint32 victim_buf_state; /* create a tag so we can lookup the buffer */ InitBufferTag(&newTag, &smgr->smgr_rlocator.locator, forkNum, blockNum); @@ -3514,9 +3704,10 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNum /* see if the block is in the buffer pool already */ LWLockAcquire(newPartitionLock, LW_SHARED); existing_buf_id = BufTableLookup(&newTag, newHash); - if (existing_buf_id >= 0) { + if (existing_buf_id >= 0) + { BufferDesc *buf; - bool valid; + bool valid; /* * Found it. Now, pin the buffer so no one can steal it from the @@ -3532,7 +3723,8 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNum *foundPtr = true; - if (!valid) { + if (!valid) + { /* * We can only get here if (a) someone else is still reading in * the page, or (b) a previous read attempt failed. We have to @@ -3540,7 +3732,8 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNum * own read attempt if the page is still not BM_VALID. * StartBufferIO does it all. */ - if (StartBufferIO(buf, true)) { + if (StartBufferIO(buf, true)) + { /* * If we get here, previous attempts to read the buffer must * have failed ... but we shall bravely try again. @@ -3573,9 +3766,10 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNum */ LWLockAcquire(newPartitionLock, LW_EXCLUSIVE); existing_buf_id = BufTableInsert(&newTag, newHash, victim_buf_hdr->buf_id); - if (existing_buf_id >= 0) { + if (existing_buf_id >= 0) + { BufferDesc *existing_buf_hdr; - bool valid; + bool valid; /* * Got a collision. Someone has already done what we were about to do. @@ -3607,7 +3801,8 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNum *foundPtr = true; - if (!valid) { + if (!valid) + { /* * We can only get here if (a) someone else is still reading in * the page, or (b) a previous read attempt failed. We have to @@ -3615,7 +3810,8 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNum * own read attempt if the page is still not BM_VALID. * StartBufferIO does it all. */ - if (StartBufferIO(existing_buf_hdr, true)) { + if (StartBufferIO(existing_buf_hdr, true)) + { /* * If we get here, previous attempts to read the buffer must * have failed ... but we shall bravely try again. @@ -3698,10 +3894,10 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNum static void InvalidateBuffer(BufferDesc *buf) { - BufferTag oldTag; - uint32 oldHash; /* hash value for oldTag */ - LWLock *oldPartitionLock; /* buffer partition lock for it */ - uint32 buf_state; + BufferTag oldTag; + uint32 oldHash; /* hash value for oldTag */ + LWLock *oldPartitionLock; /* buffer partition lock for it */ + uint32 buf_state; /* Save the original buffer tag before dropping the spinlock */ oldTag = buf->tag; @@ -3730,7 +3926,8 @@ InvalidateBuffer(BufferDesc *buf) buf_state = LockBufHdr(buf); /* If it's changed while we were waiting for lock, do nothing */ - if (!BufferTagsEqual(&buf->tag, &oldTag)) { + if (!BufferTagsEqual(&buf->tag, &oldTag)) + { UnlockBufHdr(buf, buf_state); LWLockRelease(oldPartitionLock); return; @@ -3745,7 +3942,8 @@ InvalidateBuffer(BufferDesc *buf) * yet done StartBufferIO, WaitIO will fall through and we'll effectively * be busy-looping here.) */ - if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) { + if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) + { UnlockBufHdr(buf, buf_state); LWLockRelease(oldPartitionLock); /* safety check: should definitely not be our *own* pin */ @@ -3755,7 +3953,8 @@ InvalidateBuffer(BufferDesc *buf) goto retry; } - if (!InvalidateBufferCommitLocked(buf, &oldTag, oldHash, oldPartitionLock, buf_state)) { + if (!InvalidateBufferCommitLocked(buf, &oldTag, oldHash, oldPartitionLock, buf_state)) + { pg_usleep(1000L); goto retry; } @@ -3772,15 +3971,15 @@ InvalidateBuffer(BufferDesc *buf) */ static bool InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, - LWLock *oldPartitionLock, uint32 buf_state) + LWLock *oldPartitionLock, uint32 buf_state) { - uint8 old_pcm_mode = 0; - bool release_pcm_holder = false; + uint8 old_pcm_mode = 0; + bool release_pcm_holder = false; #ifdef USE_PGRAC_CLUSTER ClusterPcmOwnEvictionCapture eviction_capture; ClusterPcmOwnResult eviction_result; - uint32 observed_flags = 0; - uint64 observed_generation = 0; + uint32 observed_flags = 0; + uint64 observed_generation = 0; /* * D5a: descriptor reuse is an exact ownership-tuple commit. Refuse to @@ -3790,22 +3989,24 @@ InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, * succeed. */ cluster_pcm_own_eviction_capture_locked(buf, &eviction_capture); - eviction_result = cluster_pcm_own_eviction_commit_locked(buf, &eviction_capture, - &observed_generation, &observed_flags); - if (eviction_result != CLUSTER_PCM_OWN_OK) { + eviction_result = cluster_pcm_own_eviction_commit_locked( + buf, &eviction_capture, &observed_generation, &observed_flags); + if (eviction_result != CLUSTER_PCM_OWN_OK) + { UnlockBufHdr(buf, buf_state); LWLockRelease(oldPartitionLock); if (eviction_result == CLUSTER_PCM_OWN_BUSY || eviction_result == CLUSTER_PCM_OWN_STALE) return false; - cluster_pcm_own_report_bump_failure( - buf, eviction_result, - observed_generation != 0 ? observed_generation : eviction_capture.generation, - observed_flags != 0 ? observed_flags : eviction_capture.flags, "buffer eviction"); + cluster_pcm_own_report_bump_failure(buf, eviction_result, + observed_generation != 0 ? observed_generation + : eviction_capture.generation, + observed_flags != 0 ? observed_flags : eviction_capture.flags, + "buffer eviction"); } old_pcm_mode = eviction_capture.pcm_state; - release_pcm_holder - = cluster_pcm_is_active() && cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(oldTag)) - && (old_pcm_mode == (uint8)PCM_LOCK_MODE_S || old_pcm_mode == (uint8)PCM_LOCK_MODE_X); + release_pcm_holder = cluster_pcm_is_active() + && cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(oldTag)) + && (old_pcm_mode == (uint8) PCM_LOCK_MODE_S || old_pcm_mode == (uint8) PCM_LOCK_MODE_X); #endif InvalidateBufferCommitTailLocked(buf, oldTag, oldHash, oldPartitionLock, buf_state, @@ -3823,10 +4024,10 @@ InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, */ static void InvalidateBufferCommitTailLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, - LWLock *oldPartitionLock, uint32 buf_state, uint8 old_pcm_mode, - bool release_pcm_holder) + LWLock *oldPartitionLock, uint32 buf_state, + uint8 old_pcm_mode, bool release_pcm_holder) { - uint32 oldFlags; + uint32 oldFlags; /* * Clear out the buffer's tag and flags. We must do this to ensure that @@ -3843,7 +4044,7 @@ InvalidateBufferCommitTailLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldH * inherit a stale BUF_TYPE_PI (the PI shape must stay reachable ONLY * through cluster_bufmgr_convert_to_pi_locked, or the D-h3 shadow stamp * could be paired with another residency's bytes). */ - buf->buffer_type = (uint8)BUF_TYPE_CURRENT; + buf->buffer_type = (uint8) BUF_TYPE_CURRENT; #endif UnlockBufHdr(buf, buf_state); @@ -3864,17 +4065,18 @@ InvalidateBufferCommitTailLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldH * immutable capture avoids the old temporary tag restoration race. On a * throwing release the buffer is absent from the mapping table but is not * yet reusable, which is the fail-closed state. */ - if (release_pcm_holder) { + if (release_pcm_holder) + { PG_TRY(); { - cluster_pcm_lock_release_saved_tag_for_eviction(*oldTag, (PcmLockMode)old_pcm_mode); + cluster_pcm_lock_release_saved_tag_for_eviction(*oldTag, (PcmLockMode) old_pcm_mode); } PG_CATCH(); { elog(LOG, "cluster PCM saved-tag eviction release failed for buffer %d mode=%d; " "old mapping is removed and descriptor was not returned to the freelist", - buf->buf_id, (int)old_pcm_mode); + buf->buf_id, (int) old_pcm_mode); PG_RE_THROW(); } PG_END_TRY(); @@ -3905,10 +4107,10 @@ InvalidateBufferCommitTailLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldH static bool InvalidateBufferTry(BufferDesc *buf) { - BufferTag oldTag; - uint32 oldHash; /* hash value for oldTag */ - LWLock *oldPartitionLock; /* buffer partition lock for it */ - uint32 buf_state; + BufferTag oldTag; + uint32 oldHash; /* hash value for oldTag */ + LWLock *oldPartitionLock; /* buffer partition lock for it */ + uint32 buf_state; /* Save the original buffer tag before dropping the spinlock */ oldTag = buf->tag; @@ -3932,7 +4134,8 @@ InvalidateBufferTry(BufferDesc *buf) * deterministically. Gated to GCS drops only — plain evictions must * not stall. No lock is held across the sleep. */ - if (cluster_bufmgr_in_gcs_drop && BufTagGetForkNum(&oldTag) == MAIN_FORKNUM) + if (cluster_bufmgr_in_gcs_drop + && BufTagGetForkNum(&oldTag) == MAIN_FORKNUM) CLUSTER_INJECTION_POINT("cluster-pcm-drop-prepin-window"); #endif @@ -3945,19 +4148,21 @@ InvalidateBufferTry(BufferDesc *buf) buf_state = LockBufHdr(buf); /* If it's changed while we were waiting for lock, do nothing */ - if (!BufferTagsEqual(&buf->tag, &oldTag)) { + if (!BufferTagsEqual(&buf->tag, &oldTag)) + { UnlockBufHdr(buf, buf_state); LWLockRelease(oldPartitionLock); return true; } - if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) { + if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) + { UnlockBufHdr(buf, buf_state); LWLockRelease(oldPartitionLock); /* safety check: should definitely not be our *own* pin */ if (GetPrivateRefCount(BufferDescriptorGetBuffer(buf)) > 0) elog(ERROR, "buffer is pinned in InvalidateBufferTry"); - return false; /* foreign pin — caller parks / fail-closes */ + return false; /* foreign pin — caller parks / fail-closes */ } return InvalidateBufferCommitLocked(buf, &oldTag, oldHash, oldPartitionLock, buf_state); @@ -3975,10 +4180,10 @@ InvalidateBufferTry(BufferDesc *buf) static bool InvalidateVictimBuffer(BufferDesc *buf_hdr) { - uint32 buf_state; - uint32 hash; - LWLock *partition_lock; - BufferTag tag; + uint32 buf_state; + uint32 hash; + LWLock *partition_lock; + BufferTag tag; Assert(GetPrivateRefCount(BufferDescriptorGetBuffer(buf_hdr)) == 1); @@ -4005,7 +4210,8 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) /* PCM-X retained images deliberately keep BM_VALID and may still have this * victim pin. The live REVOKING token, not passive refcount, is the reuse * authority; never let the clock-sweep shortcut bypass D5a. */ - if (cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(buf_hdr, buf_state)) { + if (cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(buf_hdr, buf_state)) + { UnlockBufHdr(buf_hdr, buf_state); LWLockRelease(partition_lock); return false; @@ -4016,7 +4222,8 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) * If somebody else pinned the buffer since, or even worse, dirtied it, * give up on this buffer: It's clearly in use. */ - if (BUF_STATE_GET_REFCOUNT(buf_state) != 1 || (buf_state & BM_DIRTY)) { + if (BUF_STATE_GET_REFCOUNT(buf_state) != 1 || (buf_state & BM_DIRTY)) + { Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0); UnlockBufHdr(buf_hdr, buf_state); @@ -4040,7 +4247,7 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) * PGRAC: spec-6.12h D-h3a — victim reuse retags this buffer; reset the * cluster copy label under the header lock (same stale-BUF_TYPE_PI * containment as InvalidateBuffer above). */ - buf_hdr->buffer_type = (uint8)BUF_TYPE_CURRENT; + buf_hdr->buffer_type = (uint8) BUF_TYPE_CURRENT; #endif UnlockBufHdr(buf_hdr, buf_state); @@ -4054,11 +4261,13 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) * the cache-residency bit + propagate master_holder lifecycle (HC110) * before BufTableDelete completes the eviction. */ - if (cluster_pcm_is_active() && cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(&tag)) - && buf_hdr->pcm_state != (uint8)PCM_STATE_N) { - PcmLockMode old_mode = (PcmLockMode)buf_hdr->pcm_state; + if (cluster_pcm_is_active() + && cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(&tag)) + && buf_hdr->pcm_state != (uint8) PCM_STATE_N) + { + PcmLockMode old_mode = (PcmLockMode) buf_hdr->pcm_state; - buf_hdr->tag = tag; /* restore for release helper (cleared above) */ + buf_hdr->tag = tag; /* restore for release helper (cleared above) */ cluster_pcm_lock_release_buffer_for_eviction(buf_hdr, old_mode); ClearBufferTag(&buf_hdr->tag); @@ -4066,7 +4275,7 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) * PGRAC ownership-gen: coherent N-flip + generation bump (the header * spinlock was dropped above at UnlockBufHdr) so a buf_id reuse after * this victim eviction cannot alias a stale captured generation. */ - cluster_pcm_own_transition(buf_hdr, (uint8)PCM_STATE_N, 0, + cluster_pcm_own_transition(buf_hdr, (uint8) PCM_STATE_N, 0, PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING); } #endif @@ -4087,9 +4296,9 @@ static Buffer GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) { BufferDesc *buf_hdr; - Buffer buf; - uint32 buf_state; - bool from_ring; + Buffer buf; + uint32 buf_state; + bool from_ring; /* * Ensure, while the spinlock's not yet held, that there's a free refcount @@ -4125,8 +4334,9 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) * our share-lock won't prevent hint-bit updates). We will recheck the * dirty bit after re-locking the buffer header. */ - if (buf_state & BM_DIRTY) { - LWLock *content_lock; + if (buf_state & BM_DIRTY) + { + LWLock *content_lock; Assert(buf_state & BM_TAG_VALID); Assert(buf_state & BM_VALID); @@ -4146,7 +4356,8 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) * from StrategyGetBuffer.) */ content_lock = BufferDescriptorGetContentLock(buf_hdr); - if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) { + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) + { /* * Someone else has locked the buffer, so give it up and loop back * to get another one. @@ -4162,15 +4373,18 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) * lock to inspect the page LSN, so this can't be done inside * StrategyGetBuffer. */ - if (strategy != NULL) { - XLogRecPtr lsn; + if (strategy != NULL) + { + XLogRecPtr lsn; /* Read the LSN while holding buffer header lock */ buf_state = LockBufHdr(buf_hdr); lsn = BufferGetLSN(buf_hdr); UnlockBufHdr(buf_hdr, buf_state); - if (XLogNeedsFlush(lsn) && StrategyRejectBuffer(strategy, buf_hdr, from_ring)) { + if (XLogNeedsFlush(lsn) + && StrategyRejectBuffer(strategy, buf_hdr, from_ring)) + { LWLockRelease(content_lock); UnpinBuffer(buf_hdr); goto again; @@ -4181,11 +4395,13 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) FlushBuffer(buf_hdr, NULL, IOOBJECT_RELATION, io_context); LWLockRelease(content_lock); - ScheduleBufferTagForWriteback(&BackendWritebackContext, io_context, &buf_hdr->tag); + ScheduleBufferTagForWriteback(&BackendWritebackContext, io_context, + &buf_hdr->tag); } - if (buf_state & BM_VALID) { + if (buf_state & BM_VALID) + { /* * When a BufferAccessStrategy is in use, blocks evicted from shared * buffers are counted as IOOP_EVICT in the corresponding context @@ -4202,7 +4418,8 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) * we may have been forced to release the buffer due to concurrent * pinners or erroring out. */ - pgstat_count_io_op(IOOBJECT_RELATION, io_context, from_ring ? IOOP_REUSE : IOOP_EVICT); + pgstat_count_io_op(IOOBJECT_RELATION, io_context, + from_ring ? IOOP_REUSE : IOOP_EVICT); } /* @@ -4210,7 +4427,8 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) * can fail because another backend could have pinned or dirtied the * buffer. */ - if ((buf_state & BM_TAG_VALID) && !InvalidateVictimBuffer(buf_hdr)) { + if ((buf_state & BM_TAG_VALID) && !InvalidateVictimBuffer(buf_hdr)) + { UnpinBuffer(buf_hdr); goto again; } @@ -4243,8 +4461,8 @@ GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context) static void LimitAdditionalPins(uint32 *additional_pins) { - uint32 max_backends; - int max_proportional_pins; + uint32 max_backends; + int max_proportional_pins; if (*additional_pins <= 1) return; @@ -4272,28 +4490,41 @@ LimitAdditionalPins(uint32 *additional_pins) * avoid duplicating the tracing and relpersistence related logic. */ static BlockNumber -ExtendBufferedRelCommon(BufferManagerRelation bmr, ForkNumber fork, BufferAccessStrategy strategy, - uint32 flags, uint32 extend_by, BlockNumber extend_upto, Buffer *buffers, +ExtendBufferedRelCommon(BufferManagerRelation bmr, + ForkNumber fork, + BufferAccessStrategy strategy, + uint32 flags, + uint32 extend_by, + BlockNumber extend_upto, + Buffer *buffers, uint32 *extended_by) { BlockNumber first_block; - TRACE_POSTGRESQL_BUFFER_EXTEND_START( - fork, bmr.smgr->smgr_rlocator.locator.spcOid, bmr.smgr->smgr_rlocator.locator.dbOid, - bmr.smgr->smgr_rlocator.locator.relNumber, bmr.smgr->smgr_rlocator.backend, extend_by); + TRACE_POSTGRESQL_BUFFER_EXTEND_START(fork, + bmr.smgr->smgr_rlocator.locator.spcOid, + bmr.smgr->smgr_rlocator.locator.dbOid, + bmr.smgr->smgr_rlocator.locator.relNumber, + bmr.smgr->smgr_rlocator.backend, + extend_by); if (bmr.relpersistence == RELPERSISTENCE_TEMP) - first_block - = ExtendBufferedRelLocal(bmr, fork, flags, extend_by, extend_upto, buffers, &extend_by); + first_block = ExtendBufferedRelLocal(bmr, fork, flags, + extend_by, extend_upto, + buffers, &extend_by); else - first_block = ExtendBufferedRelShared(bmr, fork, strategy, flags, extend_by, extend_upto, + first_block = ExtendBufferedRelShared(bmr, fork, strategy, flags, + extend_by, extend_upto, buffers, &extend_by); *extended_by = extend_by; - TRACE_POSTGRESQL_BUFFER_EXTEND_DONE(fork, bmr.smgr->smgr_rlocator.locator.spcOid, + TRACE_POSTGRESQL_BUFFER_EXTEND_DONE(fork, + bmr.smgr->smgr_rlocator.locator.spcOid, bmr.smgr->smgr_rlocator.locator.dbOid, bmr.smgr->smgr_rlocator.locator.relNumber, - bmr.smgr->smgr_rlocator.backend, *extended_by, first_block); + bmr.smgr->smgr_rlocator.backend, + *extended_by, + first_block); return first_block; } @@ -4303,13 +4534,18 @@ ExtendBufferedRelCommon(BufferManagerRelation bmr, ForkNumber fork, BufferAccess * shared buffers. */ static BlockNumber -ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccessStrategy strategy, - uint32 flags, uint32 extend_by, BlockNumber extend_upto, Buffer *buffers, +ExtendBufferedRelShared(BufferManagerRelation bmr, + ForkNumber fork, + BufferAccessStrategy strategy, + uint32 flags, + uint32 extend_by, + BlockNumber extend_upto, + Buffer *buffers, uint32 *extended_by) { BlockNumber first_block; - IOContext io_context = IOContextForStrategy(strategy); - instr_time io_start; + IOContext io_context = IOContextForStrategy(strategy); + instr_time io_start; #ifdef USE_PGRAC_CLUSTER /*---------- @@ -4341,13 +4577,14 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess *---------- */ ClusterHwClass hwc = CLUSTER_HW_NATIVE_LOCAL; - HwLock hwlk; + HwLock hwlk; ClusterResId hw_resid; - uint32 hw_lease_tail = 0; /* spec-6.12d: parked grant tail */ + uint32 hw_lease_tail = 0; /* spec-6.12d: parked grant tail */ if (cluster_relation_extend_lock_enabled && bmr.rel != NULL && cluster_node_id >= 0 && fork == MAIN_FORKNUM && !RecoveryInProgress() - && bmr.rel->rd_rel->relpersistence != RELPERSISTENCE_TEMP) { + && bmr.rel->rd_rel->relpersistence != RELPERSISTENCE_TEMP) + { /* * Decide whether to engage the cross-node authority from runtime * liveness, not the static configured node count (spec-5.7 §3.1d, @@ -4358,13 +4595,14 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess ClusterExtendEngage engage = cluster_extend_liveness_engage(true); if (engage == CLUSTER_EXTEND_ENGAGE_FAIL_CLOSED) - ereport(ERROR, (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), - errmsg("could not acquire the cluster relation-extend lock for \"%s\"", - RelationGetRelationName(bmr.rel)), - errdetail("The cluster coordination substrate is not ready and an " - "alive peer could not be ruled out."))); + ereport(ERROR, + (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), + errmsg("could not acquire the cluster relation-extend lock for \"%s\"", + RelationGetRelationName(bmr.rel)), + errdetail("The cluster coordination substrate is not ready and an alive peer could not be ruled out."))); - if (engage == CLUSTER_EXTEND_ENGAGE_COORDINATE) { + if (engage == CLUSTER_EXTEND_ENGAGE_COORDINATE) + { /* * A peer is alive and the substrate is ready: a permanent * relation is GLOBALIZE (authority-owned from block 0); an @@ -4373,12 +4611,11 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess */ hwc = cluster_hw_classify_persistence(bmr.rel->rd_rel->relpersistence, true); if (hwc == CLUSTER_HW_FAIL_CLOSED) - ereport(ERROR, (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), - errmsg("unlogged relation \"%s\" cannot be safely extended in a " - "multi-node cluster", - RelationGetRelationName(bmr.rel)), - errhint("Unlogged relations have no WAL authority to coordinate " - "cross-node extension."))); + ereport(ERROR, + (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), + errmsg("unlogged relation \"%s\" cannot be safely extended in a multi-node cluster", + RelationGetRelationName(bmr.rel)), + errhint("Unlogged relations have no WAL authority to coordinate cross-node extension."))); } /* @@ -4400,14 +4637,15 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess * These pages are pinned by us and not valid. While we hold the pin they * can't be acquired as victim buffers by another backend. */ - for (uint32 i = 0; i < extend_by; i++) { - Block buf_block; + for (uint32 i = 0; i < extend_by; i++) + { + Block buf_block; buffers[i] = GetVictimBuffer(strategy, io_context); buf_block = BufHdrGetBlock(GetBufferDescriptor(buffers[i] - 1)); /* new buffers are zero-filled */ - MemSet((char *)buf_block, 0, BLCKSZ); + MemSet((char *) buf_block, 0, BLCKSZ); } /* in case we need to pin an existing buffer below */ @@ -4435,26 +4673,31 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess * reconfig-revoke (AD-013) + backend-exit cleanup as the backstop on an * error path, mirroring CF (spec-5.6). */ - if (hwc == CLUSTER_HW_GLOBALIZE) { + if (hwc == CLUSTER_HW_GLOBALIZE) + { cluster_hw_resid_encode(RelationGetSmgr(bmr.rel)->smgr_rlocator.locator, fork, &hw_resid); - if (!cluster_hw_lock(&hw_resid, &hwlk)) { + if (!cluster_hw_lock(&hw_resid, &hwlk)) + { /* release the victim buffers pinned above (the extension lock is not * held yet) before failing closed, symmetric with the allocate-fail * path below (review P2). */ - for (uint32 i = 0; i < extend_by; i++) { + for (uint32 i = 0; i < extend_by; i++) + { BufferDesc *buf_hdr = GetBufferDescriptor(buffers[i] - 1); StrategyFreeBuffer(buf_hdr); UnpinBuffer(buf_hdr); } - ereport(ERROR, (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), - errmsg("could not acquire the cluster relation-extend lock for \"%s\"", - RelationGetRelationName(bmr.rel)))); + ereport(ERROR, + (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), + errmsg("could not acquire the cluster relation-extend lock for \"%s\"", + RelationGetRelationName(bmr.rel)))); } } #endif - if (!(flags & EB_SKIP_EXTENSION_LOCK)) { + if (!(flags & EB_SKIP_EXTENSION_LOCK)) + { LockRelationForExtension(bmr.rel, ExclusiveLock); if (bmr.rel) bmr.smgr = RelationGetSmgr(bmr.rel); @@ -4468,8 +4711,9 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess bmr.smgr->smgr_cached_nblocks[fork] = InvalidBlockNumber; #ifdef USE_PGRAC_CLUSTER - if (hwc == CLUSTER_HW_GLOBALIZE) { - uint32 hw_granted = 0; + if (hwc == CLUSTER_HW_GLOBALIZE) + { + uint32 hw_granted = 0; BlockNumber seed_nblocks; /* @@ -4504,45 +4748,52 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess * the only shape the hio.c lease consumer serves. */ { - uint32 hw_want = extend_by; + uint32 hw_want = extend_by; - if (cluster_hw_lease_active() && extend_upto == InvalidBlockNumber - && bmr.rel->rd_rel->relkind == RELKIND_RELATION - && (uint32)cluster_space_lease_blocks > extend_by) - hw_want = (uint32)cluster_space_lease_blocks; + if (cluster_hw_lease_active() && + extend_upto == InvalidBlockNumber && + bmr.rel->rd_rel->relkind == RELKIND_RELATION && + (uint32) cluster_space_lease_blocks > extend_by) + hw_want = (uint32) cluster_space_lease_blocks; first_block = cluster_hw_allocate(RelationGetSmgr(bmr.rel)->smgr_rlocator.locator, fork, hw_want, seed_nblocks, &hw_granted); } cluster_hw_unlock(&hwlk); - if (first_block == InvalidBlockNumber || hw_granted == 0) { + if (first_block == InvalidBlockNumber || hw_granted == 0) + { /* fail closed: release the extension lock + all victim buffers, then * raise 53RA6 (we are outside any critical section). */ if (!(flags & EB_SKIP_EXTENSION_LOCK)) UnlockRelationForExtension(bmr.rel, ExclusiveLock); - for (uint32 i = 0; i < extend_by; i++) { + for (uint32 i = 0; i < extend_by; i++) + { BufferDesc *buf_hdr = GetBufferDescriptor(buffers[i] - 1); StrategyFreeBuffer(buf_hdr); UnpinBuffer(buf_hdr); } - ereport(ERROR, (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), - errmsg("cluster relation-extend authority unavailable for \"%s\"", - RelationGetRelationName(bmr.rel)), - errhint("The HW_ALLOC round trip to the resource master could not be " - "proven; retry."))); + ereport(ERROR, + (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), + errmsg("cluster relation-extend authority unavailable for \"%s\"", + RelationGetRelationName(bmr.rel)), + errhint("The HW_ALLOC round trip to the resource master could not be proven; retry."))); } - if (hw_granted < extend_by) { - for (uint32 i = hw_granted; i < extend_by; i++) { + if (hw_granted < extend_by) + { + for (uint32 i = hw_granted; i < extend_by; i++) + { BufferDesc *buf_hdr = GetBufferDescriptor(buffers[i] - 1); StrategyFreeBuffer(buf_hdr); UnpinBuffer(buf_hdr); } extend_by = hw_granted; - } else { + } + else + { /* * PGRAC: spec-6.12d -- the over-ask surplus becomes this node's * lease. It MUST also be zero-extended below: the HWM advance @@ -4553,9 +4804,10 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess */ hw_lease_tail = hw_granted - extend_by; } - } else + } + else #endif - first_block = smgrnblocks(bmr.smgr, fork); + first_block = smgrnblocks(bmr.smgr, fork); /* * Now that we have the accurate relation size, check if the caller wants @@ -4563,15 +4815,17 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess * extensions, we might have acquired too many buffers and need to release * them. */ - if (extend_upto != InvalidBlockNumber) { - uint32 orig_extend_by = extend_by; + if (extend_upto != InvalidBlockNumber) + { + uint32 orig_extend_by = extend_by; if (first_block > extend_upto) extend_by = 0; - else if ((uint64)first_block + extend_by > extend_upto) + else if ((uint64) first_block + extend_by > extend_upto) extend_by = extend_upto - first_block; - for (uint32 i = extend_by; i < orig_extend_by; i++) { + for (uint32 i = extend_by; i < orig_extend_by; i++) + { BufferDesc *buf_hdr = GetBufferDescriptor(buffers[i] - 1); /* @@ -4582,7 +4836,8 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess UnpinBuffer(buf_hdr); } - if (extend_by == 0) { + if (extend_by == 0) + { if (!(flags & EB_SKIP_EXTENSION_LOCK)) UnlockRelationForExtension(bmr.rel, ExclusiveLock); *extended_by = extend_by; @@ -4591,10 +4846,12 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess } /* Fail if relation is already at maximum possible length */ - if ((uint64)first_block + extend_by >= MaxBlockNumber) - ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg("cannot extend relation %s beyond %u blocks", - relpath(bmr.smgr->smgr_rlocator, fork), MaxBlockNumber))); + if ((uint64) first_block + extend_by >= MaxBlockNumber) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("cannot extend relation %s beyond %u blocks", + relpath(bmr.smgr->smgr_rlocator, fork), + MaxBlockNumber))); /* * Insert buffers into buffer table, mark as IO_IN_PROGRESS. @@ -4602,13 +4859,14 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess * This needs to happen before we extend the relation, because as soon as * we do, other backends can start to read in those pages. */ - for (int i = 0; i < extend_by; i++) { - Buffer victim_buf = buffers[i]; + for (int i = 0; i < extend_by; i++) + { + Buffer victim_buf = buffers[i]; BufferDesc *victim_buf_hdr = GetBufferDescriptor(victim_buf - 1); - BufferTag tag; - uint32 hash; - LWLock *partition_lock; - int existing_id; + BufferTag tag; + uint32 hash; + LWLock *partition_lock; + int existing_id; InitBufferTag(&tag, &bmr.smgr->smgr_rlocator.locator, fork, first_block + i); hash = BufTableHashCode(&tag); @@ -4632,10 +4890,11 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess * overwrite. Since the legitimate cases should always have left a * zero-filled buffer, complain if not PageIsNew. */ - if (existing_id >= 0) { + if (existing_id >= 0) + { BufferDesc *existing_hdr = GetBufferDescriptor(existing_id); - Block buf_block; - bool valid; + Block buf_block; + bool valid; /* * Pin the existing buffer before releasing the partition lock, @@ -4655,14 +4914,13 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess buffers[i] = BufferDescriptorGetBuffer(existing_hdr); buf_block = BufHdrGetBlock(existing_hdr); - if (valid && !PageIsNew((Page)buf_block)) + if (valid && !PageIsNew((Page) buf_block)) ereport(ERROR, (errmsg("unexpected data beyond EOF in block %u of relation %s", existing_hdr->tag.blockNum, relpath(bmr.smgr->smgr_rlocator, fork)), - errhint("This has been seen to occur with buggy kernels; consider " - "updating your system."))); + errhint("This has been seen to occur with buggy kernels; consider updating your system."))); - /* + /* * We *must* do smgr[zero]extend before succeeding, else the page * will not be reserved by the kernel, and the next P_NEW call * will decide to return the same page. Clear the BM_VALID bit, @@ -4674,27 +4932,31 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess */ #ifdef USE_PGRAC_CLUSTER { - uint32 cluster_state = LockBufHdr(existing_hdr); - bool blocked + uint32 cluster_state = LockBufHdr(existing_hdr); + bool blocked = cluster_bufmgr_pcm_x_retained_image_locked(existing_hdr, cluster_state) - || (cluster_pcm_own_flags_get(existing_hdr->buf_id) & PCM_OWN_FLAG_REVOKING) - != 0; + || (cluster_pcm_own_flags_get(existing_hdr->buf_id) + & PCM_OWN_FLAG_REVOKING) != 0; UnlockBufHdr(existing_hdr, cluster_state); if (blocked) - ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), - errmsg("cannot reuse retained cluster PCM image during " - "relation extension"))); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_IN_USE), + errmsg("cannot reuse retained cluster PCM image during " + "relation extension"))); } #endif - do { - uint32 buf_state = LockBufHdr(existing_hdr); + do + { + uint32 buf_state = LockBufHdr(existing_hdr); buf_state &= ~BM_VALID; UnlockBufHdr(existing_hdr, buf_state); } while (!StartBufferIO(existing_hdr, true)); - } else { - uint32 buf_state; + } + else + { + uint32 buf_state; buf_state = LockBufHdr(victim_buf_hdr); @@ -4743,10 +5005,12 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess * no lease is installed and the range degrades to the documented * orphan-zero-page fail-safe once a later extend covers it. */ - if (hw_lease_tail > 0) { - smgrzeroextend(bmr.smgr, fork, first_block + extend_by, (int)hw_lease_tail, false); - cluster_hw_lease_install(bmr.smgr->smgr_rlocator.locator, fork, first_block + extend_by, - hw_lease_tail); + if (hw_lease_tail > 0) + { + smgrzeroextend(bmr.smgr, fork, first_block + extend_by, + (int) hw_lease_tail, false); + cluster_hw_lease_install(bmr.smgr->smgr_rlocator.locator, fork, + first_block + extend_by, hw_lease_tail); } #endif @@ -4760,32 +5024,36 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ForkNumber fork, BufferAccess if (!(flags & EB_SKIP_EXTENSION_LOCK)) UnlockRelationForExtension(bmr.rel, ExclusiveLock); - pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, IOOP_EXTEND, io_start, extend_by); + pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, IOOP_EXTEND, + io_start, extend_by); /* Set BM_VALID, terminate IO, and wake up any waiters */ - for (int i = 0; i < extend_by; i++) { - Buffer buf = buffers[i]; + for (int i = 0; i < extend_by; i++) + { + Buffer buf = buffers[i]; BufferDesc *buf_hdr = GetBufferDescriptor(buf - 1); - bool lock = false; + bool lock = false; if (flags & EB_LOCK_FIRST && i == 0) lock = true; - else if (flags & EB_LOCK_TARGET) { + else if (flags & EB_LOCK_TARGET) + { Assert(extend_upto != InvalidBlockNumber); if (first_block + i + 1 == extend_upto) lock = true; } - if (lock) { + if (lock) + { #ifdef USE_PGRAC_CLUSTER ClusterPcmDirectInitProof direct_init_proof; /* This exact buffer is in the range successfully zeroextended above * and still owns BM_IO_IN_PROGRESS + !BM_VALID. */ - cluster_bufmgr_pcm_arm_direct_init(buf_hdr, CLUSTER_PCM_DIRECT_INIT_EXTEND, - &direct_init_proof); - cluster_bufmgr_pcm_gate_direct_init(buf_hdr, CLUSTER_PCM_DIRECT_INIT_EXTEND, - &direct_init_proof); + cluster_bufmgr_pcm_arm_direct_init( + buf_hdr, CLUSTER_PCM_DIRECT_INIT_EXTEND, &direct_init_proof); + cluster_bufmgr_pcm_gate_direct_init( + buf_hdr, CLUSTER_PCM_DIRECT_INIT_EXTEND, &direct_init_proof); #endif LWLockAcquire(BufferDescriptorGetContentLock(buf_hdr), LW_EXCLUSIVE); } @@ -4813,13 +5081,14 @@ void MarkBufferDirty(Buffer buffer) { BufferDesc *bufHdr; - uint32 buf_state; - uint32 old_buf_state; + uint32 buf_state; + uint32 old_buf_state; if (!BufferIsValid(buffer)) elog(ERROR, "bad buffer ID: %d", buffer); - if (BufferIsLocal(buffer)) { + if (BufferIsLocal(buffer)) + { MarkLocalBufferDirty(buffer); return; } @@ -4827,7 +5096,8 @@ MarkBufferDirty(Buffer buffer) bufHdr = GetBufferDescriptor(buffer - 1); Assert(BufferIsPinned(buffer)); - Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr), LW_EXCLUSIVE)); + Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr), + LW_EXCLUSIVE)); #ifdef USE_PGRAC_CLUSTER @@ -4837,17 +5107,20 @@ MarkBufferDirty(Buffer buffer) * retain/release boundary. */ buf_state = LockBufHdr(bufHdr); - if (cluster_bufmgr_pcm_x_retained_image_locked(bufHdr, buf_state)) { + if (cluster_bufmgr_pcm_x_retained_image_locked(bufHdr, buf_state)) + { UnlockBufHdr(bufHdr, buf_state); - ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("cannot dirty a retained cluster PCM image"), - errdetail("buffer=%d", bufHdr->buf_id))); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot dirty a retained cluster PCM image"), + errdetail("buffer=%d", bufHdr->buf_id))); } UnlockBufHdr(bufHdr, buf_state); #endif old_buf_state = pg_atomic_read_u32(&bufHdr->state); - for (;;) { + for (;;) + { if (old_buf_state & BM_LOCKED) old_buf_state = WaitBufHdrUnlocked(bufHdr); @@ -4856,14 +5129,16 @@ MarkBufferDirty(Buffer buffer) Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0); buf_state |= BM_DIRTY | BM_JUST_DIRTIED; - if (pg_atomic_compare_exchange_u32(&bufHdr->state, &old_buf_state, buf_state)) + if (pg_atomic_compare_exchange_u32(&bufHdr->state, &old_buf_state, + buf_state)) break; } /* * If the buffer was not dirty already, do vacuum accounting. */ - if (!(old_buf_state & BM_DIRTY)) { + if (!(old_buf_state & BM_DIRTY)) + { VacuumPageDirty++; pgBufferUsage.shared_blks_dirtied++; if (VacuumCostActive) @@ -4885,26 +5160,32 @@ MarkBufferDirty(Buffer buffer) * but can save some tests in the caller. */ Buffer -ReleaseAndReadBuffer(Buffer buffer, Relation relation, BlockNumber blockNum) +ReleaseAndReadBuffer(Buffer buffer, + Relation relation, + BlockNumber blockNum) { - ForkNumber forkNum = MAIN_FORKNUM; + ForkNumber forkNum = MAIN_FORKNUM; BufferDesc *bufHdr; - if (BufferIsValid(buffer)) { + if (BufferIsValid(buffer)) + { Assert(BufferIsPinned(buffer)); - if (BufferIsLocal(buffer)) { + if (BufferIsLocal(buffer)) + { bufHdr = GetLocalBufferDescriptor(-buffer - 1); - if (bufHdr->tag.blockNum == blockNum - && BufTagMatchesRelFileLocator(&bufHdr->tag, &relation->rd_locator) - && BufTagGetForkNum(&bufHdr->tag) == forkNum) + if (bufHdr->tag.blockNum == blockNum && + BufTagMatchesRelFileLocator(&bufHdr->tag, &relation->rd_locator) && + BufTagGetForkNum(&bufHdr->tag) == forkNum) return buffer; UnpinLocalBuffer(buffer); - } else { + } + else + { bufHdr = GetBufferDescriptor(buffer - 1); /* we have pin, so it's ok to examine tag without spinlock */ - if (bufHdr->tag.blockNum == blockNum - && BufTagMatchesRelFileLocator(&bufHdr->tag, &relation->rd_locator) - && BufTagGetForkNum(&bufHdr->tag) == forkNum) + if (bufHdr->tag.blockNum == blockNum && + BufTagMatchesRelFileLocator(&bufHdr->tag, &relation->rd_locator) && + BufTagGetForkNum(&bufHdr->tag) == forkNum) return buffer; UnpinBuffer(bufHdr); } @@ -4938,23 +5219,25 @@ ReleaseAndReadBuffer(Buffer buffer, Relation relation, BlockNumber blockNum) static bool PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy) { - Buffer b = BufferDescriptorGetBuffer(buf); - bool result; + Buffer b = BufferDescriptorGetBuffer(buf); + bool result; PrivateRefCountEntry *ref; Assert(!BufferIsLocal(b)); ref = GetPrivateRefCountEntry(b, true); - if (ref == NULL) { - uint32 buf_state; - uint32 old_buf_state; + if (ref == NULL) + { + uint32 buf_state; + uint32 old_buf_state; ReservePrivateRefCountEntry(); ref = NewPrivateRefCountEntry(b); old_buf_state = pg_atomic_read_u32(&buf->state); - for (;;) { + for (;;) + { if (old_buf_state & BM_LOCKED) old_buf_state = WaitBufHdrUnlocked(buf); @@ -4963,11 +5246,14 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy) /* increase refcount */ buf_state += BUF_REFCOUNT_ONE; - if (strategy == NULL) { + if (strategy == NULL) + { /* Default case: increase usagecount unless already max. */ if (BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT) buf_state += BUF_USAGECOUNT_ONE; - } else { + } + else + { /* * Ring buffers shouldn't evict others from pool. Thus we * don't make usagecount more than 1. @@ -4976,7 +5262,9 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy) buf_state += BUF_USAGECOUNT_ONE; } - if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state, buf_state)) { + if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state, + buf_state)) + { result = (buf_state & BM_VALID) != 0; /* @@ -4990,7 +5278,9 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy) break; } } - } else { + } + else + { /* * If we previously pinned the buffer, it must surely be valid. * @@ -5034,9 +5324,9 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy) static void PinBuffer_Locked(BufferDesc *buf) { - Buffer b; + Buffer b; PrivateRefCountEntry *ref; - uint32 buf_state; + uint32 buf_state; /* * As explained, We don't expect any preexisting pins. That allows us to @@ -5078,7 +5368,7 @@ static void UnpinBuffer(BufferDesc *buf) { PrivateRefCountEntry *ref; - Buffer b = BufferDescriptorGetBuffer(buf); + Buffer b = BufferDescriptorGetBuffer(buf); Assert(!BufferIsLocal(b)); @@ -5090,9 +5380,10 @@ UnpinBuffer(BufferDesc *buf) Assert(ref->refcount > 0); ref->refcount--; - if (ref->refcount == 0) { - uint32 buf_state; - uint32 old_buf_state; + if (ref->refcount == 0) + { + uint32 buf_state; + uint32 old_buf_state; /* * Mark buffer non-accessible to Valgrind. @@ -5113,7 +5404,8 @@ UnpinBuffer(BufferDesc *buf) * it's not safe to use atomic decrement here; thus use a CAS loop. */ old_buf_state = pg_atomic_read_u32(&buf->state); - for (;;) { + for (;;) + { if (old_buf_state & BM_LOCKED) old_buf_state = WaitBufHdrUnlocked(buf); @@ -5121,12 +5413,14 @@ UnpinBuffer(BufferDesc *buf) buf_state -= BUF_REFCOUNT_ONE; - if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state, buf_state)) + if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state, + buf_state)) break; } /* Support LockBufferForCleanup() */ - if (buf_state & BM_PIN_COUNT_WAITER) { + if (buf_state & BM_PIN_COUNT_WAITER) + { /* * Acquire the buffer header lock, re-check that there's a waiter. * Another backend could have unpinned this buffer, and already @@ -5136,14 +5430,17 @@ UnpinBuffer(BufferDesc *buf) */ buf_state = LockBufHdr(buf); - if ((buf_state & BM_PIN_COUNT_WAITER) && BUF_STATE_GET_REFCOUNT(buf_state) == 1) { + if ((buf_state & BM_PIN_COUNT_WAITER) && + BUF_STATE_GET_REFCOUNT(buf_state) == 1) + { /* we just released the last pin other than the waiter's */ - int wait_backend_pgprocno = buf->wait_backend_pgprocno; + int wait_backend_pgprocno = buf->wait_backend_pgprocno; buf_state &= ~BM_PIN_COUNT_WAITER; UnlockBufHdr(buf, buf_state); ProcSendSignal(wait_backend_pgprocno); - } else + } + else UnlockBufHdr(buf, buf_state); } ForgetPrivateRefCountEntry(ref); @@ -5170,17 +5467,17 @@ UnpinBuffer(BufferDesc *buf) static void BufferSync(int flags) { - uint32 buf_state; - int buf_id; - int num_to_scan; - int num_spaces; - int num_processed; - int num_written; + uint32 buf_state; + int buf_id; + int num_to_scan; + int num_spaces; + int num_processed; + int num_written; CkptTsStatus *per_ts_stat = NULL; - Oid last_tsid; + Oid last_tsid; binaryheap *ts_heap; - int i; - int mask = BM_DIRTY; + int i; + int mask = BM_DIRTY; WritebackContext wb_context; /* Make sure we can handle the pin inside SyncOneBuffer */ @@ -5191,7 +5488,8 @@ BufferSync(int flags) * we write only permanent, dirty buffers. But at shutdown or end of * recovery, we write all dirty buffers. */ - if (!((flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY | CHECKPOINT_FLUSH_ALL)))) + if (!((flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY | + CHECKPOINT_FLUSH_ALL)))) mask |= BM_PERMANENT; /* @@ -5211,7 +5509,8 @@ BufferSync(int flags) * certainly need to be written for the next checkpoint attempt, too. */ num_to_scan = 0; - for (buf_id = 0; buf_id < NBuffers; buf_id++) { + for (buf_id = 0; buf_id < NBuffers; buf_id++) + { BufferDesc *bufHdr = GetBufferDescriptor(buf_id); /* @@ -5220,7 +5519,8 @@ BufferSync(int flags) */ buf_state = LockBufHdr(bufHdr); - if ((buf_state & mask) == mask) { + if ((buf_state & mask) == mask) + { CkptSortItem *item; buf_state |= BM_CHECKPOINT_NEEDED; @@ -5241,7 +5541,7 @@ BufferSync(int flags) } if (num_to_scan == 0) - return; /* nothing to do */ + return; /* nothing to do */ WritebackContextInit(&wb_context, &checkpoint_flush_after); @@ -5263,9 +5563,10 @@ BufferSync(int flags) * be flushed. This requires the to-be-flushed array to be sorted. */ last_tsid = InvalidOid; - for (i = 0; i < num_to_scan; i++) { + for (i = 0; i < num_to_scan; i++) + { CkptTsStatus *s; - Oid cur_tsid; + Oid cur_tsid; cur_tsid = CkptBufferIds[i].tsId; @@ -5273,8 +5574,9 @@ BufferSync(int flags) * Grow array of per-tablespace status structs, every time a new * tablespace is found. */ - if (last_tsid == InvalidOid || last_tsid != cur_tsid) { - Size sz; + if (last_tsid == InvalidOid || last_tsid != cur_tsid) + { + Size sz; num_spaces++; @@ -5285,9 +5587,9 @@ BufferSync(int flags) sz = sizeof(CkptTsStatus) * num_spaces; if (per_ts_stat == NULL) - per_ts_stat = (CkptTsStatus *)palloc(sz); + per_ts_stat = (CkptTsStatus *) palloc(sz); else - per_ts_stat = (CkptTsStatus *)repalloc(per_ts_stat, sz); + per_ts_stat = (CkptTsStatus *) repalloc(per_ts_stat, sz); s = &per_ts_stat[num_spaces - 1]; memset(s, 0, sizeof(*s)); @@ -5306,7 +5608,9 @@ BufferSync(int flags) */ last_tsid = cur_tsid; - } else { + } + else + { s = &per_ts_stat[num_spaces - 1]; } @@ -5324,12 +5628,15 @@ BufferSync(int flags) * and compute how large a portion of the total progress a single * processed buffer is. */ - ts_heap = binaryheap_allocate(num_spaces, ts_ckpt_progress_comparator, NULL); + ts_heap = binaryheap_allocate(num_spaces, + ts_ckpt_progress_comparator, + NULL); - for (i = 0; i < num_spaces; i++) { + for (i = 0; i < num_spaces; i++) + { CkptTsStatus *ts_stat = &per_ts_stat[i]; - ts_stat->progress_slice = (float8)num_to_scan / ts_stat->num_to_scan; + ts_stat->progress_slice = (float8) num_to_scan / ts_stat->num_to_scan; binaryheap_add_unordered(ts_heap, PointerGetDatum(ts_stat)); } @@ -5344,9 +5651,11 @@ BufferSync(int flags) */ num_processed = 0; num_written = 0; - while (!binaryheap_empty(ts_heap)) { + while (!binaryheap_empty(ts_heap)) + { BufferDesc *bufHdr = NULL; - CkptTsStatus *ts_stat = (CkptTsStatus *)DatumGetPointer(binaryheap_first(ts_heap)); + CkptTsStatus *ts_stat = (CkptTsStatus *) + DatumGetPointer(binaryheap_first(ts_heap)); buf_id = CkptBufferIds[ts_stat->index].buf_id; Assert(buf_id != -1); @@ -5367,8 +5676,10 @@ BufferSync(int flags) * write the buffer though we didn't need to. It doesn't seem worth * guarding against this, though. */ - if (pg_atomic_read_u32(&bufHdr->state) & BM_CHECKPOINT_NEEDED) { - if (SyncOneBuffer(buf_id, false, &wb_context) & BUF_WRITTEN) { + if (pg_atomic_read_u32(&bufHdr->state) & BM_CHECKPOINT_NEEDED) + { + if (SyncOneBuffer(buf_id, false, &wb_context) & BUF_WRITTEN) + { TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id); PendingCheckpointerStats.buf_written_checkpoints++; num_written++; @@ -5384,9 +5695,12 @@ BufferSync(int flags) ts_stat->index++; /* Have all the buffers from the tablespace been processed? */ - if (ts_stat->num_scanned == ts_stat->num_to_scan) { + if (ts_stat->num_scanned == ts_stat->num_to_scan) + { binaryheap_remove_first(ts_heap); - } else { + } + else + { /* update heap with the new progress */ binaryheap_replace_first(ts_heap, PointerGetDatum(ts_stat)); } @@ -5396,7 +5710,7 @@ BufferSync(int flags) * * (This will check for barrier events even if it doesn't sleep.) */ - CheckpointWriteDelay(flags, (double)num_processed / num_to_scan); + CheckpointWriteDelay(flags, (double) num_processed / num_to_scan); } /* @@ -5433,18 +5747,18 @@ bool BgBufferSync(WritebackContext *wb_context) { /* info obtained from freelist.c */ - int strategy_buf_id; - uint32 strategy_passes; - uint32 recent_alloc; + int strategy_buf_id; + uint32 strategy_passes; + uint32 recent_alloc; /* * Information saved between calls so we can determine the strategy * point's advance rate and avoid scanning already-cleaned buffers. */ static bool saved_info_valid = false; - static int prev_strategy_buf_id; + static int prev_strategy_buf_id; static uint32 prev_strategy_passes; - static int next_to_clean; + static int next_to_clean; static uint32 next_passes; /* Moving averages of allocation rate and clean-buffer density */ @@ -5452,26 +5766,26 @@ BgBufferSync(WritebackContext *wb_context) static float smoothed_density = 10.0; /* Potentially these could be tunables, but for now, not */ - float smoothing_samples = 16; - float scan_whole_pool_milliseconds = 120000.0; + float smoothing_samples = 16; + float scan_whole_pool_milliseconds = 120000.0; /* Used to compute how far we scan ahead */ - long strategy_delta; - int bufs_to_lap; - int bufs_ahead; - float scans_per_alloc; - int reusable_buffers_est; - int upcoming_alloc_est; - int min_scan_buffers; + long strategy_delta; + int bufs_to_lap; + int bufs_ahead; + float scans_per_alloc; + int reusable_buffers_est; + int upcoming_alloc_est; + int min_scan_buffers; /* Variables for the scanning loop proper */ - int num_to_scan; - int num_written; - int reusable_buffers; + int num_to_scan; + int num_written; + int reusable_buffers; /* Variables for final smoothed_density update */ - long new_strategy_delta; - uint32 new_recent_alloc; + long new_strategy_delta; + uint32 new_recent_alloc; /* * Find out where the freelist clock sweep currently is, and how many @@ -5487,7 +5801,8 @@ BgBufferSync(WritebackContext *wb_context) * stuff. We mark the saved state invalid so that we can recover sanely * if LRU scan is turned back on later. */ - if (bgwriter_lru_maxpages <= 0) { + if (bgwriter_lru_maxpages <= 0) + { saved_info_valid = false; return true; } @@ -5500,48 +5815,64 @@ BgBufferSync(WritebackContext *wb_context) * weird-looking coding of xxx_passes comparisons are to avoid bogus * behavior when the passes counts wrap around. */ - if (saved_info_valid) { - int32 passes_delta = strategy_passes - prev_strategy_passes; + if (saved_info_valid) + { + int32 passes_delta = strategy_passes - prev_strategy_passes; strategy_delta = strategy_buf_id - prev_strategy_buf_id; - strategy_delta += (long)passes_delta * NBuffers; + strategy_delta += (long) passes_delta * NBuffers; Assert(strategy_delta >= 0); - if ((int32)(next_passes - strategy_passes) > 0) { + if ((int32) (next_passes - strategy_passes) > 0) + { /* we're one pass ahead of the strategy point */ bufs_to_lap = strategy_buf_id - next_to_clean; #ifdef BGW_DEBUG - elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d", next_passes, - next_to_clean, strategy_passes, strategy_buf_id, strategy_delta, bufs_to_lap); + elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d", + next_passes, next_to_clean, + strategy_passes, strategy_buf_id, + strategy_delta, bufs_to_lap); #endif - } else if (next_passes == strategy_passes && next_to_clean >= strategy_buf_id) { + } + else if (next_passes == strategy_passes && + next_to_clean >= strategy_buf_id) + { /* on same pass, but ahead or at least not behind */ bufs_to_lap = NBuffers - (next_to_clean - strategy_buf_id); #ifdef BGW_DEBUG - elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d", next_passes, - next_to_clean, strategy_passes, strategy_buf_id, strategy_delta, bufs_to_lap); + elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d", + next_passes, next_to_clean, + strategy_passes, strategy_buf_id, + strategy_delta, bufs_to_lap); #endif - } else { + } + else + { /* * We're behind, so skip forward to the strategy point and start * cleaning from there. */ #ifdef BGW_DEBUG - elog(DEBUG2, "bgwriter behind: bgw %u-%u strategy %u-%u delta=%ld", next_passes, - next_to_clean, strategy_passes, strategy_buf_id, strategy_delta); + elog(DEBUG2, "bgwriter behind: bgw %u-%u strategy %u-%u delta=%ld", + next_passes, next_to_clean, + strategy_passes, strategy_buf_id, + strategy_delta); #endif next_to_clean = strategy_buf_id; next_passes = strategy_passes; bufs_to_lap = NBuffers; } - } else { + } + else + { /* * Initializing at startup or after LRU scanning had been off. Always * start at the strategy point. */ #ifdef BGW_DEBUG - elog(DEBUG2, "bgwriter initializing: strategy %u-%u", strategy_passes, strategy_buf_id); + elog(DEBUG2, "bgwriter initializing: strategy %u-%u", + strategy_passes, strategy_buf_id); #endif strategy_delta = 0; next_to_clean = strategy_buf_id; @@ -5560,9 +5891,11 @@ BgBufferSync(WritebackContext *wb_context) * * If the strategy point didn't move, we don't update the density estimate */ - if (strategy_delta > 0 && recent_alloc > 0) { - scans_per_alloc = (float)strategy_delta / (float)recent_alloc; - smoothed_density += (scans_per_alloc - smoothed_density) / smoothing_samples; + if (strategy_delta > 0 && recent_alloc > 0) + { + scans_per_alloc = (float) strategy_delta / (float) recent_alloc; + smoothed_density += (scans_per_alloc - smoothed_density) / + smoothing_samples; } /* @@ -5571,20 +5904,21 @@ BgBufferSync(WritebackContext *wb_context) * density estimate. */ bufs_ahead = NBuffers - bufs_to_lap; - reusable_buffers_est = (float)bufs_ahead / smoothed_density; + reusable_buffers_est = (float) bufs_ahead / smoothed_density; /* * Track a moving average of recent buffer allocations. Here, rather than * a true average we want a fast-attack, slow-decline behavior: we * immediately follow any increase. */ - if (smoothed_alloc <= (float)recent_alloc) + if (smoothed_alloc <= (float) recent_alloc) smoothed_alloc = recent_alloc; else - smoothed_alloc += ((float)recent_alloc - smoothed_alloc) / smoothing_samples; + smoothed_alloc += ((float) recent_alloc - smoothed_alloc) / + smoothing_samples; /* Scale the estimate by a GUC to allow more aggressive tuning. */ - upcoming_alloc_est = (int)(smoothed_alloc * bgwriter_lru_multiplier); + upcoming_alloc_est = (int) (smoothed_alloc * bgwriter_lru_multiplier); /* * If recent_alloc remains at zero for many cycles, smoothed_alloc will @@ -5607,9 +5941,10 @@ BgBufferSync(WritebackContext *wb_context) * the BGW will be called during the scan_whole_pool time; slice the * buffer pool into that many sections. */ - min_scan_buffers = (int)(NBuffers / (scan_whole_pool_milliseconds / BgWriterDelay)); + min_scan_buffers = (int) (NBuffers / (scan_whole_pool_milliseconds / BgWriterDelay)); - if (upcoming_alloc_est < (min_scan_buffers + reusable_buffers_est)) { + if (upcoming_alloc_est < (min_scan_buffers + reusable_buffers_est)) + { #ifdef BGW_DEBUG elog(DEBUG2, "bgwriter: alloc_est=%d too small, using min=%d + reusable_est=%d", upcoming_alloc_est, min_scan_buffers, reusable_buffers_est); @@ -5632,33 +5967,39 @@ BgBufferSync(WritebackContext *wb_context) reusable_buffers = reusable_buffers_est; /* Execute the LRU scan */ - while (num_to_scan > 0 && reusable_buffers < upcoming_alloc_est) { - int sync_state = SyncOneBuffer(next_to_clean, true, wb_context); + while (num_to_scan > 0 && reusable_buffers < upcoming_alloc_est) + { + int sync_state = SyncOneBuffer(next_to_clean, true, + wb_context); - if (++next_to_clean >= NBuffers) { + if (++next_to_clean >= NBuffers) + { next_to_clean = 0; next_passes++; } num_to_scan--; - if (sync_state & BUF_WRITTEN) { + if (sync_state & BUF_WRITTEN) + { reusable_buffers++; - if (++num_written >= bgwriter_lru_maxpages) { + if (++num_written >= bgwriter_lru_maxpages) + { PendingBgWriterStats.maxwritten_clean++; break; } - } else if (sync_state & BUF_REUSABLE) + } + else if (sync_state & BUF_REUSABLE) reusable_buffers++; } PendingBgWriterStats.buf_written_clean += num_written; #ifdef BGW_DEBUG - elog(DEBUG1, - "bgwriter: recent_alloc=%u smoothed=%.2f delta=%ld ahead=%d density=%.2f reusable_est=%d " - "upcoming_est=%d scanned=%d wrote=%d reusable=%d", - recent_alloc, smoothed_alloc, strategy_delta, bufs_ahead, smoothed_density, - reusable_buffers_est, upcoming_alloc_est, bufs_to_lap - num_to_scan, num_written, + elog(DEBUG1, "bgwriter: recent_alloc=%u smoothed=%.2f delta=%ld ahead=%d density=%.2f reusable_est=%d upcoming_est=%d scanned=%d wrote=%d reusable=%d", + recent_alloc, smoothed_alloc, strategy_delta, bufs_ahead, + smoothed_density, reusable_buffers_est, upcoming_alloc_est, + bufs_to_lap - num_to_scan, + num_written, reusable_buffers - reusable_buffers_est); #endif @@ -5672,13 +6013,16 @@ BgBufferSync(WritebackContext *wb_context) */ new_strategy_delta = bufs_to_lap - num_to_scan; new_recent_alloc = reusable_buffers - reusable_buffers_est; - if (new_strategy_delta > 0 && new_recent_alloc > 0) { - scans_per_alloc = (float)new_strategy_delta / (float)new_recent_alloc; - smoothed_density += (scans_per_alloc - smoothed_density) / smoothing_samples; + if (new_strategy_delta > 0 && new_recent_alloc > 0) + { + scans_per_alloc = (float) new_strategy_delta / (float) new_recent_alloc; + smoothed_density += (scans_per_alloc - smoothed_density) / + smoothing_samples; #ifdef BGW_DEBUG elog(DEBUG2, "bgwriter: cleaner density alloc=%u scan=%ld density=%.2f new smoothed=%.2f", - new_recent_alloc, new_strategy_delta, scans_per_alloc, smoothed_density); + new_recent_alloc, new_strategy_delta, + scans_per_alloc, smoothed_density); #endif } @@ -5706,9 +6050,9 @@ static int SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) { BufferDesc *bufHdr = GetBufferDescriptor(buf_id); - int result = 0; - uint32 buf_state; - BufferTag tag; + int result = 0; + uint32 buf_state; + BufferTag tag; ReservePrivateRefCountEntry(); @@ -5729,21 +6073,27 @@ SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) * A live retained image is neither reusable nor writable. Test before * advertising BUF_REUSABLE; finish/release serialize the later race with * the content-lock recheck below. */ - if (cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(bufHdr, buf_state)) { + if (cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(bufHdr, buf_state)) + { UnlockBufHdr(bufHdr, buf_state); return 0; } #endif - if (BUF_STATE_GET_REFCOUNT(buf_state) == 0 && BUF_STATE_GET_USAGECOUNT(buf_state) == 0) { + if (BUF_STATE_GET_REFCOUNT(buf_state) == 0 && + BUF_STATE_GET_USAGECOUNT(buf_state) == 0) + { result |= BUF_REUSABLE; - } else if (skip_recently_used) { + } + else if (skip_recently_used) + { /* Caller told us not to write recently-used buffers */ UnlockBufHdr(bufHdr, buf_state); return result; } - if (!(buf_state & BM_VALID) || !(buf_state & BM_DIRTY)) { + if (!(buf_state & BM_VALID) || !(buf_state & BM_DIRTY)) + { /* It's clean, so nothing to do */ UnlockBufHdr(bufHdr, buf_state); return result; @@ -5763,7 +6113,8 @@ SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) * the transition is now complete and stable. */ buf_state = LockBufHdr(bufHdr); - if (cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(bufHdr, buf_state)) { + if (cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(bufHdr, buf_state)) + { UnlockBufHdr(bufHdr, buf_state); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); UnpinBuffer(bufHdr); @@ -5827,7 +6178,7 @@ AtEOXact_Buffers(bool isCommit) void InitBufferPoolAccess(void) { - HASHCTL hash_ctl; + HASHCTL hash_ctl; memset(&PrivateRefCountArray, 0, sizeof(PrivateRefCountArray)); #ifdef USE_PGRAC_CLUSTER @@ -5838,7 +6189,8 @@ InitBufferPoolAccess(void) hash_ctl.keysize = sizeof(int32); hash_ctl.entrysize = sizeof(PrivateRefCountEntry); - PrivateRefCountHash = hash_create("PrivateRefCount", 100, &hash_ctl, HASH_ELEM | HASH_BLOBS); + PrivateRefCountHash = hash_create("PrivateRefCount", 100, &hash_ctl, + HASH_ELEM | HASH_BLOBS); /* * AtProcExit_Buffers needs LWLock access, and thereby has to be called at @@ -5878,26 +6230,30 @@ static void CheckForBufferLeaks(void) { #ifdef USE_ASSERT_CHECKING - int RefCountErrors = 0; + int RefCountErrors = 0; PrivateRefCountEntry *res; - int i; + int i; /* check the array */ - for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++) { + for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++) + { res = &PrivateRefCountArray[i]; - if (res->buffer != InvalidBuffer) { + if (res->buffer != InvalidBuffer) + { PrintBufferLeakWarning(res->buffer); RefCountErrors++; } } /* if necessary search the hash */ - if (PrivateRefCountOverflowed) { + if (PrivateRefCountOverflowed) + { HASH_SEQ_STATUS hstat; hash_seq_init(&hstat, PrivateRefCountHash); - while ((res = (PrivateRefCountEntry *)hash_seq_search(&hstat)) != NULL) { + while ((res = (PrivateRefCountEntry *) hash_seq_search(&hstat)) != NULL) + { PrintBufferLeakWarning(res->buffer); RefCountErrors++; } @@ -5932,10 +6288,11 @@ AssertBufferLocksPermitCatalogRead(void) } static void -AssertNotCatalogBufferLock(LWLock *lock, LWLockMode mode, void *unused_context) +AssertNotCatalogBufferLock(LWLock *lock, LWLockMode mode, + void *unused_context) { BufferDesc *bufHdr; - BufferTag tag; + BufferTag tag; Oid relid; if (mode != LW_EXCLUSIVE) @@ -5991,29 +6348,34 @@ void PrintBufferLeakWarning(Buffer buffer) { BufferDesc *buf; - int32 loccount; - char *path; - BackendId backend; - uint32 buf_state; + int32 loccount; + char *path; + BackendId backend; + uint32 buf_state; Assert(BufferIsValid(buffer)); - if (BufferIsLocal(buffer)) { + if (BufferIsLocal(buffer)) + { buf = GetLocalBufferDescriptor(-buffer - 1); loccount = LocalRefCount[-buffer - 1]; backend = MyBackendId; - } else { + } + else + { buf = GetBufferDescriptor(buffer - 1); loccount = GetPrivateRefCount(buffer); backend = InvalidBackendId; } /* theoretically we should lock the bufhdr here */ - path = relpathbackend(BufTagGetRelFileLocator(&buf->tag), backend, BufTagGetForkNum(&buf->tag)); + path = relpathbackend(BufTagGetRelFileLocator(&buf->tag), backend, + BufTagGetForkNum(&buf->tag)); buf_state = pg_atomic_read_u32(&buf->state); elog(WARNING, "buffer refcount leak: [%03d] " "(rel=%s, blockNum=%u, flags=0x%x, refcount=%u %d)", - buffer, path, buf->tag.blockNum, buf_state & BUF_FLAG_MASK, + buffer, path, + buf->tag.blockNum, buf_state & BUF_FLAG_MASK, BUF_STATE_GET_REFCOUNT(buf_state), loccount); pfree(path); } @@ -6062,7 +6424,8 @@ BufferGetBlockNumber(Buffer buffer) * a buffer. */ void -BufferGetTag(Buffer buffer, RelFileLocator *rlocator, ForkNumber *forknum, BlockNumber *blknum) +BufferGetTag(Buffer buffer, RelFileLocator *rlocator, ForkNumber *forknum, + BlockNumber *blknum) { BufferDesc *bufHdr; @@ -6100,14 +6463,15 @@ BufferGetTag(Buffer buffer, RelFileLocator *rlocator, ForkNumber *forknum, Block * as the second parameter. If not, pass NULL. */ static void -FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, IOContext io_context) +FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, + IOContext io_context) { - XLogRecPtr recptr; + XLogRecPtr recptr; ErrorContextCallback errcallback; - instr_time io_start; - Block bufBlock; - char *bufToWrite; - uint32 buf_state; + instr_time io_start; + Block bufBlock; + char *bufToWrite; + uint32 buf_state; #ifdef USE_PGRAC_CLUSTER @@ -6117,7 +6481,8 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, IOContext io * enter StartBufferIO for old transfer bytes. */ buf_state = LockBufHdr(buf); - if (cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state)) { + if (cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state)) + { UnlockBufHdr(buf, buf_state); return; } @@ -6148,7 +6513,7 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, IOContext io /* Setup error traceback support for ereport() */ errcallback.callback = shared_buffer_write_error_callback; - errcallback.arg = (void *)buf; + errcallback.arg = (void *) buf; errcallback.previous = error_context_stack; error_context_stack = &errcallback; @@ -6156,9 +6521,11 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, IOContext io if (reln == NULL) reln = smgropen(BufTagGetRelFileLocator(&buf->tag), InvalidBackendId); - TRACE_POSTGRESQL_BUFFER_FLUSH_START( - BufTagGetForkNum(&buf->tag), buf->tag.blockNum, reln->smgr_rlocator.locator.spcOid, - reln->smgr_rlocator.locator.dbOid, reln->smgr_rlocator.locator.relNumber); + TRACE_POSTGRESQL_BUFFER_FLUSH_START(BufTagGetForkNum(&buf->tag), + buf->tag.blockNum, + reln->smgr_rlocator.locator.spcOid, + reln->smgr_rlocator.locator.dbOid, + reln->smgr_rlocator.locator.relNumber); buf_state = LockBufHdr(buf); @@ -6207,7 +6574,8 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, IOContext io * the only local additions on top of a shipped image are hint-class * changes (ITL lazy cleanout, index kill bits), which emit no WAL. */ - if (cluster_storage_mode_enabled() && !RecoveryInProgress() && recptr > GetXLogInsertRecPtr()) + if (cluster_storage_mode_enabled() && !RecoveryInProgress() + && recptr > GetXLogInsertRecPtr()) recptr = InvalidXLogRecPtr; #endif if (buf_state & BM_PERMANENT) @@ -6225,14 +6593,18 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, IOContext io * buffer, other processes might be updating hint bits in it, so we must * copy the page to private storage if we do checksumming. */ - bufToWrite = PageSetChecksumCopy((Page)bufBlock, buf->tag.blockNum); + bufToWrite = PageSetChecksumCopy((Page) bufBlock, buf->tag.blockNum); io_start = pgstat_prepare_io_time(); /* * bufToWrite is either the shared buffer or a copy, as appropriate. */ - smgrwrite(reln, BufTagGetForkNum(&buf->tag), buf->tag.blockNum, bufToWrite, false); + smgrwrite(reln, + BufTagGetForkNum(&buf->tag), + buf->tag.blockNum, + bufToWrite, + false); #ifdef USE_PGRAC_CLUSTER @@ -6250,8 +6622,9 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, IOContext io * the master's watermark check and the holder's strict PI-only drop own * correctness). */ - if (cluster_past_image && buf->pcm_state != (uint8)PCM_STATE_N) - cluster_gcs_block_pi_write_note(buf->tag, ((PageHeader)bufToWrite)->pd_block_scn); + if (cluster_past_image && buf->pcm_state != (uint8) PCM_STATE_N) + cluster_gcs_block_pi_write_note(buf->tag, + ((PageHeader) bufToWrite)->pd_block_scn); #endif /* @@ -6272,7 +6645,8 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, IOContext io * When a strategy is not in use, the write can only be a "regular" write * of a dirty shared buffer (IOCONTEXT_NORMAL IOOP_WRITE). */ - pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, IOOP_WRITE, io_start, 1); + pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, + IOOP_WRITE, io_start, 1); pgBufferUsage.shared_blks_written++; @@ -6282,9 +6656,11 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, IOContext io */ TerminateBufferIO(buf, true, 0); - TRACE_POSTGRESQL_BUFFER_FLUSH_DONE( - BufTagGetForkNum(&buf->tag), buf->tag.blockNum, reln->smgr_rlocator.locator.spcOid, - reln->smgr_rlocator.locator.dbOid, reln->smgr_rlocator.locator.relNumber); + TRACE_POSTGRESQL_BUFFER_FLUSH_DONE(BufTagGetForkNum(&buf->tag), + buf->tag.blockNum, + reln->smgr_rlocator.locator.spcOid, + reln->smgr_rlocator.locator.dbOid, + reln->smgr_rlocator.locator.relNumber); /* Pop the error context stack */ error_context_stack = errcallback.previous; @@ -6301,24 +6677,28 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, IOContext io BlockNumber RelationGetNumberOfBlocksInFork(Relation relation, ForkNumber forkNum) { - if (RELKIND_HAS_TABLE_AM(relation->rd_rel->relkind)) { + if (RELKIND_HAS_TABLE_AM(relation->rd_rel->relkind)) + { /* * Not every table AM uses BLCKSZ wide fixed size blocks. Therefore * tableam returns the size in bytes - but for the purpose of this * routine, we want the number of blocks. Therefore divide, rounding * up. */ - uint64 szbytes; + uint64 szbytes; szbytes = table_relation_size(relation, forkNum); return (szbytes + (BLCKSZ - 1)) / BLCKSZ; - } else if (RELKIND_HAS_STORAGE(relation->rd_rel->relkind)) { + } + else if (RELKIND_HAS_STORAGE(relation->rd_rel->relkind)) + { return smgrnblocks(RelationGetSmgr(relation), forkNum); - } else + } + else Assert(false); - return 0; /* keep compiler quiet */ + return 0; /* keep compiler quiet */ } /* @@ -6359,10 +6739,10 @@ BufferIsPermanent(Buffer buffer) XLogRecPtr BufferGetLSNAtomic(Buffer buffer) { - char *page = BufferGetPage(buffer); + char *page = BufferGetPage(buffer); BufferDesc *bufHdr; - XLogRecPtr lsn; - uint32 buf_state; + XLogRecPtr lsn; + uint32 buf_state; /* * If we don't need locking for correctness, fastpath out. @@ -6404,22 +6784,25 @@ BufferGetLSNAtomic(Buffer buffer) * -------------------------------------------------------------------- */ void -DropRelationBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum, int nforks, - BlockNumber *firstDelBlock) +DropRelationBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum, + int nforks, BlockNumber *firstDelBlock) { - int i; - int j; + int i; + int j; RelFileLocatorBackend rlocator; BlockNumber nForkBlock[MAX_FORKNUM]; - uint64 nBlocksToInvalidate = 0; + uint64 nBlocksToInvalidate = 0; rlocator = smgr_reln->smgr_rlocator; /* If it's a local relation, it's localbuf.c's problem. */ - if (RelFileLocatorBackendIsTemp(rlocator)) { - if (rlocator.backend == MyBackendId) { + if (RelFileLocatorBackendIsTemp(rlocator)) + { + if (rlocator.backend == MyBackendId) + { for (j = 0; j < nforks; j++) - DropRelationLocalBuffers(rlocator.locator, forkNum[j], firstDelBlock[j]); + DropRelationLocalBuffers(rlocator.locator, forkNum[j], + firstDelBlock[j]); } return; } @@ -6446,11 +6829,13 @@ DropRelationBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum, int nforks, * that should be fine because there must not be any buffers after that * file size. */ - for (i = 0; i < nforks; i++) { + for (i = 0; i < nforks; i++) + { /* Get the number of blocks for a relation's fork */ nForkBlock[i] = smgrnblocks_cached(smgr_reln, forkNum[i]); - if (nForkBlock[i] == InvalidBlockNumber) { + if (nForkBlock[i] == InvalidBlockNumber) + { nBlocksToInvalidate = InvalidBlockNumber; break; } @@ -6463,17 +6848,19 @@ DropRelationBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum, int nforks, * We apply the optimization iff the total number of blocks to invalidate * is below the BUF_DROP_FULL_SCAN_THRESHOLD. */ - if (BlockNumberIsValid(nBlocksToInvalidate) - && nBlocksToInvalidate < BUF_DROP_FULL_SCAN_THRESHOLD) { + if (BlockNumberIsValid(nBlocksToInvalidate) && + nBlocksToInvalidate < BUF_DROP_FULL_SCAN_THRESHOLD) + { for (j = 0; j < nforks; j++) - FindAndDropRelationBuffers(rlocator.locator, forkNum[j], nForkBlock[j], - firstDelBlock[j]); + FindAndDropRelationBuffers(rlocator.locator, forkNum[j], + nForkBlock[j], firstDelBlock[j]); return; } - for (i = 0; i < NBuffers; i++) { + for (i = 0; i < NBuffers; i++) + { BufferDesc *bufHdr = GetBufferDescriptor(i); - uint32 buf_state; + uint32 buf_state; /* * We can make this a tad faster by prechecking the buffer tag before @@ -6496,11 +6883,13 @@ DropRelationBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum, int nforks, buf_state = LockBufHdr(bufHdr); - for (j = 0; j < nforks; j++) { - if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator.locator) - && BufTagGetForkNum(&bufHdr->tag) == forkNum[j] - && bufHdr->tag.blockNum >= firstDelBlock[j]) { - InvalidateBuffer(bufHdr); /* releases spinlock */ + for (j = 0; j < nforks; j++) + { + if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator.locator) && + BufTagGetForkNum(&bufHdr->tag) == forkNum[j] && + bufHdr->tag.blockNum >= firstDelBlock[j]) + { + InvalidateBuffer(bufHdr); /* releases spinlock */ break; } } @@ -6520,26 +6909,29 @@ DropRelationBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum, int nforks, void DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) { - int i; - int n = 0; + int i; + int n = 0; SMgrRelation *rels; - BlockNumber(*block)[MAX_FORKNUM + 1]; - uint64 nBlocksToInvalidate = 0; + BlockNumber (*block)[MAX_FORKNUM + 1]; + uint64 nBlocksToInvalidate = 0; RelFileLocator *locators; - bool cached = true; - bool use_bsearch; + bool cached = true; + bool use_bsearch; if (nlocators == 0) return; - rels = palloc(sizeof(SMgrRelation) * nlocators); /* non-local relations */ + rels = palloc(sizeof(SMgrRelation) * nlocators); /* non-local relations */ /* If it's a local relation, it's localbuf.c's problem. */ - for (i = 0; i < nlocators; i++) { - if (RelFileLocatorBackendIsTemp(smgr_reln[i]->smgr_rlocator)) { + for (i = 0; i < nlocators; i++) + { + if (RelFileLocatorBackendIsTemp(smgr_reln[i]->smgr_rlocator)) + { if (smgr_reln[i]->smgr_rlocator.backend == MyBackendId) DropRelationAllLocalBuffers(smgr_reln[i]->smgr_rlocator.locator); - } else + } + else rels[n++] = smgr_reln[i]; } @@ -6547,7 +6939,8 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) * If there are no non-local relations, then we're done. Release the * memory and return. */ - if (n == 0) { + if (n == 0) + { pfree(rels); return; } @@ -6556,19 +6949,23 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) * This is used to remember the number of blocks for all the relations * forks. */ - block = (BlockNumber(*)[MAX_FORKNUM + 1]) palloc(sizeof(BlockNumber) * n * (MAX_FORKNUM + 1)); + block = (BlockNumber (*)[MAX_FORKNUM + 1]) + palloc(sizeof(BlockNumber) * n * (MAX_FORKNUM + 1)); /* * We can avoid scanning the entire buffer pool if we know the exact size * of each of the given relation forks. See DropRelationBuffers. */ - for (i = 0; i < n && cached; i++) { - for (int j = 0; j <= MAX_FORKNUM; j++) { + for (i = 0; i < n && cached; i++) + { + for (int j = 0; j <= MAX_FORKNUM; j++) + { /* Get the number of blocks for a relation's fork. */ block[i][j] = smgrnblocks_cached(rels[i], j); /* We need to only consider the relation forks that exists. */ - if (block[i][j] == InvalidBlockNumber) { + if (block[i][j] == InvalidBlockNumber) + { if (!smgrexists(rels[i], j)) continue; cached = false; @@ -6584,15 +6981,19 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) * We apply the optimization iff the total number of blocks to invalidate * is below the BUF_DROP_FULL_SCAN_THRESHOLD. */ - if (cached && nBlocksToInvalidate < BUF_DROP_FULL_SCAN_THRESHOLD) { - for (i = 0; i < n; i++) { - for (int j = 0; j <= MAX_FORKNUM; j++) { + if (cached && nBlocksToInvalidate < BUF_DROP_FULL_SCAN_THRESHOLD) + { + for (i = 0; i < n; i++) + { + for (int j = 0; j <= MAX_FORKNUM; j++) + { /* ignore relation forks that doesn't exist */ if (!BlockNumberIsValid(block[i][j])) continue; /* drop all the buffers for a particular relation fork */ - FindAndDropRelationBuffers(rels[i]->smgr_rlocator.locator, j, block[i][j], 0); + FindAndDropRelationBuffers(rels[i]->smgr_rlocator.locator, + j, block[i][j], 0); } } @@ -6602,7 +7003,7 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) } pfree(block); - locators = palloc(sizeof(RelFileLocator) * n); /* non-local relations */ + locators = palloc(sizeof(RelFileLocator) * n); /* non-local relations */ for (i = 0; i < n; i++) locators[i] = rels[i]->smgr_rlocator.locator; @@ -6618,30 +7019,37 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) if (use_bsearch) pg_qsort(locators, n, sizeof(RelFileLocator), rlocator_comparator); - for (i = 0; i < NBuffers; i++) { + for (i = 0; i < NBuffers; i++) + { RelFileLocator *rlocator = NULL; BufferDesc *bufHdr = GetBufferDescriptor(i); - uint32 buf_state; + uint32 buf_state; /* * As in DropRelationBuffers, an unlocked precheck should be safe and * saves some cycles. */ - if (!use_bsearch) { - int j; + if (!use_bsearch) + { + int j; - for (j = 0; j < n; j++) { - if (BufTagMatchesRelFileLocator(&bufHdr->tag, &locators[j])) { + for (j = 0; j < n; j++) + { + if (BufTagMatchesRelFileLocator(&bufHdr->tag, &locators[j])) + { rlocator = &locators[j]; break; } } - } else { + } + else + { RelFileLocator locator; locator = BufTagGetRelFileLocator(&bufHdr->tag); - rlocator = bsearch((const void *)&(locator), locators, n, sizeof(RelFileLocator), + rlocator = bsearch((const void *) &(locator), + locators, n, sizeof(RelFileLocator), rlocator_comparator); } @@ -6651,7 +7059,7 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) buf_state = LockBufHdr(bufHdr); if (BufTagMatchesRelFileLocator(&bufHdr->tag, rlocator)) - InvalidateBuffer(bufHdr); /* releases spinlock */ + InvalidateBuffer(bufHdr); /* releases spinlock */ else UnlockBufHdr(bufHdr, buf_state); } @@ -6670,18 +7078,20 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) * -------------------------------------------------------------------- */ static void -FindAndDropRelationBuffers(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber nForkBlock, +FindAndDropRelationBuffers(RelFileLocator rlocator, ForkNumber forkNum, + BlockNumber nForkBlock, BlockNumber firstDelBlock) { BlockNumber curBlock; - for (curBlock = firstDelBlock; curBlock < nForkBlock; curBlock++) { - uint32 bufHash; /* hash value for tag */ - BufferTag bufTag; /* identity of requested block */ - LWLock *bufPartitionLock; /* buffer partition lock for it */ - int buf_id; + for (curBlock = firstDelBlock; curBlock < nForkBlock; curBlock++) + { + uint32 bufHash; /* hash value for tag */ + BufferTag bufTag; /* identity of requested block */ + LWLock *bufPartitionLock; /* buffer partition lock for it */ + int buf_id; BufferDesc *bufHdr; - uint32 buf_state; + uint32 buf_state; /* create a tag so we can lookup the buffer */ InitBufferTag(&bufTag, &rlocator, forkNum, curBlock); @@ -6708,9 +7118,10 @@ FindAndDropRelationBuffers(RelFileLocator rlocator, ForkNumber forkNum, BlockNum */ buf_state = LockBufHdr(bufHdr); - if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator) - && BufTagGetForkNum(&bufHdr->tag) == forkNum && bufHdr->tag.blockNum >= firstDelBlock) - InvalidateBuffer(bufHdr); /* releases spinlock */ + if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator) && + BufTagGetForkNum(&bufHdr->tag) == forkNum && + bufHdr->tag.blockNum >= firstDelBlock) + InvalidateBuffer(bufHdr); /* releases spinlock */ else UnlockBufHdr(bufHdr, buf_state); } @@ -6730,16 +7141,17 @@ FindAndDropRelationBuffers(RelFileLocator rlocator, ForkNumber forkNum, BlockNum void DropDatabaseBuffers(Oid dbid) { - int i; + int i; /* * We needn't consider local buffers, since by assumption the target * database isn't our own. */ - for (i = 0; i < NBuffers; i++) { + for (i = 0; i < NBuffers; i++) + { BufferDesc *bufHdr = GetBufferDescriptor(i); - uint32 buf_state; + uint32 buf_state; /* * As in DropRelationBuffers, an unlocked precheck should be safe and @@ -6750,7 +7162,7 @@ DropDatabaseBuffers(Oid dbid) buf_state = LockBufHdr(bufHdr); if (bufHdr->tag.dbOid == dbid) - InvalidateBuffer(bufHdr); /* releases spinlock */ + InvalidateBuffer(bufHdr); /* releases spinlock */ else UnlockBufHdr(bufHdr, buf_state); } @@ -6767,20 +7179,22 @@ DropDatabaseBuffers(Oid dbid) void PrintBufferDescs(void) { - int i; + int i; - for (i = 0; i < NBuffers; ++i) { + for (i = 0; i < NBuffers; ++i) + { BufferDesc *buf = GetBufferDescriptor(i); - Buffer b = BufferDescriptorGetBuffer(buf); + Buffer b = BufferDescriptorGetBuffer(buf); /* theoretically we should lock the bufhdr here */ elog(LOG, "[%02d] (freeNext=%d, rel=%s, " "blockNum=%u, flags=0x%x, refcount=%u %d)", i, buf->freeNext, - relpathbackend(BufTagGetRelFileLocator(&buf->tag), InvalidBackendId, - BufTagGetForkNum(&buf->tag)), - buf->tag.blockNum, buf->flags, buf->refcount, GetPrivateRefCount(b)); + relpathbackend(BufTagGetRelFileLocator(&buf->tag), + InvalidBackendId, BufTagGetForkNum(&buf->tag)), + buf->tag.blockNum, buf->flags, + buf->refcount, GetPrivateRefCount(b)); } } #endif @@ -6789,20 +7203,24 @@ PrintBufferDescs(void) void PrintPinnedBufs(void) { - int i; + int i; - for (i = 0; i < NBuffers; ++i) { + for (i = 0; i < NBuffers; ++i) + { BufferDesc *buf = GetBufferDescriptor(i); - Buffer b = BufferDescriptorGetBuffer(buf); + Buffer b = BufferDescriptorGetBuffer(buf); - if (GetPrivateRefCount(b) > 0) { + if (GetPrivateRefCount(b) > 0) + { /* theoretically we should lock the bufhdr here */ elog(LOG, "[%02d] (freeNext=%d, rel=%s, " "blockNum=%u, flags=0x%x, refcount=%u %d)", i, buf->freeNext, - relpathperm(BufTagGetRelFileLocator(&buf->tag), BufTagGetForkNum(&buf->tag)), - buf->tag.blockNum, buf->flags, buf->refcount, GetPrivateRefCount(b)); + relpathperm(BufTagGetRelFileLocator(&buf->tag), + BufTagGetForkNum(&buf->tag)), + buf->tag.blockNum, buf->flags, + buf->refcount, GetPrivateRefCount(b)); } } } @@ -6829,26 +7247,29 @@ PrintPinnedBufs(void) void FlushRelationBuffers(Relation rel) { - int i; + int i; BufferDesc *bufHdr; - if (RelationUsesLocalBuffers(rel)) { - for (i = 0; i < NLocBuffer; i++) { - uint32 buf_state; - instr_time io_start; + if (RelationUsesLocalBuffers(rel)) + { + for (i = 0; i < NLocBuffer; i++) + { + uint32 buf_state; + instr_time io_start; bufHdr = GetLocalBufferDescriptor(i); - if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator) - && ((buf_state = pg_atomic_read_u32(&bufHdr->state)) & (BM_VALID | BM_DIRTY)) - == (BM_VALID | BM_DIRTY)) { + if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator) && + ((buf_state = pg_atomic_read_u32(&bufHdr->state)) & + (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY)) + { ErrorContextCallback errcallback; - Page localpage; + Page localpage; - localpage = (char *)LocalBufHdrGetBlock(bufHdr); + localpage = (char *) LocalBufHdrGetBlock(bufHdr); /* Setup error traceback support for ereport() */ errcallback.callback = local_buffer_write_error_callback; - errcallback.arg = (void *)bufHdr; + errcallback.arg = (void *) bufHdr; errcallback.previous = error_context_stack; error_context_stack = &errcallback; @@ -6856,10 +7277,14 @@ FlushRelationBuffers(Relation rel) io_start = pgstat_prepare_io_time(); - smgrwrite(RelationGetSmgr(rel), BufTagGetForkNum(&bufHdr->tag), - bufHdr->tag.blockNum, localpage, false); + smgrwrite(RelationGetSmgr(rel), + BufTagGetForkNum(&bufHdr->tag), + bufHdr->tag.blockNum, + localpage, + false); - pgstat_count_io_op_time(IOOBJECT_TEMP_RELATION, IOCONTEXT_NORMAL, IOOP_WRITE, + pgstat_count_io_op_time(IOOBJECT_TEMP_RELATION, + IOCONTEXT_NORMAL, IOOP_WRITE, io_start, 1); buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED); @@ -6878,8 +7303,9 @@ FlushRelationBuffers(Relation rel) /* Make sure we can handle the pin inside the loop */ ResourceOwnerEnlargeBuffers(CurrentResourceOwner); - for (i = 0; i < NBuffers; i++) { - uint32 buf_state; + for (i = 0; i < NBuffers; i++) + { + uint32 buf_state; bufHdr = GetBufferDescriptor(i); @@ -6893,14 +7319,16 @@ FlushRelationBuffers(Relation rel) ReservePrivateRefCountEntry(); buf_state = LockBufHdr(bufHdr); - if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator) - && (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY)) { + if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator) && + (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY)) + { PinBuffer_Locked(bufHdr); LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED); FlushBuffer(bufHdr, RelationGetSmgr(rel), IOOBJECT_RELATION, IOCONTEXT_NORMAL); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); UnpinBuffer(bufHdr); - } else + } + else UnlockBufHdr(bufHdr, buf_state); } } @@ -6917,9 +7345,9 @@ FlushRelationBuffers(Relation rel) void FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels) { - int i; + int i; SMgrSortArray *srels; - bool use_bsearch; + bool use_bsearch; if (nrels == 0) return; @@ -6927,7 +7355,8 @@ FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels) /* fill-in array for qsort */ srels = palloc(sizeof(SMgrSortArray) * nrels); - for (i = 0; i < nrels; i++) { + for (i = 0; i < nrels; i++) + { Assert(!RelFileLocatorBackendIsTemp(smgrs[i]->smgr_rlocator)); srels[i].rlocator = smgrs[i]->smgr_rlocator.locator; @@ -6947,30 +7376,37 @@ FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels) /* Make sure we can handle the pin inside the loop */ ResourceOwnerEnlargeBuffers(CurrentResourceOwner); - for (i = 0; i < NBuffers; i++) { + for (i = 0; i < NBuffers; i++) + { SMgrSortArray *srelent = NULL; BufferDesc *bufHdr = GetBufferDescriptor(i); - uint32 buf_state; + uint32 buf_state; /* * As in DropRelationBuffers, an unlocked precheck should be safe and * saves some cycles. */ - if (!use_bsearch) { - int j; + if (!use_bsearch) + { + int j; - for (j = 0; j < nrels; j++) { - if (BufTagMatchesRelFileLocator(&bufHdr->tag, &srels[j].rlocator)) { + for (j = 0; j < nrels; j++) + { + if (BufTagMatchesRelFileLocator(&bufHdr->tag, &srels[j].rlocator)) + { srelent = &srels[j]; break; } } - } else { + } + else + { RelFileLocator rlocator; rlocator = BufTagGetRelFileLocator(&bufHdr->tag); - srelent = bsearch((const void *)&(rlocator), srels, nrels, sizeof(SMgrSortArray), + srelent = bsearch((const void *) &(rlocator), + srels, nrels, sizeof(SMgrSortArray), rlocator_comparator); } @@ -6981,14 +7417,16 @@ FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels) ReservePrivateRefCountEntry(); buf_state = LockBufHdr(bufHdr); - if (BufTagMatchesRelFileLocator(&bufHdr->tag, &srelent->rlocator) - && (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY)) { + if (BufTagMatchesRelFileLocator(&bufHdr->tag, &srelent->rlocator) && + (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY)) + { PinBuffer_Locked(bufHdr); LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED); FlushBuffer(bufHdr, srelent->srel, IOOBJECT_RELATION, IOCONTEXT_NORMAL); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); UnpinBuffer(bufHdr); - } else + } + else UnlockBufHdr(bufHdr, buf_state); } @@ -7006,14 +7444,15 @@ FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels) * -------------------------------------------------------------------- */ static void -RelationCopyStorageUsingBuffer(RelFileLocator srclocator, RelFileLocator dstlocator, +RelationCopyStorageUsingBuffer(RelFileLocator srclocator, + RelFileLocator dstlocator, ForkNumber forkNum, bool permanent) { - Buffer srcBuf; - Buffer dstBuf; - Page srcPage; - Page dstPage; - bool use_wal; + Buffer srcBuf; + Buffer dstBuf; + Page srcPage; + Page dstPage; + bool use_wal; BlockNumber nblocks; BlockNumber blkno; PGIOAlignedBlock buf; @@ -7028,7 +7467,8 @@ RelationCopyStorageUsingBuffer(RelFileLocator srclocator, RelFileLocator dstloca use_wal = XLogIsNeeded() && (permanent || forkNum == INIT_FORKNUM); /* Get number of blocks in the source relation. */ - nblocks = smgrnblocks(smgropen(srclocator, InvalidBackendId), forkNum); + nblocks = smgrnblocks(smgropen(srclocator, InvalidBackendId), + forkNum); /* Nothing to copy; just return. */ if (nblocks == 0) @@ -7039,24 +7479,28 @@ RelationCopyStorageUsingBuffer(RelFileLocator srclocator, RelFileLocator dstloca * relation before starting to copy block by block. */ memset(buf.data, 0, BLCKSZ); - smgrextend(smgropen(dstlocator, InvalidBackendId), forkNum, nblocks - 1, buf.data, true); + smgrextend(smgropen(dstlocator, InvalidBackendId), forkNum, nblocks - 1, + buf.data, true); /* This is a bulk operation, so use buffer access strategies. */ bstrategy_src = GetAccessStrategy(BAS_BULKREAD); bstrategy_dst = GetAccessStrategy(BAS_BULKWRITE); /* Iterate over each block of the source relation file. */ - for (blkno = 0; blkno < nblocks; blkno++) { + for (blkno = 0; blkno < nblocks; blkno++) + { CHECK_FOR_INTERRUPTS(); /* Read block from source relation. */ - srcBuf = ReadBufferWithoutRelcache(srclocator, forkNum, blkno, RBM_NORMAL, bstrategy_src, + srcBuf = ReadBufferWithoutRelcache(srclocator, forkNum, blkno, + RBM_NORMAL, bstrategy_src, permanent); LockBuffer(srcBuf, BUFFER_LOCK_SHARE); srcPage = BufferGetPage(srcBuf); - dstBuf = ReadBufferWithoutRelcache(dstlocator, forkNum, blkno, RBM_ZERO_AND_LOCK, - bstrategy_dst, permanent); + dstBuf = ReadBufferWithoutRelcache(dstlocator, forkNum, blkno, + RBM_ZERO_AND_LOCK, bstrategy_dst, + permanent); dstPage = BufferGetPage(dstBuf); START_CRIT_SECTION(); @@ -7091,13 +7535,15 @@ RelationCopyStorageUsingBuffer(RelFileLocator srclocator, RelFileLocator dstloca * -------------------------------------------------------------------- */ void -CreateAndCopyRelationData(RelFileLocator src_rlocator, RelFileLocator dst_rlocator, bool permanent) +CreateAndCopyRelationData(RelFileLocator src_rlocator, + RelFileLocator dst_rlocator, bool permanent) { RelFileLocatorBackend rlocator; - char relpersistence; + char relpersistence; /* Set the relpersistence. */ - relpersistence = permanent ? RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED; + relpersistence = permanent ? + RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED; /* * Create and copy all forks of the relation. During create database we @@ -7108,11 +7554,15 @@ CreateAndCopyRelationData(RelFileLocator src_rlocator, RelFileLocator dst_rlocat RelationCreateStorage(dst_rlocator, relpersistence, false); /* copy main fork. */ - RelationCopyStorageUsingBuffer(src_rlocator, dst_rlocator, MAIN_FORKNUM, permanent); + RelationCopyStorageUsingBuffer(src_rlocator, dst_rlocator, MAIN_FORKNUM, + permanent); /* copy those extra forks that exist */ - for (ForkNumber forkNum = MAIN_FORKNUM + 1; forkNum <= MAX_FORKNUM; forkNum++) { - if (smgrexists(smgropen(src_rlocator, InvalidBackendId), forkNum)) { + for (ForkNumber forkNum = MAIN_FORKNUM + 1; + forkNum <= MAX_FORKNUM; forkNum++) + { + if (smgrexists(smgropen(src_rlocator, InvalidBackendId), forkNum)) + { smgrcreate(smgropen(dst_rlocator, InvalidBackendId), forkNum, false); /* @@ -7123,7 +7573,8 @@ CreateAndCopyRelationData(RelFileLocator src_rlocator, RelFileLocator dst_rlocat log_smgrcreate(&dst_rlocator, forkNum); /* Copy a fork's data, block by block. */ - RelationCopyStorageUsingBuffer(src_rlocator, dst_rlocator, forkNum, permanent); + RelationCopyStorageUsingBuffer(src_rlocator, dst_rlocator, forkNum, + permanent); } } @@ -7155,14 +7606,15 @@ CreateAndCopyRelationData(RelFileLocator src_rlocator, RelFileLocator dst_rlocat void FlushDatabaseBuffers(Oid dbid) { - int i; + int i; BufferDesc *bufHdr; /* Make sure we can handle the pin inside the loop */ ResourceOwnerEnlargeBuffers(CurrentResourceOwner); - for (i = 0; i < NBuffers; i++) { - uint32 buf_state; + for (i = 0; i < NBuffers; i++) + { + uint32 buf_state; bufHdr = GetBufferDescriptor(i); @@ -7176,14 +7628,16 @@ FlushDatabaseBuffers(Oid dbid) ReservePrivateRefCountEntry(); buf_state = LockBufHdr(bufHdr); - if (bufHdr->tag.dbOid == dbid - && (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY)) { + if (bufHdr->tag.dbOid == dbid && + (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY)) + { PinBuffer_Locked(bufHdr); LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED); FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); UnpinBuffer(bufHdr); - } else + } + else UnlockBufHdr(bufHdr, buf_state); } } @@ -7251,7 +7705,8 @@ IncrBufferRefCount(Buffer buffer) ResourceOwnerEnlargeBuffers(CurrentResourceOwner); if (BufferIsLocal(buffer)) LocalRefCount[-buffer - 1]++; - else { + else + { PrivateRefCountEntry *ref; ref = GetPrivateRefCountEntry(buffer, true); @@ -7279,15 +7734,16 @@ void MarkBufferDirtyHint(Buffer buffer, bool buffer_std) { BufferDesc *bufHdr; - Page page = BufferGetPage(buffer); + Page page = BufferGetPage(buffer); #ifdef USE_PGRAC_CLUSTER - uint32 retained_state; + uint32 retained_state; #endif if (!BufferIsValid(buffer)) elog(ERROR, "bad buffer ID: %d", buffer); - if (BufferIsLocal(buffer)) { + if (BufferIsLocal(buffer)) + { MarkLocalBufferDirty(buffer); return; } @@ -7306,7 +7762,8 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) * dirty state and become eligible for stale output. */ retained_state = LockBufHdr(bufHdr); - if (cluster_bufmgr_pcm_x_retained_image_locked(bufHdr, retained_state)) { + if (cluster_bufmgr_pcm_x_retained_image_locked(bufHdr, retained_state)) + { UnlockBufHdr(bufHdr, retained_state); return; } @@ -7324,12 +7781,13 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) * is only intended to be used in cases where failing to write out the * data would be harmless anyway, it doesn't really matter. */ - if ((pg_atomic_read_u32(&bufHdr->state) & (BM_DIRTY | BM_JUST_DIRTIED)) - != (BM_DIRTY | BM_JUST_DIRTIED)) { - XLogRecPtr lsn = InvalidXLogRecPtr; - bool dirtied = false; - bool delayChkptFlags = false; - uint32 buf_state; + if ((pg_atomic_read_u32(&bufHdr->state) & (BM_DIRTY | BM_JUST_DIRTIED)) != + (BM_DIRTY | BM_JUST_DIRTIED)) + { + XLogRecPtr lsn = InvalidXLogRecPtr; + bool dirtied = false; + bool delayChkptFlags = false; + uint32 buf_state; /* * If we need to protect hint bit updates from torn writes, WAL-log a @@ -7340,7 +7798,9 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) * We don't check full_page_writes here because that logic is included * when we call XLogInsert() since the value changes dynamically. */ - if (XLogHintBitIsNeeded() && (pg_atomic_read_u32(&bufHdr->state) & BM_PERMANENT)) { + if (XLogHintBitIsNeeded() && + (pg_atomic_read_u32(&bufHdr->state) & BM_PERMANENT)) + { /* * If we must not write WAL, due to a relfilelocator-specific * condition or being in recovery, don't dirty the page. We can @@ -7349,8 +7809,8 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) * * See src/backend/storage/page/README for longer discussion. */ - if (RecoveryInProgress() - || RelFileLocatorSkippingWAL(BufTagGetRelFileLocator(&bufHdr->tag))) + if (RecoveryInProgress() || + RelFileLocatorSkippingWAL(BufTagGetRelFileLocator(&bufHdr->tag))) return; /* @@ -7386,8 +7846,9 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0); - if (!(buf_state & BM_DIRTY)) { - dirtied = true; /* Means "will be dirtied by this action" */ + if (!(buf_state & BM_DIRTY)) + { + dirtied = true; /* Means "will be dirtied by this action" */ /* * Set the page LSN if we wrote a backup block. We aren't supposed @@ -7412,7 +7873,8 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) if (delayChkptFlags) MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; - if (dirtied) { + if (dirtied) + { VacuumPageDirty++; pgBufferUsage.shared_blks_dirtied++; if (VacuumCostActive) @@ -7435,8 +7897,9 @@ UnlockBuffers(void) { BufferDesc *buf = PinCountWaitBuf; - if (buf) { - uint32 buf_state; + if (buf) + { + uint32 buf_state; buf_state = LockBufHdr(buf); @@ -7444,8 +7907,8 @@ UnlockBuffers(void) * Don't complain if flag bit not set; it could have been reset but we * got a cancel/die interrupt before getting the signal. */ - if ((buf_state & BM_PIN_COUNT_WAITER) != 0 - && buf->wait_backend_pgprocno == MyProc->pgprocno) + if ((buf_state & BM_PIN_COUNT_WAITER) != 0 && + buf->wait_backend_pgprocno == MyProc->pgprocno) buf_state &= ~BM_PIN_COUNT_WAITER; UnlockBufHdr(buf, buf_state); @@ -7470,16 +7933,16 @@ LockBuffer(Buffer buffer, int mode) { BufferDesc *buf; #ifdef USE_PGRAC_CLUSTER - bool pcm_acquired = false; + bool pcm_acquired = false; PcmLockMode pcm_mode = PCM_LOCK_MODE_N; - bool pcm_covered = false; /* took the cached-cover fast path */ - uint64 pcm_covered_gen = 0; /* ownership generation captured at cover */ - bool pcm_pending_set = false; /* we set GRANT_PENDING (W3), must clear */ + bool pcm_covered = false; /* took the cached-cover fast path */ + uint64 pcm_covered_gen = 0; /* ownership generation captured at cover */ + bool pcm_pending_set = false; /* we set GRANT_PENDING (W3), must clear */ MemoryContext pcm_error_context = CurrentMemoryContext; ClusterPcmOwnSnapshot pcm_pending_base; ClusterPcmOwnResult pcm_pending_result = CLUSTER_PCM_OWN_OK; - uint64 pcm_pending_token = 0; - uint64 pcm_committed_generation = 0; + uint64 pcm_pending_token = 0; + uint64 pcm_committed_generation = 0; ClusterPcmXHolderLedgerEntry *pcm_x_holder = NULL; ClusterPcmXWriterLedgerEntry *pcm_x_writer = NULL; bool pcm_x_writer_managed = false; @@ -7487,18 +7950,22 @@ LockBuffer(Buffer buffer, int mode) Assert(BufferIsPinned(buffer)); if (BufferIsLocal(buffer)) - return; /* local buffers need no lock */ + return; /* local buffers need no lock */ buf = GetBufferDescriptor(buffer - 1); - if (mode != BUFFER_LOCK_UNLOCK && mode != BUFFER_LOCK_SHARE && mode != BUFFER_LOCK_EXCLUSIVE) + if (mode != BUFFER_LOCK_UNLOCK && + mode != BUFFER_LOCK_SHARE && + mode != BUFFER_LOCK_EXCLUSIVE) elog(ERROR, "unrecognized buffer lock mode: %d", mode); #ifdef USE_PGRAC_CLUSTER - if (mode != BUFFER_LOCK_UNLOCK && cluster_pcm_is_active() - && cluster_bufmgr_should_pcm_track(buf)) { - uint8 pcm_initial_state = (uint8)PCM_STATE_N; - uint32 pcm_initial_flags = 0; + if (mode != BUFFER_LOCK_UNLOCK && + cluster_pcm_is_active() && + cluster_bufmgr_should_pcm_track(buf)) + { + uint8 pcm_initial_state = (uint8)PCM_STATE_N; + uint32 pcm_initial_flags = 0; pcm_mode = (mode == BUFFER_LOCK_SHARE) ? PCM_LOCK_MODE_S : PCM_LOCK_MODE_X; @@ -7508,10 +7975,10 @@ LockBuffer(Buffer buffer, int mode) * re-enqueues the same node X. The post-content-lock generation * check below remains the race-closing authority. */ if (pcm_mode == PCM_LOCK_MODE_X && cluster_gcs_block_local_cache) { - cluster_pcm_own_read(buf, &pcm_initial_state, &pcm_covered_gen, &pcm_initial_flags); - pcm_covered = cluster_pcm_x_cached_cover_bypasses_queue( - cluster_gcs_block_local_cache, pcm_mode == PCM_LOCK_MODE_X, pcm_initial_state, - pcm_initial_flags); + cluster_pcm_own_read(buf, &pcm_initial_state, &pcm_covered_gen, + &pcm_initial_flags); + pcm_covered = cluster_pcm_x_cached_cover_bypasses_queue(cluster_gcs_block_local_cache, + pcm_mode == PCM_LOCK_MODE_X, pcm_initial_state, pcm_initial_flags); } if (!pcm_covered) @@ -7621,7 +8088,8 @@ LockBuffer(Buffer buffer, int mode) } #endif - if (mode == BUFFER_LOCK_UNLOCK) { + if (mode == BUFFER_LOCK_UNLOCK) + { #ifdef USE_PGRAC_CLUSTER pcm_x_writer = cluster_bufmgr_pcm_x_writer_find(buf); pcm_x_writer_managed = pcm_x_writer != NULL; @@ -7634,7 +8102,9 @@ LockBuffer(Buffer buffer, int mode) cluster_bufmgr_pcm_x_writer_release(pcm_x_writer); cluster_bufmgr_pcm_x_holder_unregister(pcm_x_holder); #endif - } else { + } + else + { #ifdef USE_PGRAC_CLUSTER /* * GCS serve-stall round-6 (ownership P0) — cached-X BAST window. @@ -7679,15 +8149,17 @@ LockBuffer(Buffer buffer, int mode) * content lock and the downgrade path serializes under it, so no * further downgrade can intervene -- at most one fallback. */ - if (pcm_covered) { - uint8 cur_state; - uint64 cur_gen; - uint32 cur_flags; + if (pcm_covered) + { + uint8 cur_state; + uint64 cur_gen; + uint32 cur_flags; cluster_pcm_own_read(buf, &cur_state, &cur_gen, &cur_flags); if (cur_gen != pcm_covered_gen - || !cluster_pcm_mode_covers((PcmLockMode)cur_state, pcm_mode) - || (cur_flags & (PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING)) != 0) { + || !cluster_pcm_mode_covers((PcmLockMode) cur_state, pcm_mode) + || (cur_flags & (PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING)) != 0) + { cluster_pcm_note_writer_cover_stale_detected(); pcm_covered = false; LWLockRelease(BufferDescriptorGetContentLock(buf)); @@ -7711,7 +8183,7 @@ LockBuffer(Buffer buffer, int mode) } PG_CATCH(); { - ErrorData *original_error; + ErrorData *original_error; /* * Holder detach is itself allowed to encounter an LWLock ERROR. @@ -7747,7 +8219,7 @@ LockBuffer(Buffer buffer, int mode) */ else if (pcm_pending_set) cluster_pcm_own_abort_grant_after_error(buf, &pcm_pending_base, pcm_pending_token, - "LockBuffer content-lock acquire"); + "LockBuffer content-lock acquire"); ReThrowError(original_error); } PG_END_TRY(); @@ -7760,7 +8232,8 @@ LockBuffer(Buffer buffer, int mode) * prove the GRANT_PENDING consult parks it instead of acking the * grant away. */ - if (pcm_acquired && pcm_pending_set && pcm_mode == PCM_LOCK_MODE_X) { + if (pcm_acquired && pcm_pending_set && pcm_mode == PCM_LOCK_MODE_X) + { CLUSTER_INJECTION_POINT("cluster-pcm-grant-finalize-window"); /* @@ -7776,14 +8249,17 @@ LockBuffer(Buffer buffer, int mode) * W3 defect. */ CLUSTER_INJECTION_POINT("cluster-pcm-grant-finalize-deliver-invalidate"); - if (cluster_injection_should_skip("cluster-pcm-grant-finalize-deliver-invalidate")) { + if (cluster_injection_should_skip( + "cluster-pcm-grant-finalize-deliver-invalidate")) + { if (cluster_gcs_block_test_deliver_self_invalidate(buf->tag)) - elog(WARNING, "cluster W3 delivery shim: synthetic INVALIDATE was ACKed " - "instead of parked (GRANT_PENDING not honored)"); + elog(WARNING, + "cluster W3 delivery shim: synthetic INVALIDATE was ACKed instead of parked (GRANT_PENDING not honored)"); } } - if (pcm_acquired) { + if (pcm_acquired) + { /* * Finalize the grant under the header spinlock: set buffer_type + * pcm_state, bump the ownership generation, and clear @@ -7793,9 +8269,12 @@ LockBuffer(Buffer buffer, int mode) */ cluster_pcm_own_finish_grant_or_rollback( buf, &pcm_pending_base, pcm_pending_token, - (pcm_mode == PCM_LOCK_MODE_S) ? (uint8)PCM_STATE_S : (uint8)PCM_STATE_X, pcm_mode, + (pcm_mode == PCM_LOCK_MODE_S) ? (uint8) PCM_STATE_S : (uint8) PCM_STATE_X, + pcm_mode, &pcm_committed_generation); - } else if (pcm_pending_set) { + } + else if (pcm_pending_set) + { /* No durable grant (one-shot READ_IMAGE): clear the PENDING marker * we set before the acquire so it does not linger. */ cluster_pcm_own_abort_grant_or_error(buf, &pcm_pending_base, pcm_pending_token, @@ -7816,18 +8295,22 @@ LockBuffer(Buffer buffer, int mode) * LockBuffer re-acquires (cluster_pcm_mode_covers(N, ...) is false), * getting real X once the remote holder is terminal. */ - if (mode == BUFFER_LOCK_UNLOCK && buf->pcm_state == (uint8)PCM_STATE_READ_IMAGE) - cluster_pcm_own_transition(buf, (uint8)PCM_STATE_N, 0, 0); + if (mode == BUFFER_LOCK_UNLOCK + && buf->pcm_state == (uint8) PCM_STATE_READ_IMAGE) + cluster_pcm_own_transition(buf, (uint8) PCM_STATE_N, 0, 0); - if (mode == BUFFER_LOCK_UNLOCK && cluster_pcm_is_active() - && cluster_bufmgr_should_pcm_track(buf) && buf->pcm_state != (uint8)PCM_STATE_N) { + if (mode == BUFFER_LOCK_UNLOCK && + cluster_pcm_is_active() && + cluster_bufmgr_should_pcm_track(buf) && + buf->pcm_state != (uint8) PCM_STATE_N) + { /* PGRAC: spec-2.35 D4 (HC111 + HC112) — content-lock unlock path. * SCUR preserves cache residency (bit stays set so CF 2-way * forward can still target this node). XCUR delegates to the * eviction release (single-holder semantic preserved). Real * cache eviction is handled by the InvalidateBuffer / * InvalidateVictimBuffer / Drop*Buffers hook points (below). */ - PcmLockMode old_mode = (PcmLockMode)buf->pcm_state; + PcmLockMode old_mode = (PcmLockMode) buf->pcm_state; /* * PGRAC: spec-4.7a D2 — hold-until-revoked. With the node-level @@ -7871,10 +8354,11 @@ LockBufferForAuxiliaryPageInit(Buffer buffer, ClusterPcmDirectInitKind kind) return; buf = GetBufferDescriptor(buffer - 1); - if (cluster_pcm_is_active() && cluster_bufmgr_should_pcm_track(buf)) { + if (cluster_pcm_is_active() && cluster_bufmgr_should_pcm_track(buf)) + { ClusterPcmDirectInitProof direct_init_proof; - uint8 pcm_state; - uint32 flags; + uint8 pcm_state; + uint32 flags; /* * A concurrent initializer may already have established X while the @@ -7882,7 +8366,8 @@ LockBufferForAuxiliaryPageInit(Buffer buffer, ClusterPcmDirectInitKind kind) * path revalidates that grant after taking the content lock. */ cluster_pcm_own_read(buf, &pcm_state, NULL, &flags); - if (pcm_state == (uint8)PCM_STATE_X && flags == 0) { + if (pcm_state == (uint8) PCM_STATE_X && flags == 0) + { LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); return; } @@ -7931,21 +8416,23 @@ bool ConditionalLockBuffer(Buffer buffer) { BufferDesc *buf; - bool acquired; + bool acquired; #ifdef USE_PGRAC_CLUSTER - bool blocked; - uint32 buf_state; + bool blocked; + uint32 buf_state; #endif Assert(BufferIsPinned(buffer)); if (BufferIsLocal(buffer)) - return true; /* act as though we got it */ + return true; /* act as though we got it */ buf = GetBufferDescriptor(buffer - 1); - acquired = LWLockConditionalAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE); + acquired = LWLockConditionalAcquire(BufferDescriptorGetContentLock(buf), + LW_EXCLUSIVE); #ifdef USE_PGRAC_CLUSTER - if (acquired) { + if (acquired) + { /* This API bypasses LockBuffer/W1. Recreate its reservation boundary * after acquiring content authority: an already-running user wins and * serializes before revoke finish; a user arriving after REVOKING or @@ -7957,7 +8444,8 @@ ConditionalLockBuffer(Buffer buffer) cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state), buf->pcm_state, cluster_pcm_own_flags_get(buf->buf_id)); UnlockBufHdr(buf, buf_state); - if (blocked) { + if (blocked) + { LWLockRelease(BufferDescriptorGetContentLock(buf)); return false; } @@ -7975,12 +8463,17 @@ ConditionalLockBuffer(Buffer buffer) void CheckBufferIsPinnedOnce(Buffer buffer) { - if (BufferIsLocal(buffer)) { + if (BufferIsLocal(buffer)) + { if (LocalRefCount[-buffer - 1] != 1) - elog(ERROR, "incorrect local pin count: %d", LocalRefCount[-buffer - 1]); - } else { - if (GetPrivateRefCount(buffer) != 1) - elog(ERROR, "incorrect local pin count: %d", GetPrivateRefCount(buffer)); + elog(ERROR, "incorrect local pin count: %d", + LocalRefCount[-buffer - 1]); + } + else + { + if (GetPrivateRefCount(buffer) != 1) + elog(ERROR, "incorrect local pin count: %d", + GetPrivateRefCount(buffer)); } } @@ -8010,8 +8503,8 @@ LockBufferForCleanup(Buffer buffer) { BufferDesc *bufHdr; TimestampTz waitStart = 0; - bool waiting = false; - bool logged_recovery_conflict = false; + bool waiting = false; + bool logged_recovery_conflict = false; Assert(BufferIsPinned(buffer)); Assert(PinCountWaitBuf == NULL); @@ -8024,15 +8517,17 @@ LockBufferForCleanup(Buffer buffer) bufHdr = GetBufferDescriptor(buffer - 1); - for (;;) { - uint32 buf_state; + for (;;) + { + uint32 buf_state; /* Try to acquire lock */ LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); buf_state = LockBufHdr(bufHdr); Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0); - if (BUF_STATE_GET_REFCOUNT(buf_state) == 1) { + if (BUF_STATE_GET_REFCOUNT(buf_state) == 1) + { /* Successfully acquired exclusive lock with pincount 1 */ UnlockBufHdr(bufHdr, buf_state); @@ -8042,10 +8537,12 @@ LockBufferForCleanup(Buffer buffer) * deadlock_timeout for it. */ if (logged_recovery_conflict) - LogRecoveryConflict(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, waitStart, - GetCurrentTimestamp(), NULL, false); + LogRecoveryConflict(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, + waitStart, GetCurrentTimestamp(), + NULL, false); - if (waiting) { + if (waiting) + { /* reset ps display to remove the suffix if we added one */ set_ps_display_remove_suffix(); waiting = false; @@ -8053,7 +8550,8 @@ LockBufferForCleanup(Buffer buffer) return; } /* Failed, so mark myself as waiting for pincount 1 */ - if (buf_state & BM_PIN_COUNT_WAITER) { + if (buf_state & BM_PIN_COUNT_WAITER) + { UnlockBufHdr(bufHdr, buf_state); LockBuffer(buffer, BUFFER_LOCK_UNLOCK); elog(ERROR, "multiple backends attempting to wait for pincount 1"); @@ -8065,8 +8563,10 @@ LockBufferForCleanup(Buffer buffer) LockBuffer(buffer, BUFFER_LOCK_UNLOCK); /* Wait to be signaled by UnpinBuffer() */ - if (InHotStandby) { - if (!waiting) { + if (InHotStandby) + { + if (!waiting) + { /* adjust the process title to indicate that it's waiting */ set_ps_display_suffix("waiting"); waiting = true; @@ -8080,12 +8580,15 @@ LockBufferForCleanup(Buffer buffer) * not started waiting yet in this case. So, the wait start * timestamp is set after this logic. */ - if (waitStart != 0 && !logged_recovery_conflict) { + if (waitStart != 0 && !logged_recovery_conflict) + { TimestampTz now = GetCurrentTimestamp(); - if (TimestampDifferenceExceeds(waitStart, now, DeadlockTimeout)) { - LogRecoveryConflict(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, waitStart, now, NULL, - true); + if (TimestampDifferenceExceeds(waitStart, now, + DeadlockTimeout)) + { + LogRecoveryConflict(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, + waitStart, now, NULL, true); logged_recovery_conflict = true; } } @@ -8103,7 +8606,8 @@ LockBufferForCleanup(Buffer buffer) ResolveRecoveryConflictWithBufferPin(); /* Reset the published bufid */ SetStartupBufferPinWaitBufId(-1); - } else + } + else ProcWaitForSignal(PG_WAIT_BUFFER_PIN); /* @@ -8115,8 +8619,8 @@ LockBufferForCleanup(Buffer buffer) * better be safe. */ buf_state = LockBufHdr(bufHdr); - if ((buf_state & BM_PIN_COUNT_WAITER) != 0 - && bufHdr->wait_backend_pgprocno == MyProc->pgprocno) + if ((buf_state & BM_PIN_COUNT_WAITER) != 0 && + bufHdr->wait_backend_pgprocno == MyProc->pgprocno) buf_state &= ~BM_PIN_COUNT_WAITER; UnlockBufHdr(bufHdr, buf_state); @@ -8132,7 +8636,7 @@ LockBufferForCleanup(Buffer buffer) bool HoldingBufferPinThatDelaysRecovery(void) { - int bufid = GetStartupBufferPinWaitBufId(); + int bufid = GetStartupBufferPinWaitBufId(); /* * If we get woken slowly then it's possible that the Startup process was @@ -8159,11 +8663,13 @@ bool ConditionalLockBufferForCleanup(Buffer buffer) { BufferDesc *bufHdr; - uint32 buf_state, refcount; + uint32 buf_state, + refcount; Assert(BufferIsValid(buffer)); - if (BufferIsLocal(buffer)) { + if (BufferIsLocal(buffer)) + { refcount = LocalRefCount[-buffer - 1]; /* There should be exactly one pin */ Assert(refcount > 0); @@ -8188,7 +8694,8 @@ ConditionalLockBufferForCleanup(Buffer buffer) refcount = BUF_STATE_GET_REFCOUNT(buf_state); Assert(refcount > 0); - if (refcount == 1) { + if (refcount == 1) + { /* Successfully acquired exclusive lock with pincount 1 */ UnlockBufHdr(bufHdr, buf_state); return true; @@ -8212,11 +8719,12 @@ bool IsBufferCleanupOK(Buffer buffer) { BufferDesc *bufHdr; - uint32 buf_state; + uint32 buf_state; Assert(BufferIsValid(buffer)); - if (BufferIsLocal(buffer)) { + if (BufferIsLocal(buffer)) + { /* There should be exactly one pin */ if (LocalRefCount[-buffer - 1] != 1) return false; @@ -8231,12 +8739,14 @@ IsBufferCleanupOK(Buffer buffer) bufHdr = GetBufferDescriptor(buffer - 1); /* caller must hold exclusive lock on buffer */ - Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr), LW_EXCLUSIVE)); + Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr), + LW_EXCLUSIVE)); buf_state = LockBufHdr(bufHdr); Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0); - if (BUF_STATE_GET_REFCOUNT(buf_state) == 1) { + if (BUF_STATE_GET_REFCOUNT(buf_state) == 1) + { /* pincount is OK. */ UnlockBufHdr(bufHdr, buf_state); return true; @@ -8265,8 +8775,9 @@ WaitIO(BufferDesc *buf) ConditionVariable *cv = BufferDescriptorGetIOCV(buf); ConditionVariablePrepareToSleep(cv); - for (;;) { - uint32 buf_state; + for (;;) + { + uint32 buf_state; /* * It may not be necessary to acquire the spinlock to check the flag @@ -8319,14 +8830,15 @@ WaitIO(BufferDesc *buf) static bool StartBufferIO(BufferDesc *buf, bool forInput) { - uint32 buf_state; + uint32 buf_state; #ifdef USE_PGRAC_CLUSTER - bool pi_implicit_discard = false; + bool pi_implicit_discard = false; #endif ResourceOwnerEnlargeBufferIOs(CurrentResourceOwner); - for (;;) { + for (;;) + { buf_state = LockBufHdr(buf); if (!(buf_state & BM_IO_IN_PROGRESS)) @@ -8337,7 +8849,8 @@ StartBufferIO(BufferDesc *buf, bool forInput) /* Once we get here, there is definitely no I/O active on this buffer */ - if (forInput ? (buf_state & BM_VALID) : !(buf_state & BM_DIRTY)) { + if (forInput ? (buf_state & BM_VALID) : !(buf_state & BM_DIRTY)) + { /* someone else already did the I/O */ UnlockBufHdr(buf, buf_state); return false; @@ -8345,8 +8858,9 @@ StartBufferIO(BufferDesc *buf, bool forInput) #ifdef USE_PGRAC_CLUSTER /* PGRAC: spec-6.12h D-h3a — see the function header note. */ - if (forInput && buf->buffer_type == (uint8)BUF_TYPE_PI) { - buf->buffer_type = (uint8)BUF_TYPE_CURRENT; + if (forInput && buf->buffer_type == (uint8) BUF_TYPE_PI) + { + buf->buffer_type = (uint8) BUF_TYPE_CURRENT; pi_implicit_discard = true; } #endif @@ -8359,7 +8873,8 @@ StartBufferIO(BufferDesc *buf, bool forInput) cluster_lever_h_note_pi_implicit_discard(); #endif - ResourceOwnerRememberBufferIO(CurrentResourceOwner, BufferDescriptorGetBuffer(buf)); + ResourceOwnerRememberBufferIO(CurrentResourceOwner, + BufferDescriptorGetBuffer(buf)); return true; } @@ -8383,7 +8898,7 @@ StartBufferIO(BufferDesc *buf, bool forInput) static void TerminateBufferIO(BufferDesc *buf, bool clear_dirty, uint32 set_flag_bits) { - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf); @@ -8396,7 +8911,8 @@ TerminateBufferIO(BufferDesc *buf, bool clear_dirty, uint32 set_flag_bits) buf_state |= set_flag_bits; UnlockBufHdr(buf, buf_state); - ResourceOwnerForgetBufferIO(CurrentResourceOwner, BufferDescriptorGetBuffer(buf)); + ResourceOwnerForgetBufferIO(CurrentResourceOwner, + BufferDescriptorGetBuffer(buf)); ConditionVariableBroadcast(BufferDescriptorGetIOCV(buf)); } @@ -8414,28 +8930,34 @@ void AbortBufferIO(Buffer buffer) { BufferDesc *buf_hdr = GetBufferDescriptor(buffer - 1); - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf_hdr); Assert(buf_state & (BM_IO_IN_PROGRESS | BM_TAG_VALID)); - if (!(buf_state & BM_VALID)) { + if (!(buf_state & BM_VALID)) + { Assert(!(buf_state & BM_DIRTY)); UnlockBufHdr(buf_hdr, buf_state); - } else { + } + else + { Assert(buf_state & BM_DIRTY); UnlockBufHdr(buf_hdr, buf_state); /* Issue notice if this is not the first failure... */ - if (buf_state & BM_IO_ERROR) { + if (buf_state & BM_IO_ERROR) + { /* Buffer is pinned, so we can read tag without spinlock */ - char *path; + char *path; path = relpathperm(BufTagGetRelFileLocator(&buf_hdr->tag), BufTagGetForkNum(&buf_hdr->tag)); - ereport(WARNING, (errcode(ERRCODE_IO_ERROR), - errmsg("could not write block %u of %s", buf_hdr->tag.blockNum, path), - errdetail("Multiple failures --- write error might be permanent."))); + ereport(WARNING, + (errcode(ERRCODE_IO_ERROR), + errmsg("could not write block %u of %s", + buf_hdr->tag.blockNum, path), + errdetail("Multiple failures --- write error might be permanent."))); pfree(path); } } @@ -8449,14 +8971,16 @@ AbortBufferIO(Buffer buffer) static void shared_buffer_write_error_callback(void *arg) { - BufferDesc *bufHdr = (BufferDesc *)arg; + BufferDesc *bufHdr = (BufferDesc *) arg; /* Buffer is pinned, so we can read the tag without locking the spinlock */ - if (bufHdr != NULL) { - char *path - = relpathperm(BufTagGetRelFileLocator(&bufHdr->tag), BufTagGetForkNum(&bufHdr->tag)); + if (bufHdr != NULL) + { + char *path = relpathperm(BufTagGetRelFileLocator(&bufHdr->tag), + BufTagGetForkNum(&bufHdr->tag)); - errcontext("writing block %u of relation %s", bufHdr->tag.blockNum, path); + errcontext("writing block %u of relation %s", + bufHdr->tag.blockNum, path); pfree(path); } } @@ -8467,13 +8991,16 @@ shared_buffer_write_error_callback(void *arg) static void local_buffer_write_error_callback(void *arg) { - BufferDesc *bufHdr = (BufferDesc *)arg; + BufferDesc *bufHdr = (BufferDesc *) arg; - if (bufHdr != NULL) { - char *path = relpathbackend(BufTagGetRelFileLocator(&bufHdr->tag), MyBackendId, - BufTagGetForkNum(&bufHdr->tag)); + if (bufHdr != NULL) + { + char *path = relpathbackend(BufTagGetRelFileLocator(&bufHdr->tag), + MyBackendId, + BufTagGetForkNum(&bufHdr->tag)); - errcontext("writing block %u of relation %s", bufHdr->tag.blockNum, path); + errcontext("writing block %u of relation %s", + bufHdr->tag.blockNum, path); pfree(path); } } @@ -8484,8 +9011,8 @@ local_buffer_write_error_callback(void *arg) static int rlocator_comparator(const void *p1, const void *p2) { - RelFileLocator n1 = *(const RelFileLocator *)p1; - RelFileLocator n2 = *(const RelFileLocator *)p2; + RelFileLocator n1 = *(const RelFileLocator *) p1; + RelFileLocator n2 = *(const RelFileLocator *) p2; if (n1.relNumber < n2.relNumber) return -1; @@ -8512,13 +9039,14 @@ uint32 LockBufHdr(BufferDesc *desc) { SpinDelayStatus delayStatus; - uint32 old_buf_state; + uint32 old_buf_state; Assert(!BufferIsLocal(BufferDescriptorGetBuffer(desc))); init_local_spin_delay(&delayStatus); - while (true) { + while (true) + { /* set BM_LOCKED flag */ old_buf_state = pg_atomic_fetch_or_u32(&desc->state, BM_LOCKED); /* if it wasn't set before we're OK */ @@ -8541,13 +9069,14 @@ static uint32 WaitBufHdrUnlocked(BufferDesc *buf) { SpinDelayStatus delayStatus; - uint32 buf_state; + uint32 buf_state; init_local_spin_delay(&delayStatus); buf_state = pg_atomic_read_u32(&buf->state); - while (buf_state & BM_LOCKED) { + while (buf_state & BM_LOCKED) + { perform_spin_delay(&delayStatus); buf_state = pg_atomic_read_u32(&buf->state); } @@ -8563,7 +9092,7 @@ WaitBufHdrUnlocked(BufferDesc *buf) static inline int buffertag_comparator(const BufferTag *ba, const BufferTag *bb) { - int ret; + int ret; RelFileLocator rlocatora; RelFileLocator rlocatorb; @@ -8628,8 +9157,8 @@ ckpt_buforder_comparator(const CkptSortItem *a, const CkptSortItem *b) static int ts_ckpt_progress_comparator(Datum a, Datum b, void *arg) { - CkptTsStatus *sa = (CkptTsStatus *)a; - CkptTsStatus *sb = (CkptTsStatus *)b; + CkptTsStatus *sa = (CkptTsStatus *) a; + CkptTsStatus *sb = (CkptTsStatus *) b; /* we want a min-heap, so return 1 for the a < b */ if (sa->progress < sb->progress) @@ -8661,7 +9190,8 @@ WritebackContextInit(WritebackContext *context, int *max_pending) * Add buffer to list of pending writeback requests. */ void -ScheduleBufferTagForWriteback(WritebackContext *wb_context, IOContext io_context, BufferTag *tag) +ScheduleBufferTagForWriteback(WritebackContext *wb_context, IOContext io_context, + BufferTag *tag) { PendingWriteback *pending; @@ -8672,7 +9202,8 @@ ScheduleBufferTagForWriteback(WritebackContext *wb_context, IOContext io_context * Add buffer to the pending writeback array, unless writeback control is * disabled. */ - if (*wb_context->max_pending > 0) { + if (*wb_context->max_pending > 0) + { Assert(*wb_context->max_pending <= WRITEBACK_MAX_PENDING_FLUSHES); pending = &wb_context->pending_writebacks[wb_context->nr_pending++]; @@ -8706,8 +9237,8 @@ ScheduleBufferTagForWriteback(WritebackContext *wb_context, IOContext io_context void IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context) { - instr_time io_start; - int i; + instr_time io_start; + int i; if (wb_context->nr_pending == 0) return; @@ -8716,7 +9247,8 @@ IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context) * Executing the writes in-order can make them a lot faster, and allows to * merge writeback requests to consecutive blocks into larger writebacks. */ - sort_pending_writebacks(wb_context->pending_writebacks, wb_context->nr_pending); + sort_pending_writebacks(wb_context->pending_writebacks, + wb_context->nr_pending); io_start = pgstat_prepare_io_time(); @@ -8725,14 +9257,15 @@ IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context) * through the, now sorted, array of pending flushes, and look forward to * find all neighbouring (or identical) writes. */ - for (i = 0; i < wb_context->nr_pending; i++) { + for (i = 0; i < wb_context->nr_pending; i++) + { PendingWriteback *cur; PendingWriteback *next; SMgrRelation reln; - int ahead; - BufferTag tag; + int ahead; + BufferTag tag; RelFileLocator currlocator; - Size nblocks = 1; + Size nblocks = 1; cur = &wb_context->pending_writebacks[i]; tag = cur->tag; @@ -8742,12 +9275,15 @@ IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context) * Peek ahead, into following writeback requests, to see if they can * be combined with the current one. */ - for (ahead = 0; i + ahead + 1 < wb_context->nr_pending; ahead++) { + for (ahead = 0; i + ahead + 1 < wb_context->nr_pending; ahead++) + { + next = &wb_context->pending_writebacks[i + ahead + 1]; /* different file, stop */ - if (!RelFileLocatorEquals(currlocator, BufTagGetRelFileLocator(&next->tag)) - || BufTagGetForkNum(&cur->tag) != BufTagGetForkNum(&next->tag)) + if (!RelFileLocatorEquals(currlocator, + BufTagGetRelFileLocator(&next->tag)) || + BufTagGetForkNum(&cur->tag) != BufTagGetForkNum(&next->tag)) break; /* ok, block queued twice, skip */ @@ -8773,8 +9309,8 @@ IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context) * Assume that writeback requests are only issued for buffers containing * blocks of permanent relations. */ - pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, IOOP_WRITEBACK, io_start, - wb_context->nr_pending); + pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, + IOOP_WRITEBACK, io_start, wb_context->nr_pending); wb_context->nr_pending = 0; } @@ -8791,7 +9327,9 @@ TestForOldSnapshot_impl(Snapshot snapshot, Relation relation) { if (RelationAllowsEarlyPruning(relation) && (snapshot)->whenTaken < GetOldSnapshotThresholdTimestamp()) - ereport(ERROR, (errcode(ERRCODE_SNAPSHOT_TOO_OLD), errmsg("snapshot too old"))); + ereport(ERROR, + (errcode(ERRCODE_SNAPSHOT_TOO_OLD), + errmsg("snapshot too old"))); } @@ -8865,7 +9403,7 @@ TestForOldSnapshot_impl(Snapshot snapshot, Relation relation) static XLogRecPtr cluster_gcs_clamp_ship_flush_lsn(XLogRecPtr page_lsn) { - XLogRecPtr local_insert; + XLogRecPtr local_insert; if (XLogRecPtrIsInvalid(page_lsn) || RecoveryInProgress()) return page_lsn; @@ -8894,35 +9432,41 @@ cluster_bufmgr_pin_for_gcs_locked(BufferDesc *buf, uint32 buf_state) static void cluster_bufmgr_unpin_for_gcs(BufferDesc *buf) { - uint32 buf_state; - uint32 old_buf_state; + uint32 buf_state; + uint32 old_buf_state; Assert(!LWLockHeldByMe(BufferDescriptorGetContentLock(buf))); VALGRIND_MAKE_MEM_NOACCESS(BufHdrGetBlock(buf), BLCKSZ); old_buf_state = pg_atomic_read_u32(&buf->state); - for (;;) { + for (;;) + { if (old_buf_state & BM_LOCKED) old_buf_state = WaitBufHdrUnlocked(buf); buf_state = old_buf_state; buf_state -= BUF_REFCOUNT_ONE; - if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state, buf_state)) + if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state, + buf_state)) break; } - if (buf_state & BM_PIN_COUNT_WAITER) { + if (buf_state & BM_PIN_COUNT_WAITER) + { buf_state = LockBufHdr(buf); - if ((buf_state & BM_PIN_COUNT_WAITER) && BUF_STATE_GET_REFCOUNT(buf_state) == 1) { - int wait_backend_pgprocno = buf->wait_backend_pgprocno; + if ((buf_state & BM_PIN_COUNT_WAITER) && + BUF_STATE_GET_REFCOUNT(buf_state) == 1) + { + int wait_backend_pgprocno = buf->wait_backend_pgprocno; buf_state &= ~BM_PIN_COUNT_WAITER; UnlockBufHdr(buf, buf_state); ProcSendSignal(wait_backend_pgprocno); - } else + } + else UnlockBufHdr(buf, buf_state); } } @@ -8941,20 +9485,21 @@ cluster_bufmgr_unpin_for_gcs(BufferDesc *buf) bool cluster_bufmgr_probe_block_for_gcs(BufferTag tag) { - uint32 hashcode = BufTableHashCode(&tag); - LWLock *partition_lock = BufMappingPartitionLock(hashcode); - int buf_id; - bool valid = false; + uint32 hashcode = BufTableHashCode(&tag); + LWLock *partition_lock = BufMappingPartitionLock(hashcode); + int buf_id; + bool valid = false; LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id >= 0) { + if (buf_id >= 0) + { BufferDesc *buf = GetBufferDescriptor(buf_id); - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf); valid = BufferTagsEqual(&buf->tag, &tag) - && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); UnlockBufHdr(buf, buf_state); } LWLockRelease(partition_lock); @@ -8983,12 +9528,12 @@ cluster_bufmgr_read_storage_scn_for_gcs(BufferTag tag, SCN *out_page_scn) reln = smgropen(BufTagGetRelFileLocator(&tag), InvalidBackendId); smgrread(reln, BufTagGetForkNum(&tag), tag.blockNum, scratch.data); - if (!PageIsVerifiedExtended((Page)scratch.data, tag.blockNum, + if (!PageIsVerifiedExtended((Page) scratch.data, tag.blockNum, PIV_LOG_WARNING | PIV_REPORT_STAT)) return false; if (out_page_scn != NULL) - *out_page_scn = ((PageHeader)scratch.data)->pd_block_scn; + *out_page_scn = ((PageHeader) scratch.data)->pd_block_scn; return true; } @@ -9011,16 +9556,16 @@ cluster_bufmgr_read_storage_scn_for_gcs(BufferTag tag, SCN *out_page_scn) bool cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char *dst) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - LWLock *content_lock; - XLogRecPtr first_lsn; - XLogRecPtr second_lsn; - int retries; - bool stable; - Page page; + LWLock *content_lock; + XLogRecPtr first_lsn; + XLogRecPtr second_lsn; + int retries; + bool stable; + Page page; Assert(dst != NULL); Assert(out_page_lsn != NULL); @@ -9030,7 +9575,8 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) { + if (buf_id < 0) + { LWLockRelease(partition_lock); return false; } @@ -9038,17 +9584,19 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char /* Partition lock keeps the buffer from being recycled before we raw-pin. */ { - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf); /* Re-verify tag under header lock to defend against tag-rewrite * races between the partition-lock-protected lookup and the pin. */ - if (!BufferTagsEqual(&buf->tag, &tag)) { + if (!BufferTagsEqual(&buf->tag, &tag)) + { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; } - if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { + if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; @@ -9058,7 +9606,7 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char LWLockRelease(partition_lock); content_lock = BufferDescriptorGetContentLock(buf); - page = (Page)BufHdrGetBlock(buf); + page = (Page) BufHdrGetBlock(buf); /* * HC89: revalidation loop with single retry budget. retries = 0 — first @@ -9066,7 +9614,8 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char * fail-closed */ stable = false; - for (retries = 0; retries < 2; retries++) { + for (retries = 0; retries < 2; retries++) + { /* Read page_lsn under content_lock SHARED. */ LWLockAcquire(content_lock, LW_SHARED); first_lsn = PageGetLSN(page); @@ -9080,7 +9629,7 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char */ #ifdef USE_CLUSTER_UNIT if (cluster_gcs_block_test_xlog_flush_hook != NULL) - cluster_gcs_block_test_xlog_flush_hook((uint64)first_lsn); + cluster_gcs_block_test_xlog_flush_hook((uint64) first_lsn); #endif if (!XLogRecPtrIsInvalid(first_lsn)) XLogFlush(cluster_gcs_clamp_ship_flush_lsn(first_lsn)); @@ -9095,10 +9644,11 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char { uint32 buf_state = LockBufHdr(buf); bool current = BufferTagsEqual(&buf->tag, &tag) - && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); UnlockBufHdr(buf, buf_state); - if (!current) { + if (!current) + { LWLockRelease(content_lock); break; } @@ -9112,15 +9662,17 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char * consecutive LSN drift events. retries 0 + drift available means * second_lsn != first_lsn synthetically. */ - if (cluster_gcs_block_test_lsn_drift_hook != NULL) { - int drift_remaining = cluster_gcs_block_test_lsn_drift_hook(); + if (cluster_gcs_block_test_lsn_drift_hook != NULL) + { + int drift_remaining = cluster_gcs_block_test_lsn_drift_hook(); if (drift_remaining > retries) - second_lsn = first_lsn + 1; /* synthetic mismatch */ + second_lsn = first_lsn + 1; /* synthetic mismatch */ } #endif - if (BufferTagsEqual(&buf->tag, &tag) && first_lsn == second_lsn) { + if (BufferTagsEqual(&buf->tag, &tag) && first_lsn == second_lsn) + { memcpy(dst, page, BLCKSZ); *out_page_lsn = second_lsn; LWLockRelease(content_lock); @@ -9150,15 +9702,15 @@ bool cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page_lsn, void **out_page_addr, BufferDesc **out_buf) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - LWLock *content_lock; - XLogRecPtr first_lsn; - XLogRecPtr second_lsn; - int retries; - Page page; + LWLock *content_lock; + XLogRecPtr first_lsn; + XLogRecPtr second_lsn; + int retries; + Page page; Assert(out_page_lsn != NULL); Assert(out_page_addr != NULL); @@ -9173,18 +9725,20 @@ cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) { + if (buf_id < 0) + { LWLockRelease(partition_lock); return false; } buf = GetBufferDescriptor(buf_id); { - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf); if (!BufferTagsEqual(&buf->tag, &tag) - || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { + || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; @@ -9194,16 +9748,17 @@ cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page LWLockRelease(partition_lock); content_lock = BufferDescriptorGetContentLock(buf); - page = (Page)BufHdrGetBlock(buf); + page = (Page) BufHdrGetBlock(buf); - for (retries = 0; retries < 2; retries++) { + for (retries = 0; retries < 2; retries++) + { LWLockAcquire(content_lock, LW_SHARED); first_lsn = PageGetLSN(page); LWLockRelease(content_lock); #ifdef USE_CLUSTER_UNIT if (cluster_gcs_block_test_xlog_flush_hook != NULL) - cluster_gcs_block_test_xlog_flush_hook((uint64)first_lsn); + cluster_gcs_block_test_xlog_flush_hook((uint64) first_lsn); #endif if (!XLogRecPtrIsInvalid(first_lsn)) XLogFlush(cluster_gcs_clamp_ship_flush_lsn(first_lsn)); @@ -9212,10 +9767,11 @@ cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page { uint32 buf_state = LockBufHdr(buf); bool current = BufferTagsEqual(&buf->tag, &tag) - && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); UnlockBufHdr(buf, buf_state); - if (!current) { + if (!current) + { LWLockRelease(content_lock); break; } @@ -9223,15 +9779,17 @@ cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page second_lsn = PageGetLSN(page); #ifdef USE_CLUSTER_UNIT - if (cluster_gcs_block_test_lsn_drift_hook != NULL) { - int drift_remaining = cluster_gcs_block_test_lsn_drift_hook(); + if (cluster_gcs_block_test_lsn_drift_hook != NULL) + { + int drift_remaining = cluster_gcs_block_test_lsn_drift_hook(); if (drift_remaining > retries) second_lsn = first_lsn + 1; } #endif - if (BufferTagsEqual(&buf->tag, &tag) && first_lsn == second_lsn) { + if (BufferTagsEqual(&buf->tag, &tag) && first_lsn == second_lsn) + { *out_page_lsn = second_lsn; *out_page_addr = page; *out_buf = buf; @@ -9263,7 +9821,7 @@ bool cluster_bufmgr_prepare_direct_land_target_for_gcs(BufferDesc *buf, BufferTag tag, void **out_page_addr) { - uint32 buf_state; + uint32 buf_state; Assert(out_page_addr != NULL); @@ -9272,8 +9830,11 @@ cluster_bufmgr_prepare_direct_land_target_for_gcs(BufferDesc *buf, BufferTag tag return false; buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_TAG_VALID) == 0 - || (buf_state & BM_VALID) != 0 || (buf_state & BM_IO_IN_PROGRESS) != 0) { + if (!BufferTagsEqual(&buf->tag, &tag) + || (buf_state & BM_TAG_VALID) == 0 + || (buf_state & BM_VALID) != 0 + || (buf_state & BM_IO_IN_PROGRESS) != 0) + { UnlockBufHdr(buf, buf_state); return false; } @@ -9294,20 +9855,23 @@ cluster_bufmgr_prepare_direct_land_target_for_gcs(BufferDesc *buf, BufferTag tag * BM_VALID through TerminateBufferIO. On failure, leave the buffer invalid. */ void -cluster_bufmgr_finish_direct_land_target_for_gcs(BufferDesc *buf, bool valid, XLogRecPtr page_lsn) +cluster_bufmgr_finish_direct_land_target_for_gcs(BufferDesc *buf, bool valid, + XLogRecPtr page_lsn) { if (buf == NULL) return; - if (valid) { - LWLock *content_lock = BufferDescriptorGetContentLock(buf); - Page page = (Page)BufHdrGetBlock(buf); + if (valid) + { + LWLock *content_lock = BufferDescriptorGetContentLock(buf); + Page page = (Page) BufHdrGetBlock(buf); LWLockAcquire(content_lock, LW_EXCLUSIVE); PageSetLSN(page, page_lsn); LWLockRelease(content_lock); TerminateBufferIO(buf, false, BM_VALID); - } else + } + else TerminateBufferIO(buf, false, 0); } @@ -9342,27 +9906,29 @@ cluster_bufmgr_finish_direct_land_target_for_gcs(BufferDesc *buf, bool valid, XL bool cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - LWLock *content_lock; - uint32 buf_state; - bool dirty; + LWLock *content_lock; + uint32 buf_state; + bool dirty; hashcode = BufTableHashCode(&tag); partition_lock = BufMappingPartitionLock(hashcode); LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) { + if (buf_id < 0) + { LWLockRelease(partition_lock); return false; } buf = GetBufferDescriptor(buf_id); buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) { + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) + { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; @@ -9372,7 +9938,8 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) content_lock = BufferDescriptorGetContentLock(buf); LWLockAcquire(content_lock, LW_EXCLUSIVE); - if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) { + if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) + { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); return false; @@ -9384,8 +9951,10 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) * this in-memory copy for its commit stamp (the P0-2 dependency) — the * caller falls back to the one-shot read-image ship. */ - if (!BufferTagsEqual(&buf->tag, &tag) || (PcmState)buf->pcm_state != PCM_STATE_X - || cluster_itl_page_has_active_slot((Page)BufHdrGetBlock(buf))) { + if (!BufferTagsEqual(&buf->tag, &tag) + || (PcmState) buf->pcm_state != PCM_STATE_X + || cluster_itl_page_has_active_slot((Page) BufHdrGetBlock(buf))) + { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); return false; @@ -9398,7 +9967,9 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) if (dirty) FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); - if (!cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_X_TO_S_DOWNGRADE, cluster_node_id)) { + if (!cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_X_TO_S_DOWNGRADE, + cluster_node_id)) + { /* Master refused (state moved under us) — leave local X untouched. */ LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); @@ -9409,7 +9980,7 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) * ownership transition -- set pcm_state and bump the generation atomically * (header spinlock) so a cached-X writer racing the content-lock window * detects the revoke even across an X->S->X ABA. */ - cluster_pcm_own_transition(buf, (uint8)PCM_STATE_S, 0, 0); + cluster_pcm_own_transition(buf, (uint8) PCM_STATE_S, 0, 0); LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); @@ -9444,11 +10015,11 @@ Buffer cluster_bufmgr_lock_resident_for_stamp(RelFileLocator rlocator, ForkNumber forknum, BlockNumber blocknum) { - BufferTag tag; - uint32 hashcode; - uint32 buf_state; - LWLock *partition_lock; - int buf_id; + BufferTag tag; + uint32 hashcode; + uint32 buf_state; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; InitBufferTag(&tag, &rlocator, forknum, blocknum); @@ -9460,7 +10031,8 @@ cluster_bufmgr_lock_resident_for_stamp(RelFileLocator rlocator, ForkNumber forkn LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) { + if (buf_id < 0) + { /* Not resident -> transferred away (or never cached). Skip stamp. */ LWLockRelease(partition_lock); return InvalidBuffer; @@ -9495,9 +10067,11 @@ cluster_bufmgr_lock_resident_for_stamp(RelFileLocator rlocator, ForkNumber forkn * legal commit-stamp target. */ buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0 + if (!BufferTagsEqual(&buf->tag, &tag) + || (buf_state & BM_VALID) == 0 || cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state) - || (cluster_pcm_own_flags_get(buf->buf_id) & PCM_OWN_FLAG_REVOKING) != 0) { + || (cluster_pcm_own_flags_get(buf->buf_id) & PCM_OWN_FLAG_REVOKING) != 0) + { UnlockBufHdr(buf, buf_state); LWLockRelease(BufferDescriptorGetContentLock(buf)); ReleaseBuffer(BufferDescriptorGetBuffer(buf)); @@ -9553,13 +10127,13 @@ cluster_bufmgr_unlock_resident_stamp(Buffer buffer) bool cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - LWLock *content_lock; - uint32 buf_state; - bool dirty; + LWLock *content_lock; + uint32 buf_state; + bool dirty; if (master_node < 0 || master_node == cluster_node_id) return false; @@ -9569,14 +10143,16 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) { + if (buf_id < 0) + { LWLockRelease(partition_lock); return false; } buf = GetBufferDescriptor(buf_id); buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) { + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) + { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; @@ -9586,15 +10162,18 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) content_lock = BufferDescriptorGetContentLock(buf); LWLockAcquire(content_lock, LW_EXCLUSIVE); - if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) { + if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) + { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); return false; } /* Same re-verify as the local variant: tag, PCM X, quiescent. */ - if (!BufferTagsEqual(&buf->tag, &tag) || (PcmState)buf->pcm_state != PCM_STATE_X - || cluster_itl_page_has_active_slot((Page)BufHdrGetBlock(buf))) { + if (!BufferTagsEqual(&buf->tag, &tag) + || (PcmState) buf->pcm_state != PCM_STATE_X + || cluster_itl_page_has_active_slot((Page) BufHdrGetBlock(buf))) + { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); return false; @@ -9615,9 +10194,12 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) * recording X@us. Drives the renotify self-heal TAP leg (t/348 L8). */ CLUSTER_INJECTION_POINT("cluster-gcs-block-yield-notify-drop"); - if (cluster_injection_should_skip("cluster-gcs-block-yield-notify-drop")) { + if (cluster_injection_should_skip("cluster-gcs-block-yield-notify-drop")) + { /* simulated post-handoff loss: fall through to the S flip */ - } else if (!cluster_gcs_send_transition_nowait(tag, PCM_TRANS_X_TO_S_DOWNGRADE, master_node)) { + } + else if (!cluster_gcs_send_transition_nowait(tag, PCM_TRANS_X_TO_S_DOWNGRADE, master_node)) + { /* Notify not handed to transport — keep local X, master unchanged. */ LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); @@ -9628,7 +10210,7 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) * ownership transition -- set pcm_state and bump the generation atomically * (header spinlock) so a cached-X writer racing the content-lock window * detects the revoke even across an X->S->X ABA. */ - cluster_pcm_own_transition(buf, (uint8)PCM_STATE_S, 0, 0); + cluster_pcm_own_transition(buf, (uint8) PCM_STATE_S, 0, 0); LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); @@ -9653,12 +10235,12 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) bool cluster_bufmgr_renotify_s_for_gcs(BufferTag tag, int32 master_node) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - uint32 buf_state; - bool is_s; + uint32 buf_state; + bool is_s; if (master_node < 0 || master_node == cluster_node_id) return false; @@ -9668,7 +10250,8 @@ cluster_bufmgr_renotify_s_for_gcs(BufferTag tag, int32 master_node) LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) { + if (buf_id < 0) + { LWLockRelease(partition_lock); return false; } @@ -9676,7 +10259,7 @@ cluster_bufmgr_renotify_s_for_gcs(BufferTag tag, int32 master_node) buf_state = LockBufHdr(buf); is_s = BufferTagsEqual(&buf->tag, &tag) && (buf_state & BM_VALID) != 0 - && (PcmState)buf->pcm_state == PCM_STATE_S; + && (PcmState) buf->pcm_state == PCM_STATE_S; UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); @@ -9698,17 +10281,17 @@ bool cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_page_lsn, char *dst, ClusterSfDepVec *out_dep_vec) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - LWLock *content_lock; - XLogRecPtr first_lsn; - XLogRecPtr second_lsn; - int retries; - bool copied_stable; - bool stable; - Page page; + LWLock *content_lock; + XLogRecPtr first_lsn; + XLogRecPtr second_lsn; + int retries; + bool copied_stable; + bool stable; + Page page; if (dst == NULL || out_page_lsn == NULL || out_dep_vec == NULL) return false; @@ -9721,18 +10304,20 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) { + if (buf_id < 0) + { LWLockRelease(partition_lock); return false; } buf = GetBufferDescriptor(buf_id); { - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf); if (!BufferTagsEqual(&buf->tag, &tag) - || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { + || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; @@ -9742,18 +10327,20 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa LWLockRelease(partition_lock); content_lock = BufferDescriptorGetContentLock(buf); - page = (Page)BufHdrGetBlock(buf); + page = (Page) BufHdrGetBlock(buf); copied_stable = false; stable = false; - for (retries = 0; retries < 2; retries++) { + for (retries = 0; retries < 2; retries++) + { LWLockAcquire(content_lock, LW_SHARED); { uint32 buf_state = LockBufHdr(buf); bool current = BufferTagsEqual(&buf->tag, &tag) - && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); UnlockBufHdr(buf, buf_state); - if (!current) { + if (!current) + { LWLockRelease(content_lock); break; } @@ -9762,7 +10349,8 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa memcpy(dst, page, BLCKSZ); second_lsn = PageGetLSN(page); if (BufferTagsEqual(&buf->tag, &tag) && !XLogRecPtrIsInvalid(first_lsn) - && first_lsn == second_lsn) { + && first_lsn == second_lsn) + { *out_page_lsn = second_lsn; LWLockRelease(content_lock); copied_stable = true; @@ -9771,11 +10359,12 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa LWLockRelease(content_lock); } - if (copied_stable) { + if (copied_stable) + { ClusterSfDepVec existing_vec; - bool has_existing_deps; - bool add_local_dep; - uint32 buf_state; + bool has_existing_deps; + bool add_local_dep; + uint32 buf_state; /* * Keep lock ordering compatible with receiver install: @@ -9784,17 +10373,18 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa * the dep store after releasing content_lock, while the raw pin keeps * this descriptor/tag from being recycled. */ - has_existing_deps - = cluster_sf_dep_vec_for_ship(BufferDescriptorGetBuffer(buf), &existing_vec); + has_existing_deps = + cluster_sf_dep_vec_for_ship(BufferDescriptorGetBuffer(buf), &existing_vec); cluster_sf_dep_vec_reset(out_dep_vec); if (has_existing_deps) - (void)cluster_sf_dep_vec_union(out_dep_vec, &existing_vec); + (void) cluster_sf_dep_vec_union(out_dep_vec, &existing_vec); buf_state = LockBufHdr(buf); - add_local_dep = (buf_state & (BM_DIRTY | BM_JUST_DIRTIED)) != 0 || !has_existing_deps; + add_local_dep = (buf_state & (BM_DIRTY | BM_JUST_DIRTIED)) != 0 + || !has_existing_deps; UnlockBufHdr(buf, buf_state); if (add_local_dep) - (void)cluster_sf_dep_vec_set(out_dep_vec, cluster_node_id, *out_page_lsn); + (void) cluster_sf_dep_vec_set(out_dep_vec, cluster_node_id, *out_page_lsn); stable = !cluster_sf_dep_vec_is_empty(out_dep_vec); } @@ -9823,11 +10413,11 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa * authoritative lost-write check). No PCM state is mutated. */ int -cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, ClusterGcsRedeclareCallback cb, - void *arg) +cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, + ClusterGcsRedeclareCallback cb, void *arg) { - int i; - int end; + int i; + int end; Assert(cb != NULL); @@ -9837,20 +10427,23 @@ cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, ClusterGcsRedec if (end > NBuffers) end = NBuffers; - for (i = start_buf; i < end; i++) { + for (i = start_buf; i < end; i++) + { BufferDesc *buf = GetBufferDescriptor(i); - uint32 buf_state; - uint8 mode; - BufferTag tag; - LWLock *content_lock; - XLogRecPtr page_lsn; - SCN page_scn; /* spec-2.41 D3 — re-declare SCN carrier */ + uint32 buf_state; + uint8 mode; + BufferTag tag; + LWLock *content_lock; + XLogRecPtr page_lsn; + SCN page_scn; /* spec-2.41 D3 — re-declare SCN carrier */ buf_state = LockBufHdr(buf); mode = buf->pcm_state; - if ((buf_state & BM_VALID) == 0 || (buf_state & BM_IO_IN_PROGRESS) != 0 - || (mode != (uint8)PCM_STATE_S && mode != (uint8)PCM_STATE_X) - || !cluster_bufmgr_should_pcm_track(buf)) { + if ((buf_state & BM_VALID) == 0 + || (buf_state & BM_IO_IN_PROGRESS) != 0 + || (mode != (uint8) PCM_STATE_S && mode != (uint8) PCM_STATE_X) + || !cluster_bufmgr_should_pcm_track(buf)) + { UnlockBufHdr(buf, buf_state); continue; } @@ -9860,11 +10453,11 @@ cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, ClusterGcsRedec content_lock = BufferDescriptorGetContentLock(buf); LWLockAcquire(content_lock, LW_SHARED); - page_lsn = PageGetLSN((Page)BufHdrGetBlock(buf)); + page_lsn = PageGetLSN((Page) BufHdrGetBlock(buf)); /* PGRAC: spec-2.41 D3 — also read pd_block_scn so the re-declare carries * the cross-node version for the detector's SCN watermark (alongside * page_lsn for the redo-coverage required_lsn). */ - page_scn = ((PageHeader)BufHdrGetBlock(buf))->pd_block_scn; + page_scn = ((PageHeader) BufHdrGetBlock(buf))->pd_block_scn; LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); @@ -9904,18 +10497,18 @@ ClusterBufmgrGcsDropResult cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode, XLogRecPtr *out_page_lsn, SCN *out_page_scn) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - uint32 buf_state; - XLogRecPtr page_lsn = InvalidXLogRecPtr; - SCN page_scn = InvalidScn; /* PGRAC: spec-2.41 D3 — page version for the ACK SCN carrier */ - bool was_dirty = false; - uint8 saved_pcm_state; - uint64 staged_gen; /* PGRAC W2: ownership gen captured at stage-N */ + uint32 buf_state; + XLogRecPtr page_lsn = InvalidXLogRecPtr; + SCN page_scn = InvalidScn; /* PGRAC: spec-2.41 D3 — page version for the ACK SCN carrier */ + bool was_dirty = false; + uint8 saved_pcm_state; + uint64 staged_gen; /* PGRAC W2: ownership gen captured at stage-N */ - (void)expected_mode; + (void) expected_mode; if (out_page_lsn != NULL) *out_page_lsn = InvalidXLogRecPtr; @@ -9927,7 +10520,8 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) { + if (buf_id < 0) + { LWLockRelease(partition_lock); return CLUSTER_BUFMGR_GCS_DROP_NOT_RESIDENT; } @@ -9937,7 +10531,8 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode /* Re-verify tag under the header lock to defend against a tag-rewrite race * between the partition-lock lookup and the raw pin (copy_block_for_gcs * convention). */ - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) { + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) + { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return CLUSTER_BUFMGR_GCS_DROP_NOT_RESIDENT; @@ -9950,7 +10545,8 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode * ClusterBufmgrGcsDropResult). Bail before the WAL flush — nothing is * dropped, so the HC123 flush-before-drop invariant is not owed yet. */ - if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) { + if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) + { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return CLUSTER_BUFMGR_GCS_DROP_PINNED; @@ -9971,15 +10567,15 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode * cluster_bufmgr_copy_block_for_gcs and cluster_bufmgr_redeclare_scan_chunk. * The pin keeps the buffer identity stable, so the pre-pin tag match holds. */ - cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); /* raw-pin + unlock header */ + cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); /* raw-pin + unlock header */ LWLockRelease(partition_lock); { - LWLock *content_lock = BufferDescriptorGetContentLock(buf); - Page page = (Page)BufHdrGetBlock(buf); + LWLock *content_lock = BufferDescriptorGetContentLock(buf); + Page page = (Page) BufHdrGetBlock(buf); LWLockAcquire(content_lock, LW_SHARED); page_lsn = PageGetLSN(page); - page_scn = ((PageHeader)page)->pd_block_scn; + page_scn = ((PageHeader) page)->pd_block_scn; LWLockRelease(content_lock); } cluster_bufmgr_unpin_for_gcs(buf); @@ -10007,7 +10603,8 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode * cluster_bufmgr_drop_block_for_gcs_no_wire below). */ buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) { + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) + { UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_NOT_RESIDENT; } @@ -10017,14 +10614,15 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode * unpinned for the XLogFlush re-fails the drop (bounded contract; the * pcm_state is untouched so nothing observes a half-dropped state). */ - if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) { + if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) + { UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_PINNED; } saved_pcm_state = buf->pcm_state; - staged_gen = cluster_pcm_own_gen_get(buf->buf_id); /* PGRAC W2 */ - buf->pcm_state = (uint8)PCM_STATE_N; + staged_gen = cluster_pcm_own_gen_get(buf->buf_id); /* PGRAC W2 */ + buf->pcm_state = (uint8) PCM_STATE_N; /* PGRAC: spec-6.12h D-h1 — keep a Past Image instead of dropping. */ if (cluster_bufmgr_convert_to_pi_locked(buf, buf_state)) @@ -10037,8 +10635,8 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode * (it was cleared for the eviction hook) so the still-resident copy keeps * its true PCM state. */ - cluster_bufmgr_in_gcs_drop = true; /* gates the drop-prepin inject */ - if (!InvalidateBufferTry(buf)) /* releases the header spinlock */ + cluster_bufmgr_in_gcs_drop = true; /* gates the drop-prepin inject */ + if (!InvalidateBufferTry(buf)) /* releases the header spinlock */ { cluster_bufmgr_in_gcs_drop = false; @@ -10063,7 +10661,7 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode */ CLUSTER_INJECTION_POINT("cluster-pcm-restore-aba-force-round"); if (cluster_injection_should_skip("cluster-pcm-restore-aba-force-round")) - cluster_pcm_own_transition(buf, (uint8)PCM_STATE_N, 0, 0); + cluster_pcm_own_transition(buf, (uint8) PCM_STATE_N, 0, 0); buf_state = LockBufHdr(buf); @@ -10079,11 +10677,13 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode * moved, the block was legitimately re-owned and dropped in between, * so leave it at N. */ - if (BufferTagsEqual(&buf->tag, &tag) && buf->pcm_state == (uint8)PCM_STATE_N - && cluster_pcm_own_gen_get(buf->buf_id) == staged_gen) + if (BufferTagsEqual(&buf->tag, &tag) && + buf->pcm_state == (uint8) PCM_STATE_N && + cluster_pcm_own_gen_get(buf->buf_id) == staged_gen) buf->pcm_state = saved_pcm_state; - else if (BufferTagsEqual(&buf->tag, &tag) && buf->pcm_state == (uint8)PCM_STATE_N - && cluster_pcm_own_gen_get(buf->buf_id) != staged_gen) + else if (BufferTagsEqual(&buf->tag, &tag) && + buf->pcm_state == (uint8) PCM_STATE_N && + cluster_pcm_own_gen_get(buf->buf_id) != staged_gen) cluster_pcm_note_restore_aba_detected(); UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_PINNED; @@ -10119,11 +10719,11 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode SCN cluster_bufmgr_read_block_scn_for_gcs(BufferDesc *buf) { - LWLock *content_lock = BufferDescriptorGetContentLock(buf); - SCN scn; + LWLock *content_lock = BufferDescriptorGetContentLock(buf); + SCN scn; LWLockAcquire(content_lock, LW_SHARED); - scn = ((PageHeader)BufHdrGetBlock(buf))->pd_block_scn; + scn = ((PageHeader) BufHdrGetBlock(buf))->pd_block_scn; LWLockRelease(content_lock); return scn; } @@ -10144,7 +10744,7 @@ bool cluster_bufmgr_block_is_extension_for_gcs(BufferTag tag) { SMgrRelation reln = smgropen(BufTagGetRelFileLocator(&tag), InvalidBackendId); - ForkNumber fork = BufTagGetForkNum(&tag); + ForkNumber fork = BufTagGetForkNum(&tag); /* Drop any cached size so a concurrent extension is not missed. */ smgrrelease(reln); @@ -10155,12 +10755,12 @@ bool cluster_bufmgr_refresh_block_from_storage_for_gcs(BufferDesc *buf, SCN *out_page_scn) { PGAlignedBlock scratch; - BufferTag tag = buf->tag; + BufferTag tag = buf->tag; SMgrRelation reln; - LWLock *content_lock; - uint32 buf_state; - bool dirty; - bool write_permitted; + LWLock *content_lock; + uint32 buf_state; + bool dirty; + bool write_permitted; if (out_page_scn != NULL) *out_page_scn = InvalidScn; @@ -10175,13 +10775,12 @@ cluster_bufmgr_refresh_block_from_storage_for_gcs(BufferDesc *buf, SCN *out_page */ reln = smgropen(BufTagGetRelFileLocator(&tag), InvalidBackendId); smgrread(reln, BufTagGetForkNum(&tag), tag.blockNum, scratch.data); - if (!PageIsVerifiedExtended((Page)scratch.data, tag.blockNum, + if (!PageIsVerifiedExtended((Page) scratch.data, tag.blockNum, PIV_LOG_WARNING | PIV_REPORT_STAT)) - ereport( - ERROR, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("invalid page in block %u of relation %u during GCS storage-fallback refresh", - tag.blockNum, tag.relNumber))); + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("invalid page in block %u of relation %u during GCS storage-fallback refresh", + tag.blockNum, tag.relNumber))); content_lock = BufferDescriptorGetContentLock(buf); LWLockAcquire(content_lock, LW_EXCLUSIVE); @@ -10189,15 +10788,16 @@ cluster_bufmgr_refresh_block_from_storage_for_gcs(BufferDesc *buf, SCN *out_page buf_state = LockBufHdr(buf); dirty = (buf_state & BM_DIRTY) != 0; UnlockBufHdr(buf, buf_state); - if (dirty || !write_permitted) { + if (dirty || !write_permitted) + { LWLockRelease(content_lock); return false; } - memcpy((char *)BufHdrGetBlock(buf), scratch.data, BLCKSZ); + memcpy((char *) BufHdrGetBlock(buf), scratch.data, BLCKSZ); LWLockRelease(content_lock); if (out_page_scn != NULL) - *out_page_scn = ((PageHeader)scratch.data)->pd_block_scn; + *out_page_scn = ((PageHeader) scratch.data)->pd_block_scn; return true; } @@ -10241,8 +10841,9 @@ static bool cluster_bufmgr_convert_to_pi_locked(BufferDesc *buf, uint32 buf_state) { if (!cluster_past_image) - return false; /* wave off: caller drops as today */ - if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) { + return false; /* wave off: caller drops as today */ + if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) + { /* Pinned: fail-safe fallback to the plain drop (single lock-free * atomic tick; same class as PG's own buf->state atomics under * this spinlock). */ @@ -10251,7 +10852,7 @@ cluster_bufmgr_convert_to_pi_locked(BufferDesc *buf, uint32 buf_state) } buf_state &= ~(BM_VALID | BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR); - buf->buffer_type = (uint8)BUF_TYPE_PI; + buf->buffer_type = (uint8) BUF_TYPE_PI; /* D-h3a ship-SCN boundary stamp — see the header note on why the * clock sample must sit inside this lock hold. An unarmed clock * stamps InvalidScn and the D-h3 gate fails closed to UNUSABLE. */ @@ -10277,11 +10878,11 @@ cluster_bufmgr_convert_to_pi_locked(BufferDesc *buf, uint32 buf_state) PcmLockMode cluster_bufmgr_block_pcm_state(BufferTag tag) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - uint32 buf_state; + uint32 buf_state; PcmLockMode mode = PCM_LOCK_MODE_N; hashcode = BufTableHashCode(&tag); @@ -10289,7 +10890,8 @@ cluster_bufmgr_block_pcm_state(BufferTag tag) LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) { + if (buf_id < 0) + { LWLockRelease(partition_lock); return PCM_LOCK_MODE_N; } @@ -10297,7 +10899,7 @@ cluster_bufmgr_block_pcm_state(BufferTag tag) buf_state = LockBufHdr(buf); if (BufferTagsEqual(&buf->tag, &tag) && (buf_state & BM_VALID) != 0) - mode = (PcmLockMode)buf->pcm_state; + mode = (PcmLockMode) buf->pcm_state; UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); @@ -10316,19 +10918,20 @@ cluster_bufmgr_block_pcm_state(BufferTag tag) bool cluster_bufmgr_block_grant_pending(BufferTag tag) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - uint32 buf_state; - bool pending = false; + uint32 buf_state; + bool pending = false; hashcode = BufTableHashCode(&tag); partition_lock = BufMappingPartitionLock(hashcode); LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) { + if (buf_id < 0) + { LWLockRelease(partition_lock); return false; } @@ -10377,15 +10980,15 @@ ClusterBufmgrGcsDropResult cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn, XLogRecPtr *out_page_lsn) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - uint32 buf_state; - XLogRecPtr page_lsn = InvalidXLogRecPtr; - bool was_dirty = false; - uint8 saved_pcm_state; - uint64 staged_gen; /* PGRAC W2: ownership gen captured at stage-N */ + uint32 buf_state; + XLogRecPtr page_lsn = InvalidXLogRecPtr; + bool was_dirty = false; + uint8 saved_pcm_state; + uint64 staged_gen; /* PGRAC W2: ownership gen captured at stage-N */ if (out_page_lsn != NULL) *out_page_lsn = InvalidXLogRecPtr; @@ -10395,14 +10998,16 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) { + if (buf_id < 0) + { LWLockRelease(partition_lock); return CLUSTER_BUFMGR_GCS_DROP_NOT_RESIDENT; } buf = GetBufferDescriptor(buf_id); buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) { + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) + { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return CLUSTER_BUFMGR_GCS_DROP_NOT_RESIDENT; @@ -10414,14 +11019,15 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn * retryable deny instead of parking InvalidateBuffer's pin-wait loop in * the dispatch pump). */ - if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) { + if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) + { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return CLUSTER_BUFMGR_GCS_DROP_PINNED; } was_dirty = (buf_state & BM_DIRTY) != 0; { - Page page = (Page)BufHdrGetBlock(buf); + Page page = (Page) BufHdrGetBlock(buf); page_lsn = PageGetLSN(page); } @@ -10439,7 +11045,8 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn * the current page (Rule 8.A — never grant a stale image over a * committed write). expected_lsn == Invalid skips the gate. */ - if (!XLogRecPtrIsInvalid(expected_lsn) && page_lsn != expected_lsn) { + if (!XLogRecPtrIsInvalid(expected_lsn) && page_lsn != expected_lsn) + { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return CLUSTER_BUFMGR_GCS_DROP_STALE; @@ -10471,14 +11078,16 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn * one -- BLOCKER A). */ buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) { + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) + { UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_NOT_RESIDENT; } /* PGRAC: GCS serve-stall round-5 (A2) — a pin acquired during the * XLogFlush window re-fails the drop (nothing observed half-dropped). */ - if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) { + if (BUF_STATE_GET_REFCOUNT(buf_state) != 0) + { UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_PINNED; } @@ -10489,15 +11098,16 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn * during the (blocking) XLogFlush window above. A LSN past expected_lsn * means the ship image is stale; refuse without touching pcm_state * (Rule 8.A). */ - if (!XLogRecPtrIsInvalid(expected_lsn) - && PageGetLSN((Page)BufHdrGetBlock(buf)) != expected_lsn) { + if (!XLogRecPtrIsInvalid(expected_lsn) && + PageGetLSN((Page) BufHdrGetBlock(buf)) != expected_lsn) + { UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_STALE; } saved_pcm_state = buf->pcm_state; - staged_gen = cluster_pcm_own_gen_get(buf->buf_id); /* PGRAC W2 */ - buf->pcm_state = (uint8)PCM_STATE_N; + staged_gen = cluster_pcm_own_gen_get(buf->buf_id); /* PGRAC W2 */ + buf->pcm_state = (uint8) PCM_STATE_N; /* * PGRAC: spec-6.12h D-h1 — keep a Past Image instead of dropping (the @@ -10508,8 +11118,8 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn /* PGRAC: GCS serve-stall round-5 (A2) — bounded drop; restore the * residency mode on a raced pin (mirrors the invalidate wrapper). */ - cluster_bufmgr_in_gcs_drop = true; /* gates the drop-prepin inject */ - if (!InvalidateBufferTry(buf)) /* releases the header spinlock */ + cluster_bufmgr_in_gcs_drop = true; /* gates the drop-prepin inject */ + if (!InvalidateBufferTry(buf)) /* releases the header spinlock */ { cluster_bufmgr_in_gcs_drop = false; @@ -10527,7 +11137,7 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn /* W2 RED force-round — see the twin arm above. */ CLUSTER_INJECTION_POINT("cluster-pcm-restore-aba-force-round"); if (cluster_injection_should_skip("cluster-pcm-restore-aba-force-round")) - cluster_pcm_own_transition(buf, (uint8)PCM_STATE_N, 0, 0); + cluster_pcm_own_transition(buf, (uint8) PCM_STATE_N, 0, 0); buf_state = LockBufHdr(buf); @@ -10538,11 +11148,13 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn * the InvalidateBuffer window does not get a stale pre-drop state * restored over it. */ - if (BufferTagsEqual(&buf->tag, &tag) && buf->pcm_state == (uint8)PCM_STATE_N - && cluster_pcm_own_gen_get(buf->buf_id) == staged_gen) + if (BufferTagsEqual(&buf->tag, &tag) && + buf->pcm_state == (uint8) PCM_STATE_N && + cluster_pcm_own_gen_get(buf->buf_id) == staged_gen) buf->pcm_state = saved_pcm_state; - else if (BufferTagsEqual(&buf->tag, &tag) && buf->pcm_state == (uint8)PCM_STATE_N - && cluster_pcm_own_gen_get(buf->buf_id) != staged_gen) + else if (BufferTagsEqual(&buf->tag, &tag) && + buf->pcm_state == (uint8) PCM_STATE_N && + cluster_pcm_own_gen_get(buf->buf_id) != staged_gen) cluster_pcm_note_restore_aba_detected(); UnlockBufHdr(buf, buf_state); return CLUSTER_BUFMGR_GCS_DROP_PINNED; @@ -10641,7 +11253,8 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, result = CLUSTER_PCM_OWN_STALE; else if ((buf_state & BM_IO_ERROR) != 0) result = CLUSTER_PCM_OWN_CORRUPT; - else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) { + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) + { /* * PGRAC: a dirty N page here is legitimate, not corruption evidence: * relation extension (PageInit + MarkBufferDirty) and recovery redo @@ -10653,13 +11266,14 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, * and report BUSY: one flush converges the state and the image pump * retries against a clean page. */ - PinBuffer_Locked(buf); /* consumes the buffer header lock */ + PinBuffer_Locked(buf); /* consumes the buffer header lock */ LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED); FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); LWLockRelease(BufferDescriptorGetContentLock(buf)); UnpinBuffer(buf); return CLUSTER_PCM_OWN_BUSY; - } else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + } + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) result = CLUSTER_PCM_OWN_BUSY; else { result = cluster_pcm_own_reservation_begin_exact(buf->buf_id, expected_n->generation, @@ -10677,7 +11291,8 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, reln = smgropen(BufTagGetRelFileLocator(&tag), InvalidBackendId); smgrread(reln, BufTagGetForkNum(&tag), tag.blockNum, scratch.data); if (!PageIsVerifiedExtended((Page)scratch.data, tag.blockNum, - PIV_LOG_WARNING | PIV_REPORT_STAT)) { + PIV_LOG_WARNING | PIV_REPORT_STAT)) + { result = CLUSTER_PCM_OWN_CORRUPT; } @@ -10691,13 +11306,15 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, result = CLUSTER_PCM_OWN_STALE; else if ((buf_state & BM_IO_ERROR) != 0) result = CLUSTER_PCM_OWN_CORRUPT; - else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) { + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) + { /* PGRAC: REVOKING already blocks data writes, so dirt appearing * between the flush above and this recheck can only be an * idempotent hint-bit write. Retry via BUSY; the next pass * flushes it and converges. */ result = CLUSTER_PCM_OWN_BUSY; - } else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + } + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) result = CLUSTER_PCM_OWN_BUSY; else { memcpy((char *)BufHdrGetBlock(buf), scratch.data, BLCKSZ); @@ -10754,32 +11371,35 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, * the final byte boundary is serialized by the content lock instead. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_begin_s_revoke(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_s, - ClusterPcmOwnSnapshot *out_revoking) +cluster_bufmgr_pcm_own_begin_s_revoke(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected_s, + ClusterPcmOwnSnapshot *out_revoking) { ClusterPcmOwnResult live_result; ClusterPcmOwnResult result; - uint64 live_token; - uint64 reservation_token = 0; - uint32 flags; - uint32 buf_state; + uint64 live_token; + uint64 reservation_token = 0; + uint32 flags; + uint32 buf_state; if (buf == NULL || expected_s == NULL || out_revoking == NULL) return CLUSTER_PCM_OWN_INVALID; memset(out_revoking, 0, sizeof(*out_revoking)); - if (expected_s->pcm_state != (uint8)PCM_STATE_S || expected_s->flags != 0) + if (expected_s->pcm_state != (uint8) PCM_STATE_S || expected_s->flags != 0) return CLUSTER_PCM_OWN_INVALID; if (ClusterPcmOwnArray == NULL) return CLUSTER_PCM_OWN_NOT_READY; buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &expected_s->tag) || (buf_state & BM_VALID) == 0 + if (!BufferTagsEqual(&buf->tag, &expected_s->tag) + || (buf_state & BM_VALID) == 0 || cluster_pcm_own_gen_get(buf->buf_id) != expected_s->generation - || buf->pcm_state != (uint8)PCM_STATE_S) + || buf->pcm_state != (uint8) PCM_STATE_S) result = CLUSTER_PCM_OWN_STALE; else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) result = CLUSTER_PCM_OWN_CORRUPT; - else { + else + { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -10787,9 +11407,11 @@ cluster_bufmgr_pcm_own_begin_s_revoke(BufferDesc *buf, const ClusterPcmOwnSnapsh result = live_result; else if (live_token != expected_s->reservation_token) result = CLUSTER_PCM_OWN_STALE; - else { + else + { result = cluster_pcm_own_reservation_begin_exact( - buf->buf_id, expected_s->generation, PCM_OWN_FLAG_REVOKING, &reservation_token); + buf->buf_id, expected_s->generation, PCM_OWN_FLAG_REVOKING, + &reservation_token); if (result == CLUSTER_PCM_OWN_OK) cluster_pcm_own_snapshot_locked(buf, out_revoking); } @@ -10801,17 +11423,17 @@ cluster_bufmgr_pcm_own_begin_s_revoke(BufferDesc *buf, const ClusterPcmOwnSnapsh /* Abort only the matching S-source staging reservation. */ ClusterPcmOwnResult cluster_bufmgr_pcm_own_abort_s_revoke(BufferDesc *buf, - const ClusterPcmOwnSnapshot *expected_revoking) + const ClusterPcmOwnSnapshot *expected_revoking) { ClusterPcmOwnResult live_result; ClusterPcmOwnResult result; - uint64 live_token; - uint32 flags; - uint32 buf_state; + uint64 live_token; + uint32 flags; + uint32 buf_state; if (buf == NULL || expected_revoking == NULL) return CLUSTER_PCM_OWN_INVALID; - if (expected_revoking->pcm_state != (uint8)PCM_STATE_S + if (expected_revoking->pcm_state != (uint8) PCM_STATE_S || expected_revoking->flags != PCM_OWN_FLAG_REVOKING || expected_revoking->reservation_token == 0) return CLUSTER_PCM_OWN_INVALID; @@ -10819,13 +11441,15 @@ cluster_bufmgr_pcm_own_abort_s_revoke(BufferDesc *buf, return CLUSTER_PCM_OWN_NOT_READY; buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &expected_revoking->tag) || (buf_state & BM_VALID) == 0 + if (!BufferTagsEqual(&buf->tag, &expected_revoking->tag) + || (buf_state & BM_VALID) == 0 || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation - || buf->pcm_state != (uint8)PCM_STATE_S) + || buf->pcm_state != (uint8) PCM_STATE_S) result = CLUSTER_PCM_OWN_STALE; else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) result = CLUSTER_PCM_OWN_CORRUPT; - else { + else + { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -10839,8 +11463,8 @@ cluster_bufmgr_pcm_own_abort_s_revoke(BufferDesc *buf, result = CLUSTER_PCM_OWN_STALE; else result = cluster_pcm_own_reservation_abort_exact( - buf->buf_id, expected_revoking->generation, expected_revoking->reservation_token, - PCM_OWN_FLAG_REVOKING); + buf->buf_id, expected_revoking->generation, + expected_revoking->reservation_token, PCM_OWN_FLAG_REVOKING); } UnlockBufHdr(buf, buf_state); return result; @@ -10854,32 +11478,35 @@ cluster_bufmgr_pcm_own_abort_s_revoke(BufferDesc *buf, * abort, and retained finish; callers never inspect the ownership sidecar. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_begin_x_revoke(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_x, - ClusterPcmOwnSnapshot *out_revoking) +cluster_bufmgr_pcm_own_begin_x_revoke(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected_x, + ClusterPcmOwnSnapshot *out_revoking) { ClusterPcmOwnResult live_result; ClusterPcmOwnResult result; - uint64 live_token; - uint64 reservation_token = 0; - uint32 flags; - uint32 buf_state; + uint64 live_token; + uint64 reservation_token = 0; + uint32 flags; + uint32 buf_state; if (buf == NULL || expected_x == NULL || out_revoking == NULL) return CLUSTER_PCM_OWN_INVALID; memset(out_revoking, 0, sizeof(*out_revoking)); - if (expected_x->pcm_state != (uint8)PCM_STATE_X || expected_x->flags != 0) + if (expected_x->pcm_state != (uint8) PCM_STATE_X || expected_x->flags != 0) return CLUSTER_PCM_OWN_INVALID; if (ClusterPcmOwnArray == NULL) return CLUSTER_PCM_OWN_NOT_READY; buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &expected_x->tag) || (buf_state & BM_VALID) == 0 + if (!BufferTagsEqual(&buf->tag, &expected_x->tag) + || (buf_state & BM_VALID) == 0 || cluster_pcm_own_gen_get(buf->buf_id) != expected_x->generation - || buf->pcm_state != (uint8)PCM_STATE_X) + || buf->pcm_state != (uint8) PCM_STATE_X) result = CLUSTER_PCM_OWN_STALE; else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) result = CLUSTER_PCM_OWN_CORRUPT; - else { + else + { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -10887,9 +11514,11 @@ cluster_bufmgr_pcm_own_begin_x_revoke(BufferDesc *buf, const ClusterPcmOwnSnapsh result = live_result; else if (live_token != expected_x->reservation_token) result = CLUSTER_PCM_OWN_STALE; - else { + else + { result = cluster_pcm_own_reservation_begin_exact( - buf->buf_id, expected_x->generation, PCM_OWN_FLAG_REVOKING, &reservation_token); + buf->buf_id, expected_x->generation, PCM_OWN_FLAG_REVOKING, + &reservation_token); if (result == CLUSTER_PCM_OWN_OK) cluster_pcm_own_snapshot_locked(buf, out_revoking); } @@ -10903,17 +11532,17 @@ cluster_bufmgr_pcm_own_begin_x_revoke(BufferDesc *buf, const ClusterPcmOwnSnapsh */ ClusterPcmOwnResult cluster_bufmgr_pcm_own_abort_x_revoke(BufferDesc *buf, - const ClusterPcmOwnSnapshot *expected_revoking) + const ClusterPcmOwnSnapshot *expected_revoking) { ClusterPcmOwnResult live_result; ClusterPcmOwnResult result; - uint64 live_token; - uint32 flags; - uint32 buf_state; + uint64 live_token; + uint32 flags; + uint32 buf_state; if (buf == NULL || expected_revoking == NULL) return CLUSTER_PCM_OWN_INVALID; - if (expected_revoking->pcm_state != (uint8)PCM_STATE_X + if (expected_revoking->pcm_state != (uint8) PCM_STATE_X || expected_revoking->flags != PCM_OWN_FLAG_REVOKING || expected_revoking->reservation_token == 0) return CLUSTER_PCM_OWN_INVALID; @@ -10921,13 +11550,15 @@ cluster_bufmgr_pcm_own_abort_x_revoke(BufferDesc *buf, return CLUSTER_PCM_OWN_NOT_READY; buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &expected_revoking->tag) || (buf_state & BM_VALID) == 0 + if (!BufferTagsEqual(&buf->tag, &expected_revoking->tag) + || (buf_state & BM_VALID) == 0 || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation - || buf->pcm_state != (uint8)PCM_STATE_X) + || buf->pcm_state != (uint8) PCM_STATE_X) result = CLUSTER_PCM_OWN_STALE; else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) result = CLUSTER_PCM_OWN_CORRUPT; - else { + else + { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -10941,8 +11572,8 @@ cluster_bufmgr_pcm_own_abort_x_revoke(BufferDesc *buf, result = CLUSTER_PCM_OWN_STALE; else result = cluster_pcm_own_reservation_abort_exact( - buf->buf_id, expected_revoking->generation, expected_revoking->reservation_token, - PCM_OWN_FLAG_REVOKING); + buf->buf_id, expected_revoking->generation, + expected_revoking->reservation_token, PCM_OWN_FLAG_REVOKING); } UnlockBufHdr(buf, buf_state); return result; @@ -11043,30 +11674,30 @@ cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(BufferDesc *buf, * REVOKING token prevents reuse until DRAIN. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_finish_revoke_retain(BufferDesc *buf, - const ClusterPcmOwnSnapshot *expected_revoking, - XLogRecPtr expected_lsn, - ClusterPcmOwnSnapshot *out_retained) +cluster_bufmgr_pcm_own_finish_revoke_retain( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, + XLogRecPtr expected_lsn, ClusterPcmOwnSnapshot *out_retained) { ClusterPcmOwnResult live_result; ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; - BufferTag tag; - LWLock *content_lock; - uint64 committed_generation; - uint64 live_token; - uint32 flags; - uint32 buf_state; - bool source_is_s; - bool source_is_x; + BufferTag tag; + LWLock *content_lock; + uint64 committed_generation; + uint64 live_token; + uint32 flags; + uint32 buf_state; + bool source_is_s; + bool source_is_x; ClusterPcmXRevokeFinishMode finish_mode; if (out_retained != NULL) memset(out_retained, 0, sizeof(*out_retained)); if (buf == NULL || expected_revoking == NULL || out_retained == NULL) return CLUSTER_PCM_OWN_INVALID; - source_is_s = expected_revoking->pcm_state == (uint8)PCM_STATE_S; - source_is_x = expected_revoking->pcm_state == (uint8)PCM_STATE_X; - if ((!source_is_s && !source_is_x) || expected_revoking->flags != PCM_OWN_FLAG_REVOKING + source_is_s = expected_revoking->pcm_state == (uint8) PCM_STATE_S; + source_is_x = expected_revoking->pcm_state == (uint8) PCM_STATE_X; + if ((!source_is_s && !source_is_x) + || expected_revoking->flags != PCM_OWN_FLAG_REVOKING || expected_revoking->reservation_token == 0) return CLUSTER_PCM_OWN_INVALID; if (ClusterPcmOwnArray == NULL) @@ -11084,7 +11715,8 @@ cluster_bufmgr_pcm_own_finish_revoke_retain(BufferDesc *buf, LWLockAcquire(content_lock, LW_EXCLUSIVE); buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0 + if (!BufferTagsEqual(&buf->tag, &tag) + || (buf_state & BM_VALID) == 0 || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation || buf->pcm_state != expected_revoking->pcm_state) result = CLUSTER_PCM_OWN_STALE; @@ -11092,7 +11724,8 @@ cluster_bufmgr_pcm_own_finish_revoke_retain(BufferDesc *buf, result = CLUSTER_PCM_OWN_CORRUPT; else if ((buf_state & BM_IO_IN_PROGRESS) != 0) result = CLUSTER_PCM_OWN_BUSY; - else { + else + { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -11104,18 +11737,21 @@ cluster_bufmgr_pcm_own_finish_revoke_retain(BufferDesc *buf, result = CLUSTER_PCM_OWN_BUSY; else if (live_token != expected_revoking->reservation_token) result = CLUSTER_PCM_OWN_STALE; - else if (PageGetLSN((Page)BufHdrGetBlock(buf)) != expected_lsn) + else if (PageGetLSN((Page) BufHdrGetBlock(buf)) != expected_lsn) result = CLUSTER_PCM_OWN_STALE; } - if (result == CLUSTER_PCM_OWN_OK) { + if (result == CLUSTER_PCM_OWN_OK) + { result = cluster_pcm_own_revoke_retain_commit_exact( - buf->buf_id, expected_revoking->generation, expected_revoking->reservation_token, - &committed_generation); - if (result == CLUSTER_PCM_OWN_OK) { - buf->pcm_state = (uint8)PCM_STATE_N; - buf->buffer_type = (uint8)BUF_TYPE_PI; - buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR); + buf->buf_id, expected_revoking->generation, + expected_revoking->reservation_token, &committed_generation); + if (result == CLUSTER_PCM_OWN_OK) + { + buf->pcm_state = (uint8) PCM_STATE_N; + buf->buffer_type = (uint8) BUF_TYPE_PI; + buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED | + BM_CHECKPOINT_NEEDED | BM_IO_ERROR); cluster_pcm_own_snapshot_locked(buf, out_retained); } } @@ -11129,21 +11765,22 @@ cluster_bufmgr_pcm_own_finish_revoke_retain(BufferDesc *buf, * generation+1 and current token bind a DRAIN to one image round; a delayed * DRAIN cannot clear REVOKING on a later transfer. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, uint64 source_generation) +cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, + uint64 source_generation) { ClusterPcmOwnResult live_result; ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; BufferDesc *buf; - BufferTag lookup_tag; - LWLock *content_lock; - LWLock *partition_lock; - uint64 committed_generation; - uint64 live_token; - uint64 retained_token = 0; - uint32 hashcode; - uint32 flags; - uint32 buf_state; - int buf_id; + BufferTag lookup_tag; + LWLock *content_lock; + LWLock *partition_lock; + uint64 committed_generation; + uint64 live_token; + uint64 retained_token = 0; + uint32 hashcode; + uint32 flags; + uint32 buf_state; + int buf_id; if (tag == NULL || source_generation == UINT64_MAX) return CLUSTER_PCM_OWN_INVALID; @@ -11155,7 +11792,8 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, uint64 sourc partition_lock = BufMappingPartitionLock(hashcode); LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&lookup_tag, hashcode); - if (buf_id < 0) { + if (buf_id < 0) + { LWLockRelease(partition_lock); return CLUSTER_PCM_OWN_STALE; } @@ -11169,11 +11807,14 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, uint64 sourc * ordinary content-lock callers). */ buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, tag) || (buf_state & BM_VALID) == 0 - || buf->buffer_type != (uint8)BUF_TYPE_PI || buf->pcm_state != (uint8)PCM_STATE_N + if (!BufferTagsEqual(&buf->tag, tag) + || (buf_state & BM_VALID) == 0 + || buf->buffer_type != (uint8) BUF_TYPE_PI + || buf->pcm_state != (uint8) PCM_STATE_N || cluster_pcm_own_gen_get(buf->buf_id) != committed_generation) result = CLUSTER_PCM_OWN_STALE; - else { + else + { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -11195,13 +11836,17 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, uint64 sourc LWLockAcquire(content_lock, LW_EXCLUSIVE); buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, tag) || (buf_state & BM_VALID) == 0 - || buf->buffer_type != (uint8)BUF_TYPE_PI || buf->pcm_state != (uint8)PCM_STATE_N + if (!BufferTagsEqual(&buf->tag, tag) + || (buf_state & BM_VALID) == 0 + || buf->buffer_type != (uint8) BUF_TYPE_PI + || buf->pcm_state != (uint8) PCM_STATE_N || cluster_pcm_own_gen_get(buf->buf_id) != committed_generation) result = CLUSTER_PCM_OWN_STALE; - else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR)) != 0) + else if ((buf_state + & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR)) != 0) result = CLUSTER_PCM_OWN_CORRUPT; - else { + else + { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -11214,8 +11859,8 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, uint64 sourc else if (live_token != retained_token) result = CLUSTER_PCM_OWN_STALE; else { - result = cluster_pcm_own_revoke_retain_release_exact(buf->buf_id, committed_generation, - retained_token); + result = cluster_pcm_own_revoke_retain_release_exact( + buf->buf_id, committed_generation, retained_token); if (result == CLUSTER_PCM_OWN_OK) { /* * The exact token is the last stale-write fence. Release it @@ -11310,23 +11955,25 @@ cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, uint64 source_ bool cluster_bufmgr_block_is_pi(BufferTag tag) { - uint32 hash = BufTableHashCode(&tag); - LWLock *partition_lock = BufMappingPartitionLock(hash); - int buf_id; + uint32 hash = BufTableHashCode(&tag); + LWLock *partition_lock = BufMappingPartitionLock(hash); + int buf_id; BufferDesc *buf; - uint32 buf_state; - bool is_pi; + uint32 buf_state; + bool is_pi; LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hash); - if (buf_id < 0) { + if (buf_id < 0) + { LWLockRelease(partition_lock); return false; } buf = GetBufferDescriptor(buf_id); buf_state = LockBufHdr(buf); - is_pi = BufferTagsEqual(&buf->tag, &tag) && buf->buffer_type == (uint8)BUF_TYPE_PI - && (buf_state & BM_VALID) == 0; + is_pi = BufferTagsEqual(&buf->tag, &tag) + && buf->buffer_type == (uint8) BUF_TYPE_PI + && (buf_state & BM_VALID) == 0; UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); @@ -11336,11 +11983,11 @@ cluster_bufmgr_block_is_pi(BufferTag tag) bool cluster_bufmgr_discard_pi_block(BufferTag tag) { - uint32 hash = BufTableHashCode(&tag); - LWLock *partition_lock = BufMappingPartitionLock(hash); - int buf_id; + uint32 hash = BufTableHashCode(&tag); + LWLock *partition_lock = BufMappingPartitionLock(hash); + int buf_id; BufferDesc *buf; - uint32 buf_state; + uint32 buf_state; LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hash); @@ -11350,8 +11997,11 @@ cluster_bufmgr_discard_pi_block(BufferTag tag) buf = GetBufferDescriptor(buf_id); buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || buf->buffer_type != (uint8)BUF_TYPE_PI - || (buf_state & BM_VALID) != 0 || BUF_STATE_GET_REFCOUNT(buf_state) != 0) { + if (!BufferTagsEqual(&buf->tag, &tag) + || buf->buffer_type != (uint8) BUF_TYPE_PI + || (buf_state & BM_VALID) != 0 + || BUF_STATE_GET_REFCOUNT(buf_state) != 0) + { UnlockBufHdr(buf, buf_state); return false; } @@ -11360,14 +12010,14 @@ cluster_bufmgr_discard_pi_block(BufferTag tag) * A PI carries no residency claim (D-h1 set pcm_state N at conversion), * but clear it again under the lock so InvalidateBuffer's eviction hook * can never see a stale mode and emit a release wire from LMON. */ - buf->pcm_state = (uint8)PCM_STATE_N; + buf->pcm_state = (uint8) PCM_STATE_N; /* PGRAC: spec-6.12h D-h3a — hygiene: drop the shadow stamp with the PI. * Correctness never depends on this clear (the D-h3 consumer * re-validates the PI shape + tag under this same header lock, and * InvalidateBuffer below breaks the shape), but a directive-discarded * slot should not linger as a plausible-looking stamp. */ cluster_pi_shadow_clear(buf->buf_id); - InvalidateBuffer(buf); /* releases the header spinlock */ + InvalidateBuffer(buf); /* releases the header spinlock */ return true; } @@ -11396,50 +12046,57 @@ cluster_bufmgr_discard_pi_block(BufferTag tag) bool cluster_bufmgr_snapshot_pi_block(BufferTag tag, char *dst, SCN *out_ship_scn) { - uint32 hash = BufTableHashCode(&tag); - LWLock *partition_lock = BufMappingPartitionLock(hash); - int buf_id; + uint32 hash = BufTableHashCode(&tag); + LWLock *partition_lock = BufMappingPartitionLock(hash); + int buf_id; BufferDesc *buf; - uint32 buf_state; - SCN ship_scn; - bool intact; + uint32 buf_state; + SCN ship_scn; + bool intact; *out_ship_scn = InvalidScn; LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hash); - if (buf_id < 0) { + if (buf_id < 0) + { LWLockRelease(partition_lock); return false; } buf = GetBufferDescriptor(buf_id); buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || buf->buffer_type != (uint8)BUF_TYPE_PI - || (buf_state & BM_VALID) != 0 || (buf_state & BM_TAG_VALID) == 0 - || (buf_state & BM_IO_IN_PROGRESS) != 0) { + if (!BufferTagsEqual(&buf->tag, &tag) + || buf->buffer_type != (uint8) BUF_TYPE_PI + || (buf_state & BM_VALID) != 0 + || (buf_state & BM_TAG_VALID) == 0 + || (buf_state & BM_IO_IN_PROGRESS) != 0) + { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; } ship_scn = cluster_pi_shadow_read(buf->buf_id); - if (!SCN_VALID(ship_scn)) { + if (!SCN_VALID(ship_scn)) + { /* Unstamped PI (clock unarmed at conversion): the recovery boundary * is unprovable, so the PI is useless as a base (gate UNUSABLE). */ UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); return false; } - cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); /* raw-pin + unlock header */ + cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); /* raw-pin + unlock header */ LWLockRelease(partition_lock); - memcpy(dst, (char *)BufHdrGetBlock(buf), BLCKSZ); + memcpy(dst, (char *) BufHdrGetBlock(buf), BLCKSZ); buf_state = LockBufHdr(buf); /* SCN_CMP_OK: identity recheck of the same slot (same-stamp equality), * not a Lamport-order comparison. */ - intact = BufferTagsEqual(&buf->tag, &tag) && buf->buffer_type == (uint8)BUF_TYPE_PI - && (buf_state & BM_VALID) == 0 && cluster_pi_shadow_read(buf->buf_id) == ship_scn; + intact = BufferTagsEqual(&buf->tag, &tag) + && buf->buffer_type == (uint8) BUF_TYPE_PI + && (buf_state & BM_VALID) == 0 + && cluster_pi_shadow_read(buf->buf_id) == ship_scn; UnlockBufHdr(buf, buf_state); cluster_bufmgr_unpin_for_gcs(buf); @@ -11470,7 +12127,7 @@ cluster_bufmgr_flush_seq_page_to_storage(Buffer buffer) { Assert(BufferIsValid(buffer)); if (BufferIsLocal(buffer)) - return; /* temp/local buffers are never CF-shared */ + return; /* temp/local buffers are never CF-shared */ FlushOneBuffer(buffer); } @@ -11522,8 +12179,8 @@ cluster_bufmgr_flush_seq_page_to_storage(Buffer buffer) uint32 cluster_bufmgr_flush_and_release_x_for_leave(void) { - uint32 flushed = 0; - int i; + uint32 flushed = 0; + int i; /* CL-I9: must run with a real ResourceOwner (backend/checkpointer). */ Assert(CurrentResourceOwner != NULL); @@ -11534,20 +12191,22 @@ cluster_bufmgr_flush_and_release_x_for_leave(void) */ ResourceOwnerEnlargeBuffers(CurrentResourceOwner); - for (i = 0; i < NBuffers; i++) { + for (i = 0; i < NBuffers; i++) + { BufferDesc *bufHdr = GetBufferDescriptor(i); - uint32 buf_state; + uint32 buf_state; ReservePrivateRefCountEntry(); buf_state = LockBufHdr(bufHdr); - if ((buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY) - && (PcmState)bufHdr->pcm_state == PCM_STATE_X) { + if ((buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY) && + (PcmState) bufHdr->pcm_state == PCM_STATE_X) + { ClusterPcmOwnResult bump_result; - uint32 observed_flags = 0; - uint64 observed_generation = 0; + uint32 observed_flags = 0; + uint64 observed_generation = 0; - PinBuffer_Locked(bufHdr); /* releases the header spinlock */ + PinBuffer_Locked(bufHdr); /* releases the header spinlock */ LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED); FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); @@ -11560,20 +12219,22 @@ cluster_bufmgr_flush_and_release_x_for_leave(void) buf_state = LockBufHdr(bufHdr); /* PGRAC ownership-gen: bump under the held spinlock (X -> N release * after the block is durable on shared storage). */ - bump_result - = cluster_pcm_own_bump_locked(bufHdr, 0, 0, &observed_generation, &observed_flags); - if (bump_result != CLUSTER_PCM_OWN_OK) { + bump_result = cluster_pcm_own_bump_locked(bufHdr, 0, 0, + &observed_generation, &observed_flags); + if (bump_result != CLUSTER_PCM_OWN_OK) + { UnlockBufHdr(bufHdr, buf_state); UnpinBuffer(bufHdr); cluster_pcm_own_report_bump_failure(bufHdr, bump_result, observed_generation, - observed_flags, "clean-leave X release"); + observed_flags, "clean-leave X release"); } - bufHdr->pcm_state = (uint8)PCM_STATE_N; + bufHdr->pcm_state = (uint8) PCM_STATE_N; UnlockBufHdr(bufHdr, buf_state); UnpinBuffer(bufHdr); flushed++; - } else + } + else UnlockBufHdr(bufHdr, buf_state); } @@ -11615,18 +12276,19 @@ bool cluster_bufmgr_block_write_permitted(Buffer buffer) { BufferDesc *buf; - PcmState state; - uint32 buf_state; - uint32 own_flags; + PcmState state; + uint32 buf_state; + uint32 own_flags; if (BufferIsLocal(buffer)) - return true; /* temp / local buffers are never CF-shared */ + return true; /* temp / local buffers are never CF-shared */ buf = GetBufferDescriptor(buffer - 1); buf_state = LockBufHdr(buf); - state = (PcmState)buf->pcm_state; + state = (PcmState) buf->pcm_state; own_flags = cluster_pcm_own_flags_get(buf->buf_id); if (cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state) - || (own_flags & PCM_OWN_FLAG_REVOKING) != 0) { + || (own_flags & PCM_OWN_FLAG_REVOKING) != 0) + { UnlockBufHdr(buf, buf_state); return false; } @@ -11634,8 +12296,8 @@ cluster_bufmgr_block_write_permitted(Buffer buffer) if (state == PCM_STATE_READ_IMAGE) return false; if (cluster_read_scache && state == PCM_STATE_S) - return false; /* spec-6.12a: S is a revoked-write state */ + return false; /* spec-6.12a: S is a revoked-write state */ return true; } -#endif /* USE_PGRAC_CLUSTER */ +#endif /* USE_PGRAC_CLUSTER */ From 10cd1f2c491467fd98f7de362425a0c6ab8d8ce5 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 05:48:14 -0700 Subject: [PATCH 12/56] fix(cluster): forward-declare pcm_x_domain_slot for the early dump iterators --- src/backend/cluster/cluster_pcm_x_convert.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index d635c21d72..bbb506f86d 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -562,6 +562,8 @@ typedef enum PcmXDirectoryMatch { static void pcm_x_runtime_fail_closed_impl(const char *site_file, int site_line); +static PcmXSlotHeader *pcm_x_domain_slot(PcmXAllocatorKind kind, PcmXSlotRef ref, + const BufferTag *expected_tag, uint32 allowed_state_mask); /* Capture each internal fail-closed arm (file:line) while keeping the many * zero-argument call sites in this file textually unchanged. */ From 2daddda210e7ab2364c9cbe432af368cd8752c0f Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 05:59:30 -0700 Subject: [PATCH 13/56] feat(cluster): local-tag dump, terminal-kick note, fail-closed exit lines Completes the evidence surfaces for the t/400 L3 chain per the root-cause audit: - pcm_x_ltag_ rows: local tag slot flags/round/refs/drain generations under the local partition lock, exposing the foreign REVOKE_BARRIER shape on remote participants. - pcm_x_terminal_last_note row: the arm and result that terminated the most recent terminal kick (detach / leg arm / stage / local retire apply / ack resolve) with a same-shape repetition count, closing the counter-free terminal path. - The 33 goto requester_fail_closed sites now record their source line too, so a client-visible failure names the arm even through the fail-closed fallthrough (this run proved cleanup PRESERVE_FAIL_CLOSED at cluster_gcs_block.c:8226 fusing node2 after a STALE acquire). - PcmXShmemHeader 36480 -> 36504 with pins synced. --- src/backend/cluster/cluster_debug.c | 21 +++- src/backend/cluster/cluster_gcs_block.c | 103 +++++++++++------- src/backend/cluster/cluster_pcm_x_convert.c | 101 +++++++++++++++++ src/include/cluster/cluster_pcm_x_convert.h | 27 ++++- .../t/400_pcm_x_queue_4node_liveness.pl | 3 +- src/test/cluster_unit/test_cluster_debug.c | 13 +++ .../cluster_unit/test_cluster_pcm_x_convert.c | 2 +- 7 files changed, 229 insertions(+), 41 deletions(-) diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 53c8d6ad4c..a73b0aa22b 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -2017,7 +2017,7 @@ dump_pcm(ReturnSetInfo *rsinfo) { Size cursor = 0; Size index = 0; - char line[256]; + char line[384]; char key[64]; while (cluster_pcm_x_master_tag_debug_next(&cursor, &index, line, sizeof(line))) { @@ -2029,6 +2029,25 @@ dump_pcm(ReturnSetInfo *rsinfo) snprintf(key, sizeof(key), "pcm_x_ticket_%zu", index); emit_row(rsinfo, "pcm", key, line); } + cursor = 0; + while (cluster_pcm_x_local_tag_debug_next(&cursor, &index, line, sizeof(line))) { + snprintf(key, sizeof(key), "pcm_x_ltag_%zu", index); + emit_row(rsinfo, "pcm", key, line); + } + { + uint32 note_op = 0; + uint32 note_result = 0; + uint32 note_count = 0; + uint64 note_ticket = 0; + + if (cluster_pcm_x_terminal_note_read(¬e_op, ¬e_result, ¬e_ticket, + ¬e_count)) { + snprintf(line, sizeof(line), + "op=%u result=%u ticket=" UINT64_FORMAT " count=%u", note_op, + note_result, note_ticket, note_count); + emit_row(rsinfo, "pcm", "pcm_x_terminal_last_note", line); + } + } } emit_row(rsinfo, "pcm", "pcm_x_queue_enqueue_count", fmt_int64((int64)stats.enqueue_count)); emit_row(rsinfo, "pcm", "pcm_x_queue_admit_count", fmt_int64((int64)stats.admit_count)); diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 497eff39b5..ae1de65759 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -8086,6 +8086,12 @@ static int gcs_block_pcm_x_requester_fail_line = 0; goto requester_done; \ } while (0) +#define GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED() \ + do { \ + gcs_block_pcm_x_requester_fail_line = __LINE__; \ + goto requester_fail_closed; \ + } while (0) + int cluster_gcs_pcm_x_requester_last_fail_line(void) { @@ -8489,7 +8495,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (result == PCM_X_QUEUE_OK) break; if (result == PCM_X_QUEUE_CORRUPT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); if (!cluster_gcs_pcm_x_nested_guard_retryable(result)) GCS_BLOCK_PCM_X_REQUESTER_DONE(); if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, @@ -8507,7 +8513,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim result = gcs_block_pcm_x_fetch_own_result(own_result); if (result != PCM_X_QUEUE_OK) { if (result == PCM_X_QUEUE_CORRUPT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); GCS_BLOCK_PCM_X_REQUESTER_DONE(); } if (!BufferTagsEqual(&initial_own.tag, &buf->tag) || initial_own.generation == UINT64_MAX) { @@ -8518,7 +8524,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim = cluster_pcm_own_classify_live_flags(initial_own.flags, initial_own.reservation_token); result = gcs_block_pcm_x_fetch_own_result(live_result); if (result == PCM_X_QUEUE_CORRUPT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); if (initial_own.pcm_state == (uint8)PCM_STATE_READ_IMAGE || result == PCM_X_QUEUE_BUSY) { if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, cluster_epoch, master_session)) { @@ -8557,7 +8563,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action(GCS_BLOCK_PCM_X_RETRY_SITE_JOIN, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, cluster_epoch, master_session)) { result = PCM_X_QUEUE_NOT_READY; @@ -8582,7 +8588,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_LEADER_REKEY, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, cluster_epoch, master_session)) { result = PCM_X_QUEUE_NOT_READY; @@ -8601,7 +8607,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_CLAIM, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, cluster_epoch, master_session)) { result = PCM_X_QUEUE_NOT_READY; @@ -8620,7 +8626,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_CUTOFF, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); gcs_block_pcm_x_requester_cleanup_context.cutoff_started = false; if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, cluster_epoch, master_session)) { @@ -8639,7 +8645,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim = cluster_lmd_graph_remove_edge_by_waiter_exact_result(&waiter_vertex); if (remove_result == CLUSTER_LMD_GRAPH_REMOVE_STALE) { cluster_lmd_pcm_convert_wfg_note_exact_remove_stale(); - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } if (remove_result == CLUSTER_LMD_GRAPH_REMOVE_REMOVED) cluster_lmd_pcm_convert_wfg_note_remove(); @@ -8647,7 +8653,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim follower_graph_generation); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); follower_graph_generation = 0; gcs_block_pcm_x_requester_cleanup_context.wfg_live = false; gcs_block_pcm_x_requester_cleanup_context.wfg_generation = 0; @@ -8667,7 +8673,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (result == PCM_X_QUEUE_OK) { fail_site = "follower-refresh-compare"; if (!cluster_gcs_pcm_x_role_refresh_exact(&handle, &fresh_handle)) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); handle = fresh_handle; gcs_block_pcm_x_requester_cleanup_context.handle = handle; goto requester_role_dispatch; @@ -8683,12 +8689,12 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim } continue; } - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_CLAIM, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); fail_site = "follower-wfg-snapshot"; result = cluster_pcm_x_local_follower_wfg_snapshot_exact(&handle, &follower_snapshot); @@ -8701,7 +8707,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim &waiter_vertex, &blocker_vertex, 1, handle.identity.request_id); if (follower_graph_generation == 0) { cluster_lmd_pcm_convert_wfg_note_replace_fail(); - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } cluster_lmd_pcm_convert_wfg_note_replace(); gcs_block_pcm_x_requester_cleanup_context.wfg_live = true; @@ -8717,7 +8723,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim = cluster_lmd_graph_remove_edge_by_waiter_exact_result(&waiter_vertex); if (remove_result == CLUSTER_LMD_GRAPH_REMOVE_STALE) { cluster_lmd_pcm_convert_wfg_note_exact_remove_stale(); - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } if (remove_result == CLUSTER_LMD_GRAPH_REMOVE_REMOVED) cluster_lmd_pcm_convert_wfg_note_remove(); @@ -8725,7 +8731,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim gcs_block_pcm_x_requester_cleanup_context.wfg_live = false; gcs_block_pcm_x_requester_cleanup_context.wfg_generation = 0; if (retry_action != GCS_BLOCK_PCM_X_RETRY_RESNAPSHOT_WFG) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } } else { retry_action = cluster_gcs_pcm_x_requester_retry_action( @@ -8742,7 +8748,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (result == PCM_X_QUEUE_OK) { fail_site = "follower-refresh-compare"; if (!cluster_gcs_pcm_x_role_refresh_exact(&handle, &fresh_handle)) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); handle = fresh_handle; gcs_block_pcm_x_requester_cleanup_context.handle = handle; goto requester_role_dispatch; @@ -8751,7 +8757,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim GCS_BLOCK_PCM_X_RETRY_SITE_ROLE_REFRESH, result); } if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, cluster_epoch, master_session)) { @@ -8763,7 +8769,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim GCS_BLOCK_PCM_X_REQUESTER_DONE(); } else { result = PCM_X_QUEUE_CORRUPT; - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } /* Node leader: every iteration reconstructs the exact durable outbound @@ -8779,12 +8785,12 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_PROGRESS, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); goto requester_wait; } if (progress.role != PCM_X_LOCAL_ROLE_NODE_LEADER || progress.master_node != master_node || progress.master_session_incarnation != master_session) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); if (progress.member_state == PCM_XL_NODE_LEADER) { if ((progress.pending_opcode == 0 && progress.last_response_opcode == 0) @@ -8803,7 +8809,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, arm_result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT && retry_action != GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } } else if (progress.pending_opcode == PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM || (progress.pending_opcode == 0 @@ -8822,7 +8828,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, arm_result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT && retry_action != GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } } } else if (progress.member_state == PCM_XL_REMOTE_WAIT) { @@ -8835,7 +8841,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim own_result = cluster_bufmgr_pcm_own_snapshot(buf, &reservation_base); result = gcs_block_pcm_x_fetch_own_result(own_result); if (result != PCM_X_QUEUE_OK) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); self_source = progress.image.source_node == (uint32)cluster_node_id; if (self_source) result = cluster_gcs_pcm_x_adopt_self_image(buf, &handle, &reservation_base, @@ -8845,14 +8851,14 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim &progress.identity); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); own_result = cluster_bufmgr_pcm_own_begin_x_reservation( buf, &reservation_base, &reservation_token); result = gcs_block_pcm_x_fetch_own_result(own_result); } cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); reservation_started = true; gcs_block_pcm_x_requester_cleanup_context.reservation_started = true; } @@ -8869,7 +8875,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_IMAGE_FETCH, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } } if (image_installed) { @@ -8886,7 +8892,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, arm_result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT && retry_action != GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } } } @@ -8908,7 +8914,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, arm_result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT && retry_action != GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } } else if (progress.pending_opcode == 0 && progress.last_response_opcode == PGRAC_IC_MSG_PCM_X_COMMIT_X) { @@ -8927,7 +8933,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim } cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); ownership_committed = true; gcs_block_pcm_x_requester_cleanup_context.ownership_committed = true; } @@ -8939,7 +8945,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim staged = cluster_gcs_pcm_x_stage_frame( PGRAC_IC_MSG_PCM_X_FINAL_ACK, master_node, &final_ack, sizeof(final_ack)); else - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } else if (progress.pending_opcode == PGRAC_IC_MSG_PCM_X_FINAL_ACK) { PcmXFinalAckPayload final_ack; PcmXLocalReliableToken token; @@ -8955,7 +8961,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_REPLAY_ARM, arm_result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } } } else if (progress.member_state == PCM_XL_GRANTED) { @@ -8976,9 +8982,9 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim result = PCM_X_QUEUE_OK; break; } else - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } else - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); requester_wait: if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, @@ -11951,6 +11957,8 @@ cluster_gcs_pcm_x_terminal_kick(const PcmXTicketRef *ref) if (ref->grant_generation == 0) { result = gcs_block_pcm_x_cancel_terminal_cleanup_exact(ref); cluster_pcm_x_stats_note_queue_result(result); + cluster_pcm_x_terminal_note(PCM_X_TERMINAL_NOTE_CANCEL_CLEANUP, (uint32)result, + ref->handle.ticket_id); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { gcs_block_pcm_x_master_drive_fail_closed(result); return; @@ -11988,6 +11996,8 @@ cluster_gcs_pcm_x_terminal_kick(const PcmXTicketRef *ref) /* All ACKs may already be present. Detach first; global ticket order is * enforced inside the engine and the periodic oldest-ticket scan retries. */ result = cluster_pcm_x_master_detach_terminal_exact(ref); + cluster_pcm_x_terminal_note(PCM_X_TERMINAL_NOTE_DETACH, (uint32)result, + ref->handle.ticket_id); if (result == PCM_X_QUEUE_OK) { gcs_block_pcm_x_master_drive_tag(&ref->identity.tag, ref->identity.cluster_epoch); return; @@ -12038,8 +12048,13 @@ cluster_gcs_pcm_x_terminal_kick(const PcmXTicketRef *ref) ref, (PcmXTerminalLegKind)kind, node, responder_session, &token); if (result == PCM_X_QUEUE_STALE || result == PCM_X_QUEUE_NOT_READY) continue; - if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { + cluster_pcm_x_terminal_note(kind == PCM_X_TERMINAL_LEG_DRAIN + ? PCM_X_TERMINAL_NOTE_ARM_DRAIN + : PCM_X_TERMINAL_NOTE_ARM_RETIRE, + (uint32)result, ref->handle.ticket_id); return; + } if (kind == PCM_X_TERMINAL_LEG_DRAIN) { memset(&poll, 0, sizeof(poll)); poll.ref = *ref; @@ -12047,8 +12062,13 @@ cluster_gcs_pcm_x_terminal_kick(const PcmXTicketRef *ref) /* Even self DRAIN is application traffic. Dispatch it through * the tag shard so holder-byte release and terminal publication * remain one single-consumer action. */ - (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_DRAIN_POLL, node, &poll, - sizeof(poll)); + cluster_pcm_x_terminal_note( + PCM_X_TERMINAL_NOTE_DRAIN_STAGE, + cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_DRAIN_POLL, node, &poll, + sizeof(poll)) + ? (uint32)PCM_X_QUEUE_OK + : (uint32)PCM_X_QUEUE_NOT_READY, + ref->handle.ticket_id); return; } else { runtime = cluster_pcm_x_runtime_snapshot(); @@ -12058,15 +12078,24 @@ cluster_gcs_pcm_x_terminal_kick(const PcmXTicketRef *ref) retire.retire_through_ticket_id = ref->handle.ticket_id; retire.sender_node = node; if (node != cluster_node_id) { - (void)gcs_block_pcm_x_stage_retire_up_to(node, &retire, &ref->identity.tag); + cluster_pcm_x_terminal_note( + PCM_X_TERMINAL_NOTE_RETIRE_STAGE, + gcs_block_pcm_x_stage_retire_up_to(node, &retire, &ref->identity.tag) + ? (uint32)PCM_X_QUEUE_OK + : (uint32)PCM_X_QUEUE_NOT_READY, + ref->handle.ticket_id); return; } result = gcs_block_pcm_x_local_retire_apply_and_wake(&retire, cluster_node_id, responder_session); + cluster_pcm_x_terminal_note(PCM_X_TERMINAL_NOTE_RETIRE_LOCAL, (uint32)result, + ref->handle.ticket_id); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) return; result = cluster_pcm_x_master_retire_ack_resolve_exact( &retire, cluster_node_id, responder_session, &resolved); + cluster_pcm_x_terminal_note(PCM_X_TERMINAL_NOTE_RETIRE_ACK_RESOLVE, + (uint32)result, ref->handle.ticket_id); } if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) return; diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index bbb506f86d..b01c9c313b 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -564,6 +564,8 @@ typedef enum PcmXDirectoryMatch { static void pcm_x_runtime_fail_closed_impl(const char *site_file, int site_line); static PcmXSlotHeader *pcm_x_domain_slot(PcmXAllocatorKind kind, PcmXSlotRef ref, const BufferTag *expected_tag, uint32 allowed_state_mask); +static void pcm_x_local_gate_acquire_guarded(LWLock *lock, LWLockMode mode, + PcmXLocalTagSlot *tag_slot); /* Capture each internal fail-closed arm (file:line) while keeping the many * zero-argument call sites in this file textually unchanged. */ @@ -1240,6 +1242,105 @@ cluster_pcm_x_master_ticket_debug_next(Size *cursor_io, Size *index_out, char *b } +/* Diagnostic iterator over live local tag slots; partition-lock coherent copy + * like the master iterators, but under the local lock partition. */ +bool +cluster_pcm_x_local_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, Size buflen) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXAllocatorView view; + PcmXLocalTagSlot copy; + Size i; + + if (header == NULL || cursor_io == NULL || index_out == NULL || buf == NULL || buflen == 0) + return false; + if (!pcm_x_allocator_entry_unlocked(header) + || !pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_TAG, &view)) + return false; + for (i = *cursor_io; i < view.capacity; i++) { + PcmXLocalTagSlot *raw = (PcmXLocalTagSlot *)pcm_x_allocator_slot(&view, i); + PcmXLocalTagSlot *locked; + PcmXSlotRef tag_ref; + BufferTag tag; + uint32 partition; + + if (raw == NULL || pcm_x_slot_state_read(&raw->slot) != PCM_X_TAG_LIVE + || !pcm_x_slot_generation_read(&raw->slot, &tag_ref.slot_generation)) + continue; + tag_ref.slot_index = i; + tag = raw->tag; + pg_read_barrier(); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&tag)); + pcm_x_local_gate_acquire_guarded(&header->local_locks[partition].lock, LW_SHARED, NULL); + locked = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, tag_ref, &tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (locked == NULL) { + LWLockRelease(&header->local_locks[partition].lock); + continue; + } + memcpy(©, locked, sizeof(copy)); + LWLockRelease(&header->local_locks[partition].lock); + snprintf(buf, buflen, + "rel=%u blk=%u flags=0x%x round=%u master=%d ref_node=%d ref_ticket=" UINT64_FORMAT + " ref_grant=" UINT64_FORMAT " holder_ticket=" UINT64_FORMAT + " blocker_ticket=" UINT64_FORMAT " drain_gen=" UINT64_FORMAT + " holder_drain_gen=" UINT64_FORMAT + " members=%zu head=%ld leader=%ld active_writer=%ld", + copy.tag.relNumber, copy.tag.blockNum, pcm_x_slot_flags_read(©.slot), + copy.local_round, copy.master_node, copy.ref.identity.node_id, + copy.ref.handle.ticket_id, copy.ref.grant_generation, + copy.holder_ref.handle.ticket_id, copy.blocker_snapshot_ref.handle.ticket_id, + copy.terminal_drain_generation, copy.holder_terminal_drain_generation, + copy.membership_count, pcm_x_debug_index(copy.head_index), + pcm_x_debug_index(copy.leader_index), pcm_x_debug_index(copy.active_writer_index)); + *index_out = i; + *cursor_io = i + 1; + return true; + } + *cursor_io = view.capacity; + return false; +} + + +void +cluster_pcm_x_terminal_note(uint32 op, uint32 result, uint64 ticket_id) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (header == NULL) + return; + if (header->terminal_note_op == op && header->terminal_note_result == result + && header->terminal_note_ticket == ticket_id) + header->terminal_note_count++; + else { + header->terminal_note_op = op; + header->terminal_note_result = result; + header->terminal_note_ticket = ticket_id; + header->terminal_note_count = 1; + } +} + + +bool +cluster_pcm_x_terminal_note_read(uint32 *op_out, uint32 *result_out, uint64 *ticket_out, + uint32 *count_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (header == NULL || header->terminal_note_count == 0) + return false; + if (op_out != NULL) + *op_out = header->terminal_note_op; + if (result_out != NULL) + *result_out = header->terminal_note_result; + if (ticket_out != NULL) + *ticket_out = header->terminal_note_ticket; + if (count_out != NULL) + *count_out = header->terminal_note_count; + return true; +} + + /* Reserve one slot under the sole allocator authority. */ PcmXAllocatorResult cluster_pcm_x_allocator_reserve(PcmXAllocatorKind kind, PcmXSlotRef *ref_out, diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h index c2fbd809eb..7a498b6ea0 100644 --- a/src/include/cluster/cluster_pcm_x_convert.h +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -1207,8 +1207,27 @@ typedef struct PcmXShmemHeader { * Readers are diagnostic-only (pg_cluster_state dump) and tolerate an * in-progress overwrite after such a reactivation. */ char fail_closed_site[PCM_X_FAIL_CLOSED_SITE_LEN]; + /* Terminal-kick termination evidence: the arm (PCM_X_TERMINAL_NOTE_*) and + * result that ended the most recent terminal kick on this node, with a + * same-shape repetition count. Last-writer-wins diagnostics only. */ + uint32 terminal_note_op; + uint32 terminal_note_result; + uint32 terminal_note_count; + uint32 terminal_note_reserved; + uint64 terminal_note_ticket; } PcmXShmemHeader; +/* Terminal-kick note arms (diagnostic identifiers, not protocol state). */ +#define PCM_X_TERMINAL_NOTE_NONE 0 +#define PCM_X_TERMINAL_NOTE_DETACH 1 +#define PCM_X_TERMINAL_NOTE_CANCEL_CLEANUP 2 +#define PCM_X_TERMINAL_NOTE_ARM_DRAIN 3 +#define PCM_X_TERMINAL_NOTE_ARM_RETIRE 4 +#define PCM_X_TERMINAL_NOTE_DRAIN_STAGE 5 +#define PCM_X_TERMINAL_NOTE_RETIRE_STAGE 6 +#define PCM_X_TERMINAL_NOTE_RETIRE_LOCAL 7 +#define PCM_X_TERMINAL_NOTE_RETIRE_ACK_RESOLVE 8 + StaticAssertDecl(sizeof(PcmXShmemLayout) == 440, "PCM-X shmem layout ABI"); StaticAssertDecl(sizeof(PcmXAllocatorState) == 32, "PCM-X allocator state ABI"); StaticAssertDecl(sizeof(PcmXPeerFrontier) == 48, "PCM-X peer frontier ABI"); @@ -1233,7 +1252,7 @@ StaticAssertDecl(offsetof(PcmXShmemHeader, peer_frontiers) == 33664, StaticAssertDecl(offsetof(PcmXShmemHeader, stats) == 35200, "PCM-X stats offset"); StaticAssertDecl(offsetof(PcmXShmemHeader, outbound_targets) == 35376, "PCM-X outbound target frontier array offset"); -StaticAssertDecl(sizeof(PcmXShmemHeader) == 36480, "PCM-X shmem header ABI"); +StaticAssertDecl(sizeof(PcmXShmemHeader) == 36504, "PCM-X shmem header ABI"); typedef enum PcmXAttachResult { PCM_X_ATTACH_OK = 0, @@ -1306,6 +1325,12 @@ extern bool cluster_pcm_x_master_tag_debug_next(Size *cursor_io, Size *index_out Size buflen); extern bool cluster_pcm_x_master_ticket_debug_next(Size *cursor_io, Size *index_out, char *buf, Size buflen); +extern bool cluster_pcm_x_local_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, + Size buflen); +/* Terminal-kick termination note: which arm/result ended the latest kick. */ +extern void cluster_pcm_x_terminal_note(uint32 op, uint32 result, uint64 ticket_id); +extern bool cluster_pcm_x_terminal_note_read(uint32 *op_out, uint32 *result_out, uint64 *ticket_out, + uint32 *count_out); extern PcmXStepResult cluster_pcm_x_master_step(PcmXMasterTicketState current, PcmXMasterEvent event, uint32 guards, PcmXMasterTicketState *next); diff --git a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl index be688942b4..1d409e7c5c 100644 --- a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl +++ b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl @@ -468,7 +468,8 @@ sub write_file q{SELECT string_agg(key || '=[' || value || ']', ' ' ORDER BY key) FROM pg_cluster_state WHERE category = 'pcm' - AND (key LIKE 'pcm_x_tag_%' OR key LIKE 'pcm_x_ticket_%')}, + AND (key LIKE 'pcm_x_tag_%' OR key LIKE 'pcm_x_ticket_%' + OR key LIKE 'pcm_x_ltag_%' OR key = 'pcm_x_terminal_last_note')}, timeout => 10); } // 'probe-failed'; $slots =~ s/\n/ | /g; diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 4ed77bfc2c..486e1d082f 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -953,6 +953,19 @@ cluster_pcm_x_master_ticket_debug_next(Size *cursor_io, Size *index_out, char *b return false; } +bool +cluster_pcm_x_local_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, Size buflen) +{ + return false; +} + +bool +cluster_pcm_x_terminal_note_read(uint32 *op_out, uint32 *result_out, uint64 *ticket_out, + uint32 *count_out) +{ + return false; +} + /* PGRAC spec-2.30 D9 R10 stub audit — 9 transition counter accessors. */ uint64 cluster_pcm_get_trans_n_to_s_count(void) diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 4cb034d816..7155b4a435 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -2215,7 +2215,7 @@ UT_TEST(test_runtime_layout_abi_and_offsets_are_exact) UT_ASSERT_EQ(offsetof(PcmXShmemHeader, peer_frontiers), 33664); UT_ASSERT_EQ(offsetof(PcmXShmemHeader, stats), 35200); UT_ASSERT_EQ(offsetof(PcmXShmemHeader, outbound_targets), 35376); - UT_ASSERT_EQ(sizeof(PcmXShmemHeader), 36480); + UT_ASSERT_EQ(sizeof(PcmXShmemHeader), 36504); } UT_TEST(test_lwlock_held_limit_is_shared_200) From 623e24763645dddc9617ed7106058ca0a807b6d0 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 05:49:54 +0800 Subject: [PATCH 14/56] fix(cluster): retire cross-lane holder terminal under newer writer REVOKE_BARRIER t/400 L3 four-node hot-block dead-pump: a fully-DRAINED external holder/blocker terminal (older granted cohort) could not retire its own holder lane because the (REVOKE_BARRIER && !same_ref_dual) guard in pcm_x_local_retire_candidate_at and pcm_x_local_holder_detach_terminal_exact treated a newer QUEUED writer REVOKE_BARRIER as blocking. The RETIRE watermark never advanced (retire_acked stuck at master own bit) and the FIFO wedged. Exempt a cross-lane older holder terminal via pcm_x_local_cross_lane_holder_terminal_retirable (writer_flags==0 so not same-ref dual, HOLDER_TERMINAL_MASK fully drained, grant_generation!=0); retiring it clears HOLDER_TERMINAL_MASK so the empty-frozen-round path drops REVOKE_BARRIER and promotes the writer, while detach_terminal_common preserves every writer ref/round/barrier/FIFO field. TDD RED->GREEN: test_local_cross_lane_holder_terminal_retires_under_revoke_barrier. Spec: spec-2.36a --- src/backend/cluster/cluster_pcm_x_convert.c | 33 +++++- .../cluster_unit/test_cluster_pcm_x_convert.c | 106 +++++++++++++++++- 2 files changed, 136 insertions(+), 3 deletions(-) diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index b01c9c313b..7cf81be644 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -19080,6 +19080,33 @@ pcm_x_local_ready_leader_wake_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag } +/* + * A fully-drained cross-lane holder/blocker terminal (no writer-terminal + * companion, so it is not a same-ref dual) belongs to an already-revoked older + * cohort: its S lane has finished DRAIN and it carries a completed grant. The + * newer writer's REVOKE_BARRIER guards that writer's own revoke round, not this + * older holder lane's terminal cleanup, so the older lane may retire itself + * while every writer ref / round / barrier / FIFO field is preserved. Retiring + * it also clears HOLDER_TERMINAL_MASK, which lets the empty-frozen-round path + * finally drop the REVOKE_BARRIER. Cancel-duals (grant_generation == 0) are + * routed through the same-ref-dual path and are excluded here. + * + * Precondition: the caller MUST have validated + * pcm_x_local_holder_lane_retire_state() == OK before consulting this helper. + * The flags-only test here does not itself re-prove the DRAIN evidence + * (holder_terminal_drain_generation, a clear reliable leg, a DRAIN_POLL last + * response); both call sites run that check immediately above the guard. + */ +static inline bool +pcm_x_local_cross_lane_holder_terminal_retirable(uint32 flags, const PcmXTicketRef *external_ref) +{ + return (flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) == 0 + && (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) + == PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK + && external_ref != NULL && external_ref->grant_generation != 0; +} + + static PcmXQueueResult pcm_x_local_retire_candidate_at(Size slot_index, const PcmXRetirePayload *request, int32 authenticated_master_node, @@ -19189,7 +19216,8 @@ pcm_x_local_retire_candidate_at(Size slot_index, const PcmXRetirePayload *reques if (external_ref->handle.ticket_id == request->retire_through_ticket_id) *contains_watermark_out = true; if (external_ref->handle.ticket_id <= request->retire_through_ticket_id) { - if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual) { + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual + && !pcm_x_local_cross_lane_holder_terminal_retirable(flags, external_ref)) { result = PCM_X_QUEUE_NOT_READY; goto candidate_done; } @@ -19298,7 +19326,8 @@ pcm_x_local_holder_detach_terminal_exact(const PcmXTicketRef *ref, int32 authent goto holder_detach_release_gate; goto holder_detach_domain_done; } - if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual) { + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual + && !pcm_x_local_cross_lane_holder_terminal_retirable(flags, external_ref)) { result = PCM_X_QUEUE_NOT_READY; goto holder_detach_release_gate; } diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 7155b4a435..13d481ce50 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -9689,6 +9689,109 @@ UT_TEST(test_local_non_source_blocker_participant_drains_and_retires_exactly) assert_local_queue_baseline(header); } +/* + * RED for the t/400 cross-lane retirement deadlock (2026-07-18): a fully + * DRAINED external holder/blocker terminal (an older granted cohort) must be + * able to retire its own holder lane even while a newer QUEUED writer still + * holds the tag's REVOKE_BARRIER. Before the fix the (REVOKE_BARRIER && + * !same_ref_dual) guard at cluster_pcm_x_convert.c:19192 / :19301 returns + * NOT_READY, the RETIRE watermark never advances, and the FIFO wedges. + */ +UT_TEST(test_local_cross_lane_holder_terminal_retires_under_revoke_barrier) +{ + PcmXShmemHeader *header; + PcmXLocalHolderKey holder_keys[2]; + PcmXLocalHolderKey new_holder_key; + PcmXLocalHolderHandle holders[2]; + PcmXLocalHolderHandle new_holder; + PcmXLocalHolderHandle holder_copies[2]; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalCutoff cutoff; + PcmXLocalTagSlot *tag_slot; + PcmXTicketRef probe_ref; + PcmXTicketRef writer_ref_before; + PcmXDrainPollPayload poll; + PcmXRetirePayload retire; + PcmXSlotRef found; + const uint64 master_session = UINT64_C(1801); + uint32 flags_before; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + for (i = 0; i < 2; i++) { + holder_keys[i] + = make_local_holder_key(7115, 0, (uint32)(21 + i), UINT64_C(71501) + i, 4 + i); + UT_ASSERT_EQ(register_active_local_holder(&holder_keys[i], &holders[i]), PCM_X_QUEUE_OK); + } + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(7115, 3, 23, UINT64_C(71503)); + probe_ref.handle.ticket_id = UINT64_C(95001); + probe_ref.handle.queue_generation = UINT64_C(11); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact( + &probe_ref, 1, master_session, holder_copies, 2, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, holder_copies, 2, + NULL, 0, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact(&probe_ref.identity.tag, + probe_ref.identity.cluster_epoch, + 1, master_session, &cutoff), + PCM_X_QUEUE_OK); + for (i = 0; i < 2; i++) + UT_ASSERT_EQ(release_active_local_holder(&holders[i]), PCM_X_QUEUE_OK); + + tag_slot = &local_tag_slots(header)[holder_snapshot.tag_slot.slot_index]; + memset(&poll, 0, sizeof(poll)); + poll.ref = probe_ref; + poll.ref.grant_generation = UINT64_C(101); + poll.drain_generation = UINT64_C(42); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, + PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + + /* Keep the tag resident for the newer writer cohort. */ + new_holder_key = make_local_holder_key(7115, 0, 24, UINT64_C(71504), 6); + UT_ASSERT_EQ(register_active_local_holder(&new_holder_key, &new_holder), PCM_X_QUEUE_OK); + + /* Model the newer writer's in-flight revoke round: REVOKE_BARRIER set with + * no writer terminal bits (same_ref_dual stays false) -- exactly the t/400 + * dump flags=0x31 with the external blocker ticket == RETIRE watermark. */ + pg_atomic_write_u32(&tag_slot->slot.state_flags, + pg_atomic_read_u32(&tag_slot->slot.state_flags) + | (PCM_X_LOCAL_TAG_F_REVOKE_BARRIER << PCM_X_SLOT_FLAGS_SHIFT)); + flags_before = test_slot_flags(&tag_slot->slot); + UT_ASSERT((flags_before & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(flags_before & PCM_X_LOCAL_TAG_F_TERMINAL_MASK, 0); + writer_ref_before = tag_slot->ref; + + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = probe_ref.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = poll.ref.handle.ticket_id; + retire.sender_node = 0; + + /* The older cross-lane holder terminal must retire its own lane. */ + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + + /* Holder terminal lane cleared; the newer writer's REVOKE_BARRIER and ref + * are preserved and the tag stays resident. */ + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, 0); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT(ticket_refs_equal(&tag_slot->ref, &writer_ref_before)); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &probe_ref.identity.tag, &found), + PCM_X_DIRECTORY_OK); + + UT_ASSERT_EQ(release_active_local_holder(&new_holder), PCM_X_QUEUE_OK); +} + UT_TEST(test_local_cancelled_non_source_participant_gen0_drains_and_retires_exactly) { PcmXShmemHeader *header; @@ -14192,7 +14295,7 @@ UT_TEST(test_local_retire_episode_lock_errors_fail_closed) int main(void) { - UT_PLAN(246); + UT_PLAN(247); UT_RUN(test_image_id_domain_is_canonical_and_bounded); UT_RUN(test_wire_abi_sizes_are_exact); UT_RUN(test_wire_abi_offsets_are_exact); @@ -14362,6 +14465,7 @@ main(void) UT_RUN(test_local_transfer_prepare_commit_and_final_ack_are_exact); UT_RUN(test_local_tag_only_holder_transfer_persists_until_exact_drain); UT_RUN(test_local_non_source_blocker_participant_drains_and_retires_exactly); + UT_RUN(test_local_cross_lane_holder_terminal_retires_under_revoke_barrier); UT_RUN(test_local_cancelled_non_source_participant_gen0_drains_and_retires_exactly); UT_RUN(test_local_cancelled_participant_gen0_drain_requires_frozen_round); UT_RUN(test_local_holder_drain_validates_frozen_round_before_terminal_publish); From 3a3b97b5185f7d039fe64ac3f91af0f5d5e22ccd Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 20:36:57 -0700 Subject: [PATCH 15/56] test(cluster): rebuild cross-lane retire test on production paths; tighten exemption to distinct tickets The t/400 cross-lane regression test previously modelled the newer writer by registering an S holder and OR-ing REVOKE_BARRIER by hand, leaving tag_slot->ref zero and the writer-preservation assertions vacuous. Rebuild it through the production chain (join -> claim -> revoke-cutoff -> ENQUEUE -> ADMIT_ACK/CONFIRM, then the older cohort's grant-time PROBE / blocker ACK / DRAIN), pin the closed round of one, the ticketed writer ref and the FIFO/round/reliable bytes across the retire, and drive the unwedged writer through its full production grant chain to the queue baseline. Tighten pcm_x_local_cross_lane_holder_terminal_retirable() to take the tag slot and refuse a writer-lane ref that aliases the external ref or collides with its master-unique ticket id: such states cannot be an older cohort and keep the pre-exemption NOT_READY verdict. Deliberately no ticket-order requirement beyond that -- a cancelled predecessor leader's duplicate-ACK tombstone legally lingers in tag_slot->ref with an older ticket until the successor's ENQUEUE arm retires it, and refusing there could wedge the empty-frozen path permanently if the successor exits before arming. Verified: rebuilt test RED against pre-fix production (retire NOT_READY, HOLDER_TERMINAL_MASK stuck, drain generation retained); new defense-probe test RED against the flags-only helper (alias retired as OK); both GREEN after the change; cluster_unit 182 binaries all green; check-format clean. --- src/backend/cluster/cluster_pcm_x_convert.c | 38 +- .../cluster_unit/test_cluster_pcm_x_convert.c | 370 +++++++++++++++--- 2 files changed, 353 insertions(+), 55 deletions(-) diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index 7cf81be644..fc5c9b223e 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -19091,19 +19091,40 @@ pcm_x_local_ready_leader_wake_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag * finally drop the REVOKE_BARRIER. Cancel-duals (grant_generation == 0) are * routed through the same-ref-dual path and are excluded here. * + * The exemption is strictly cross-lane: a writer-lane ref that aliases the + * external ref (or merely collides with its master-unique ticket id) cannot + * be an older cohort and keeps the pre-exemption NOT_READY verdict. There is + * deliberately no ticket-order requirement beyond that: a cancelled + * predecessor leader's duplicate-ACK tombstone legally lingers in + * tag_slot->ref with an OLDER ticket until the successor's ENQUEUE arm + * retires it, and refusing retirement there could wedge the empty-frozen + * path permanently if the successor exits before arming. + * * Precondition: the caller MUST have validated * pcm_x_local_holder_lane_retire_state() == OK before consulting this helper. - * The flags-only test here does not itself re-prove the DRAIN evidence + * The test here does not itself re-prove the DRAIN evidence * (holder_terminal_drain_generation, a clear reliable leg, a DRAIN_POLL last * response); both call sites run that check immediately above the guard. */ static inline bool -pcm_x_local_cross_lane_holder_terminal_retirable(uint32 flags, const PcmXTicketRef *external_ref) +pcm_x_local_cross_lane_holder_terminal_retirable(const PcmXLocalTagSlot *tag_slot, uint32 flags, + const PcmXTicketRef *external_ref) { - return (flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) == 0 - && (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) - == PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK - && external_ref != NULL && external_ref->grant_generation != 0; + if (tag_slot == NULL || external_ref == NULL) + return false; + if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) != 0 + || (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) + != PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK + || external_ref->grant_generation == 0) + return false; + /* Per-session ticket ids are master-unique (monotonic allocator, exhaustion + * fails closed), so id equality alone is complete for the alias class. Do + * not "simplify" this to full-ref equality: that would stop refusing an id + * collision under a distinct identity and REDUCE fail-closed coverage. */ + if (!pcm_x_ticket_ref_is_zero(&tag_slot->ref) + && external_ref->handle.ticket_id == tag_slot->ref.handle.ticket_id) + return false; + return true; } @@ -19217,7 +19238,8 @@ pcm_x_local_retire_candidate_at(Size slot_index, const PcmXRetirePayload *reques *contains_watermark_out = true; if (external_ref->handle.ticket_id <= request->retire_through_ticket_id) { if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual - && !pcm_x_local_cross_lane_holder_terminal_retirable(flags, external_ref)) { + && !pcm_x_local_cross_lane_holder_terminal_retirable(tag_slot, flags, + external_ref)) { result = PCM_X_QUEUE_NOT_READY; goto candidate_done; } @@ -19327,7 +19349,7 @@ pcm_x_local_holder_detach_terminal_exact(const PcmXTicketRef *ref, int32 authent goto holder_detach_domain_done; } if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual - && !pcm_x_local_cross_lane_holder_terminal_retirable(flags, external_ref)) { + && !pcm_x_local_cross_lane_holder_terminal_retirable(tag_slot, flags, external_ref)) { result = PCM_X_QUEUE_NOT_READY; goto holder_detach_release_gate; } diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 13d481ce50..6b2fd23e99 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -9689,50 +9689,107 @@ UT_TEST(test_local_non_source_blocker_participant_drains_and_retires_exactly) assert_local_queue_baseline(header); } +typedef struct TestLocalCrossLaneWedge { + PcmXLocalHandle writer; + PcmXLocalWriterClaim writer_claim; + PcmXLocalCutoff writer_cutoff; + PcmXAdmitAckPayload admit_ack; + PcmXTicketRef external_ref; + PcmXLocalTagSlot *tag_slot; + uint64 master_session; +} TestLocalCrossLaneWedge; + /* - * RED for the t/400 cross-lane retirement deadlock (2026-07-18): a fully - * DRAINED external holder/blocker terminal (an older granted cohort) must be - * able to retire its own holder lane even while a newer QUEUED writer still - * holds the tag's REVOKE_BARRIER. Before the fix the (REVOKE_BARRIER && - * !same_ref_dual) guard at cluster_pcm_x_convert.c:19192 / :19301 returns - * NOT_READY, the RETIRE watermark never advances, and the FIFO wedges. + * Build the exact t/400 cross-lane wedge shape through production paths only. + * + * Older external cohort (external_ticket): this node participates as two S + * holders; PROBE freezes them, the blocker ACK tombstones the set, and DRAIN + * later upgrades the locator in place to the completed grant ref, leaving + * HOLDER_TERMINAL_READY|DRAINED on the tag. + * + * Newer local writer (writer_ticket > external_ticket): joins the same tag + * through join -> claim -> revoke-cutoff (the production REVOKE_BARRIER set + * point, closing a round of one) -> ENQUEUE arm -> ADMIT_ACK (installs the + * ticketed writer ref into tag_slot->ref) -> ADMIT_CONFIRM arm/ack. + * + * Final flags shape is the t/400 dump's 0x31: REVOKE_BARRIER + + * HOLDER_TERMINAL_READY|DRAINED with no writer-terminal bits. */ -UT_TEST(test_local_cross_lane_holder_terminal_retires_under_revoke_barrier) +static void +prepare_local_cross_lane_wedge(BlockNumber block, uint64 master_session, uint64 external_ticket, + uint64 writer_ticket, TestLocalCrossLaneWedge *fixture) { PcmXShmemHeader *header; PcmXLocalHolderKey holder_keys[2]; - PcmXLocalHolderKey new_holder_key; PcmXLocalHolderHandle holders[2]; - PcmXLocalHolderHandle new_holder; PcmXLocalHolderHandle holder_copies[2]; PcmXLocalHolderSnapshot holder_snapshot; PcmXLocalBlockerSnapshot blocker_snapshot; - PcmXLocalCutoff cutoff; - PcmXLocalTagSlot *tag_slot; + PcmXLocalCutoff participant_cutoff; + PcmXWaitIdentity writer_identity; + PcmXEnqueuePayload enqueue; + PcmXPhasePayload admit_confirm; + PcmXLocalReliableToken token; PcmXTicketRef probe_ref; - PcmXTicketRef writer_ref_before; PcmXDrainPollPayload poll; - PcmXRetirePayload retire; - PcmXSlotRef found; - const uint64 master_session = UINT64_C(1801); - uint32 flags_before; + uint32 flags; int i; + UT_ASSERT_NOT_NULL(fixture); + memset(fixture, 0, sizeof(*fixture)); + fixture->master_session = master_session; init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; bind_local_master(1, UINT64_C(9), master_session); for (i = 0; i < 2; i++) { - holder_keys[i] - = make_local_holder_key(7115, 0, (uint32)(21 + i), UINT64_C(71501) + i, 4 + i); + holder_keys[i] = make_local_holder_key(block, 0, (uint32)(21 + i), + master_session + UINT64_C(1) + (uint64)i, 4 + i); UT_ASSERT_EQ(register_active_local_holder(&holder_keys[i], &holders[i]), PCM_X_QUEUE_OK); } + /* The newer local writer queues first through the production chain (join + * -> claim -> revoke-cutoff -> ENQUEUE -> ADMIT). The older external + * cohort was enqueued at the master before it (external_ticket < + * writer_ticket), so its grant-time PROBE reaches this node only after + * the local writer's round has closed -- the t/400 interleaving. */ + writer_identity = make_wait_identity(block, 0, 3, master_session + UINT64_C(4)); + UT_ASSERT_EQ( + cluster_pcm_x_local_join_begin(&writer_identity, 1, master_session, &fixture->writer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(fixture->writer.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&fixture->writer, &fixture->writer_claim), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_begin_revoke_cutoff_exact(&fixture->writer, &fixture->writer_cutoff), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&fixture->writer, &enqueue, &token), + PCM_X_QUEUE_OK); + memset(&fixture->admit_ack, 0, sizeof(fixture->admit_ack)); + fixture->admit_ack.ref.identity = writer_identity; + fixture->admit_ack.ref.handle.ticket_id = writer_ticket; + fixture->admit_ack.ref.handle.queue_generation = UINT64_C(1); + fixture->admit_ack.prehandle = enqueue.prehandle; + fixture->admit_ack.result = PCM_X_QUEUE_OK; + fixture->admit_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&fixture->writer, &fixture->admit_ack, 1, + master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_admit_confirm_arm_exact(&fixture->writer, &admit_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&fixture->writer, &admit_confirm, 1, + master_session), + PCM_X_QUEUE_OK); + + /* The older cohort's grant-time PROBE now freezes this node's S holders; + * the blocker ACK tombstones the set. */ memset(&probe_ref, 0, sizeof(probe_ref)); - probe_ref.identity = make_wait_identity(7115, 3, 23, UINT64_C(71503)); - probe_ref.handle.ticket_id = UINT64_C(95001); + probe_ref.identity = make_wait_identity(block, 3, 23, master_session + UINT64_C(3)); + probe_ref.handle.ticket_id = external_ticket; probe_ref.handle.queue_generation = UINT64_C(11); UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact( &probe_ref, 1, master_session, holder_copies, 2, &holder_snapshot), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holder_snapshot.holder_count, 2); UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, &holder_snapshot, holder_copies, 2, NULL, 0, &blocker_snapshot), @@ -9740,56 +9797,274 @@ UT_TEST(test_local_cross_lane_holder_terminal_retires_under_revoke_barrier) UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact(&probe_ref.identity.tag, - probe_ref.identity.cluster_epoch, - 1, master_session, &cutoff), + UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact( + &probe_ref.identity.tag, probe_ref.identity.cluster_epoch, 1, master_session, + &participant_cutoff), PCM_X_QUEUE_OK); for (i = 0; i < 2; i++) UT_ASSERT_EQ(release_active_local_holder(&holders[i]), PCM_X_QUEUE_OK); - tag_slot = &local_tag_slots(header)[holder_snapshot.tag_slot.slot_index]; + /* The older cohort's DRAIN closes after the newer writer queued. */ memset(&poll, 0, sizeof(poll)); poll.ref = probe_ref; poll.ref.grant_generation = UINT64_C(101); poll.drain_generation = UINT64_C(42); UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, + fixture->external_ref = poll.ref; + + /* Pin the wedge shape: same tag slot for both cohorts, dump flags 0x31, + * a ticketed queued writer and a closed round of exactly one member. */ + UT_ASSERT_EQ(fixture->writer.tag_slot.slot_index, holder_snapshot.tag_slot.slot_index); + fixture->tag_slot = &local_tag_slots(header)[holder_snapshot.tag_slot.slot_index]; + flags = test_slot_flags(&fixture->tag_slot->slot); + UT_ASSERT((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + UT_ASSERT_EQ(flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK, 0); + UT_ASSERT_EQ(fixture->tag_slot->ref.handle.ticket_id, writer_ticket); + UT_ASSERT_EQ(fixture->tag_slot->ref.handle.queue_generation, UINT64_C(1)); + UT_ASSERT_EQ(fixture->tag_slot->ref.grant_generation, 0); + UT_ASSERT(memcmp(&fixture->tag_slot->ref.identity, &writer_identity, sizeof(writer_identity)) + == 0); + UT_ASSERT_EQ(fixture->tag_slot->membership_count, 1); + UT_ASSERT_EQ(fixture->tag_slot->closed_round_member_count, 1); + UT_ASSERT_EQ(fixture->tag_slot->head_index, fixture->writer.membership_slot.slot_index); + UT_ASSERT_EQ(fixture->tag_slot->tail_index, fixture->writer.membership_slot.slot_index); + UT_ASSERT_EQ(fixture->tag_slot->leader_index, fixture->writer.membership_slot.slot_index); + UT_ASSERT_EQ(fixture->tag_slot->active_writer_index, + fixture->writer.membership_slot.slot_index); + UT_ASSERT(ticket_refs_equal(&fixture->tag_slot->blocker_snapshot_ref, &fixture->external_ref)); + UT_ASSERT_EQ(fixture->tag_slot->holder_terminal_drain_generation, poll.drain_generation); +} - /* Keep the tag resident for the newer writer cohort. */ - new_holder_key = make_local_holder_key(7115, 0, 24, UINT64_C(71504), 6); - UT_ASSERT_EQ(register_active_local_holder(&new_holder_key, &new_holder), PCM_X_QUEUE_OK); - - /* Model the newer writer's in-flight revoke round: REVOKE_BARRIER set with - * no writer terminal bits (same_ref_dual stays false) -- exactly the t/400 - * dump flags=0x31 with the external blocker ticket == RETIRE watermark. */ - pg_atomic_write_u32(&tag_slot->slot.state_flags, - pg_atomic_read_u32(&tag_slot->slot.state_flags) - | (PCM_X_LOCAL_TAG_F_REVOKE_BARRIER << PCM_X_SLOT_FLAGS_SHIFT)); - flags_before = test_slot_flags(&tag_slot->slot); - UT_ASSERT((flags_before & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); - UT_ASSERT_EQ(flags_before & PCM_X_LOCAL_TAG_F_TERMINAL_MASK, 0); +/* + * RED for the t/400 cross-lane retirement deadlock (2026-07-18): a fully + * DRAINED external holder/blocker terminal (an older granted cohort) must be + * able to retire its own holder lane even while a newer QUEUED writer still + * holds the tag's REVOKE_BARRIER. Before the fix the (REVOKE_BARRIER && + * !same_ref_dual) guard at cluster_pcm_x_convert.c:19192 / :19301 returns + * NOT_READY, the RETIRE watermark never advances, and the FIFO wedges. + * + * Rebuilt 2026-07-19 (review follow-up): the newer writer is a real + * production cohort (join -> claim -> revoke-cutoff -> ENQUEUE -> ADMIT), + * so the barrier, the closed round of one and the ticketed writer ref are + * all genuine; the retire must consume exactly the holder lane, keep every + * writer/round/FIFO byte, and the writer must afterwards progress through + * its full grant chain to the queue baseline (no barrier or tag leak). + */ +UT_TEST(test_local_cross_lane_holder_terminal_retires_under_revoke_barrier) +{ + TestLocalCrossLaneWedge fixture; + PcmXShmemHeader *header; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalTagSlot *tag_slot; + PcmXLocalHandle refreshed; + PcmXLocalProgress progress; + PcmXGrantPayload prepare; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + PcmXLocalReliableToken token; + PcmXTicketRef writer_ref_before; + PcmXReliableLegState writer_reliable_before; + PcmXDrainPollPayload poll; + PcmXRetirePayload retire; + PcmXSlotRef found; + const uint64 master_session = UINT64_C(1801); + uint64 cutoff_sequence_before; + uint64 next_sequence_before; + Size membership_before; + Size closed_before; + Size head_before; + Size tail_before; + Size leader_before; + Size active_writer_before; + uint32 local_round_before; + + prepare_local_cross_lane_wedge(7115, master_session, UINT64_C(95001), UINT64_C(95002), + &fixture); + header = ClusterPcmXConvertShmem; + tag_slot = fixture.tag_slot; writer_ref_before = tag_slot->ref; - + writer_reliable_before = tag_slot->reliable; + cutoff_sequence_before = tag_slot->cutoff_sequence; + next_sequence_before = tag_slot->next_sequence; + membership_before = tag_slot->membership_count; + closed_before = tag_slot->closed_round_member_count; + head_before = tag_slot->head_index; + tail_before = tag_slot->tail_index; + leader_before = tag_slot->leader_index; + active_writer_before = tag_slot->active_writer_index; + local_round_before = tag_slot->local_round; + + /* RETIRE watermark covers only the older external ticket. */ memset(&retire, 0, sizeof(retire)); - retire.cluster_epoch = probe_ref.identity.cluster_epoch; + retire.cluster_epoch = fixture.external_ref.identity.cluster_epoch; retire.master_session_incarnation = master_session; - retire.retire_through_ticket_id = poll.ref.handle.ticket_id; + retire.retire_through_ticket_id = fixture.external_ref.handle.ticket_id; retire.sender_node = 0; - - /* The older cross-lane holder terminal must retire its own lane. */ UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - /* Holder terminal lane cleared; the newer writer's REVOKE_BARRIER and ref - * are preserved and the tag stays resident. */ + /* Exactly the holder lane was consumed: blocker locator, its reliable leg + * and the drain generation are gone and HOLDER_TERMINAL_MASK cleared. */ UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, 0); + UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &(PcmXTicketRef){ 0 })); + UT_ASSERT_EQ(tag_slot->blocker_set_generation, 0); + UT_ASSERT_EQ(tag_slot->holder_terminal_drain_generation, 0); + + /* The newer writer's round is byte-stable: barrier, ticketed ref, closed + * round, FIFO indexes and the writer reliable leg are all untouched. */ UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_TERMINAL_MASK, 0); UT_ASSERT(ticket_refs_equal(&tag_slot->ref, &writer_ref_before)); - UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &probe_ref.identity.tag, &found), + UT_ASSERT(memcmp(&tag_slot->reliable, &writer_reliable_before, sizeof(writer_reliable_before)) + == 0); + UT_ASSERT_EQ(tag_slot->cutoff_sequence, cutoff_sequence_before); + UT_ASSERT_EQ(tag_slot->next_sequence, next_sequence_before); + UT_ASSERT_EQ(tag_slot->membership_count, membership_before); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, closed_before); + UT_ASSERT_EQ(tag_slot->head_index, head_before); + UT_ASSERT_EQ(tag_slot->tail_index, tail_before); + UT_ASSERT_EQ(tag_slot->leader_index, leader_before); + UT_ASSERT_EQ(tag_slot->active_writer_index, active_writer_before); + UT_ASSERT_EQ(tag_slot->local_round, local_round_before); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, + &fixture.external_ref.identity.tag, &found), PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&fixture.writer.identity, &refreshed), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(refreshed.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&fixture.writer, &progress), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.member_state, PCM_XL_REMOTE_WAIT); - UT_ASSERT_EQ(release_active_local_holder(&new_holder), PCM_X_QUEUE_OK); + /* The unwedged writer must now progress through its full production grant + * chain and drain to the queue baseline -- the deadlock did not merely + * move into a barrier or tag leak. Its closed revoke round has no S + * holders left (new pins are correctly fenced BARRIER_CLOSED), so the + * writer's own PROBE freezes an empty set. */ + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact( + &fixture.admit_ack.ref, 1, master_session, &holder_copy, 1, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holder_snapshot.holder_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact( + &fixture.admit_ack.ref, 1, master_session, &holder_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact( + &fixture.admit_ack.ref, blocker_snapshot.set_generation, 1, master_session), + PCM_X_QUEUE_OK); + + memset(&prepare, 0, sizeof(prepare)); + prepare.ref = fixture.admit_ack.ref; + prepare.ref.grant_generation = UINT64_C(102); + UT_ASSERT( + cluster_pcm_x_image_id_encode(1, master_session + UINT64_C(200), &prepare.image.image_id)); + prepare.image.source_own_generation = UINT64_C(9); + prepare.image.page_scn = UINT64_C(10); + prepare.image.page_lsn = UINT64_C(11); + prepare.image.source_node = 2; + prepare.image.page_checksum = UINT32_C(12); + UT_ASSERT_EQ( + cluster_pcm_x_local_prepare_grant_exact(&fixture.writer, &prepare, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_install_ready_arm_exact( + &fixture.writer, &prepare.ref, &prepare.image, &install_ready, &token), + PCM_X_QUEUE_OK); + memset(&commit, 0, sizeof(commit)); + commit.ref = prepare.ref; + commit.phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; + UT_ASSERT_EQ(cluster_pcm_x_local_commit_x_exact(&fixture.writer, &commit, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_final_ack_arm_exact(&fixture.writer, 1, &final_ack, &token), + PCM_X_QUEUE_OK); + memset(&final_commit, 0, sizeof(final_commit)); + final_commit.ref = prepare.ref; + final_commit.phase = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; + UT_ASSERT_EQ(cluster_pcm_x_local_final_commit_ack_exact(&fixture.writer, &final_commit, 1, + master_session, &final_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&fixture.writer_claim), + PCM_X_QUEUE_OK); + memset(&poll, 0, sizeof(poll)); + poll.ref = prepare.ref; + poll.drain_generation = UINT64_C(43); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = prepare.ref.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = prepare.ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&fixture.writer, &progress), + PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, + &fixture.external_ref.identity.tag, &found), + PCM_X_DIRECTORY_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + assert_local_queue_baseline(header); +} + +/* + * Defense probes for the cross-lane retirement exemption: it must stay + * strictly cross-lane. A holder lane that aliases the writer lane's ref (or + * collides with its ticket id) keeps the pre-exemption NOT_READY verdict. + * Both alias states are protocol-impossible (master tickets are unique and + * same-ref forms route through the same-ref-dual path), so they are built by + * direct slot pokes like the CORRUPT arms elsewhere in this file. A distinct + * OLDER writer-lane ref, however, is legal (a cancelled predecessor leader's + * duplicate-ACK tombstone lingers in tag_slot->ref until the successor's + * ENQUEUE arm retires it), so the exemption deliberately has no ticket-order + * requirement: refusing there could wedge the empty-frozen path permanently + * if the successor dies before arming. + */ +UT_TEST(test_local_cross_lane_retire_exemption_requires_distinct_ticket) +{ + TestLocalCrossLaneWedge fixture; + PcmXLocalTagSlot *tag_slot; + PcmXTicketRef saved_ref; + PcmXRetirePayload retire; + const uint64 master_session = UINT64_C(1809); + + prepare_local_cross_lane_wedge(7116, master_session, UINT64_C(95001), UINT64_C(95002), + &fixture); + tag_slot = fixture.tag_slot; + saved_ref = tag_slot->ref; + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.external_ref.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = fixture.external_ref.handle.ticket_id; + retire.sender_node = 0; + + /* Sub-case order matters: the two NOT_READY alias probes must run before + * the distinct-older-ticket case, which consumes the holder lane. */ + + /* Full alias: writer lane ref == external holder lane ref. */ + tag_slot->ref = fixture.external_ref; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_NOT_READY); + + /* Ticket-id collision under a distinct identity. */ + tag_slot->ref = saved_ref; + tag_slot->ref.handle.ticket_id = fixture.external_ref.handle.ticket_id; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_NOT_READY); + + /* Distinct older writer-lane ref (cancelled-predecessor tombstone shape): + * the exemption must still let the external holder lane retire. */ + tag_slot->ref = saved_ref; + tag_slot->ref.handle.ticket_id = fixture.external_ref.handle.ticket_id - 1; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, 0); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + tag_slot->ref = saved_ref; + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); } UT_TEST(test_local_cancelled_non_source_participant_gen0_drains_and_retires_exactly) @@ -14466,6 +14741,7 @@ main(void) UT_RUN(test_local_tag_only_holder_transfer_persists_until_exact_drain); UT_RUN(test_local_non_source_blocker_participant_drains_and_retires_exactly); UT_RUN(test_local_cross_lane_holder_terminal_retires_under_revoke_barrier); + UT_RUN(test_local_cross_lane_retire_exemption_requires_distinct_ticket); UT_RUN(test_local_cancelled_non_source_participant_gen0_drains_and_retires_exactly); UT_RUN(test_local_cancelled_participant_gen0_drain_requires_frozen_round); UT_RUN(test_local_holder_drain_validates_frozen_round_before_terminal_publish); From efb2ec5640db6a354e8e5efe1c37614bd2884516 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 20:51:13 -0700 Subject: [PATCH 16/56] fix(cluster): wait out transient BUSY at the PCM-X remote-reservation preflight Any non-OK remote-reservation preflight used to fail the whole PCM-X runtime closed (t/400 fuse family, requester exit line 8854). BUSY there only means a revoke/grant lifecycle flag is still live on this node's own tuple -- ordinary contention the next iteration re-snapshots -- so route it through the requester retry table as a wait, never a runtime fuse. STALE keeps the fail-closed verdict for now: it means an interleaved revoke consumed the queued identity's enqueue-time base_own_generation, which the grant/final-ack chain hard-pins at every hop (master final-ack prepare/finalize, local final-ack arm, writer-holder authority, bufmgr claim); absorbing it needs a master-visible rebase amendment. Capture the preflight own-tuple evidence (live/base generations, pcm_state, flags) and print it in the requester fail-closed LOG so the next t/400 run names the exact STALE arm. Verified: retry-table case RED (BUSY mapped fail-closed) then GREEN; driver source-scan pin RED (no preflight retry dispatch) then GREEN; cluster_unit 182 binaries all green; full macOS build clean; check-format clean. --- src/backend/cluster/cluster_gcs_block.c | 45 ++++++++++++++++++- src/include/cluster/cluster_gcs_block.h | 12 ++++- .../cluster_unit/test_cluster_gcs_block.c | 30 +++++++++++++ 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index ae1de65759..e91f021513 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -8080,6 +8080,20 @@ static bool gcs_block_pcm_x_requester_exit_hook_registered = false; * writer error names the exact escape arm instead of just the result code. */ static int gcs_block_pcm_x_requester_fail_line = 0; +/* Own-tuple evidence captured at the most recent non-OK remote-reservation + * preflight. Diagnostic only: printed by the fail-closed LOG so a STALE + * verdict names which preflight arm fired (base-generation drift vs a + * non-N pcm_state vs a live lifecycle flag) without a debugger. */ +typedef struct GcsBlockPcmXPreflightEvidence { + bool valid; + uint8 pcm_state; + uint32 own_flags; + uint64 live_generation; + uint64 base_own_generation; +} GcsBlockPcmXPreflightEvidence; + +static GcsBlockPcmXPreflightEvidence gcs_block_pcm_x_requester_preflight_evidence; + #define GCS_BLOCK_PCM_X_REQUESTER_DONE() \ do { \ gcs_block_pcm_x_requester_fail_line = __LINE__; \ @@ -8441,6 +8455,9 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim || MyProc->pgprocno < 0 || MyBackendId <= 0 || cluster_node_id < 0 || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) return PCM_X_QUEUE_INVALID; + /* Preflight evidence must belong to this acquire, never a previous one. */ + memset(&gcs_block_pcm_x_requester_preflight_evidence, 0, + sizeof(gcs_block_pcm_x_requester_preflight_evidence)); memset(&handle, 0, sizeof(handle)); handle.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; handle.membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; @@ -8850,8 +8867,21 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim result = cluster_gcs_pcm_x_remote_reservation_preflight(&reservation_base, &progress.identity); cluster_pcm_x_stats_note_queue_result(result); - if (result != PCM_X_QUEUE_OK) - GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + if (result != PCM_X_QUEUE_OK) { + GcsBlockPcmXPreflightEvidence *evidence + = &gcs_block_pcm_x_requester_preflight_evidence; + + evidence->valid = true; + evidence->pcm_state = reservation_base.pcm_state; + evidence->own_flags = reservation_base.flags; + evidence->live_generation = reservation_base.generation; + evidence->base_own_generation = progress.identity.base_own_generation; + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, result); + if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + goto requester_wait; + } own_result = cluster_bufmgr_pcm_own_begin_x_reservation( buf, &reservation_base, &reservation_token); result = gcs_block_pcm_x_fetch_own_result(own_result); @@ -9007,6 +9037,17 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim (unsigned int)progress.pending_opcode, (unsigned int)progress.last_response_opcode, (unsigned long long)request_id, (unsigned long long)wait_seq))); + if (gcs_block_pcm_x_requester_preflight_evidence.valid) + ereport( + LOG, + (errmsg("PCM-X requester reservation-preflight evidence"), + errdetail( + "live_gen=%llu base_gen=%llu pcm_state=%u own_flags=%u", + (unsigned long long)gcs_block_pcm_x_requester_preflight_evidence.live_generation, + (unsigned long long) + gcs_block_pcm_x_requester_preflight_evidence.base_own_generation, + (unsigned int)gcs_block_pcm_x_requester_preflight_evidence.pcm_state, + (unsigned int)gcs_block_pcm_x_requester_preflight_evidence.own_flags))); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) result = PCM_X_QUEUE_CORRUPT; diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index 53f3a8ac24..e3d2e8e7b4 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -176,7 +176,8 @@ typedef enum GcsBlockPcmXRequesterSite { GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, GCS_BLOCK_PCM_X_RETRY_SITE_IMAGE_FETCH, GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_ARM, - GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_REPLAY_ARM + GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_REPLAY_ARM, + GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT } GcsBlockPcmXRequesterSite; typedef enum GcsBlockPcmXRetryAction { @@ -407,6 +408,15 @@ cluster_gcs_pcm_x_requester_retry_action(GcsBlockPcmXRequesterSite site, PcmXQue if (result == PCM_X_QUEUE_BAD_STATE) return GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS; break; + case GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT: + /* BUSY is a transient own-slot lifecycle flag (a revoke/grant on this + * node mid-flight); wait and re-snapshot. STALE means an interleaved + * revoke consumed the queued identity's base_own_generation, which the + * grant/final-ack chain cannot absorb without a master-visible rebase; + * keep the fail-closed verdict until that amendment lands. */ + if (result == PCM_X_QUEUE_BUSY) + return GCS_BLOCK_PCM_X_RETRY_WAIT; + break; case GCS_BLOCK_PCM_X_RETRY_SITE_WFG_CLEAR: case GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_ARM: break; diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 328c70cbb7..90cc0a35a7 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -2821,6 +2821,22 @@ UT_TEST(test_pcm_x_requester_driver_owns_fifo_and_transfer_lifecycles) UT_ASSERT_NOT_NULL( strstr(driver, "cluster_gcs_pcm_x_role_refresh_exact(&handle, &fresh_handle)")); UT_ASSERT_NOT_NULL(strstr(driver, "initial_own.pcm_state != (uint8)PCM_STATE_X")); + { + const char *preflight; + const char *preflight_retry; + + preflight = strstr(driver, "cluster_gcs_pcm_x_remote_reservation_preflight("); + preflight_retry + = preflight != NULL + ? strstr(preflight, "GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT") + : NULL; + UT_ASSERT_NOT_NULL(preflight); + UT_ASSERT_NOT_NULL(preflight_retry); + /* The transient-BUSY wait dispatch must sit between the preflight and + * the image fetch; a non-wait verdict is the only fail-closed exit. */ + if (preflight != NULL && preflight_retry != NULL && remote_fetch != NULL) + UT_ASSERT(preflight < preflight_retry && preflight_retry < remote_fetch); + } if (follower_snapshot != NULL && graph_replace != NULL && graph_clear != NULL) UT_ASSERT(graph_clear < follower_snapshot && follower_snapshot < graph_replace && graph_replace < driver_end); @@ -2895,6 +2911,20 @@ UT_TEST(test_pcm_x_requester_retry_policy_is_operation_exact) GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS }, { GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_REPLAY_ARM, PCM_X_QUEUE_STALE, GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + /* Remote-reservation preflight: BUSY is a transient own-slot lifecycle + * (a revoke/grant flag mid-flight on this node) and must wait, never + * close the runtime. STALE stays fail-closed for now: it means the + * enqueue-time base_own_generation was consumed by a revoke while the + * request was queued, which the grant/final-ack chain cannot absorb + * without a master-visible rebase (pending protocol amendment). */ + { GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, PCM_X_QUEUE_BUSY, + GCS_BLOCK_PCM_X_RETRY_WAIT }, + { GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, PCM_X_QUEUE_STALE, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, PCM_X_QUEUE_CORRUPT, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, PCM_X_QUEUE_COUNTER_EXHAUSTED, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, }; size_t i; From ecfcd41611a8587d9a2cad718d3e1b3d868440fe Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 00:34:00 -0700 Subject: [PATCH 17/56] feat(cluster): absorb interleaved-revoke base drift via one-shot INSTALL_READY grant rebase A queued PCM-X writer that held S/X at ENQUEUE loses its enqueue-time base_own_generation when an interleaved revoke commits while it waits; the grant then arrived un-installable (reservation preflight STALE) and fused the whole runtime -- the t/400 clean-N drift fuse (evidence: live_gen=base+1, pcm_state=N, flags=0). Rebase design (A'): the immutable PcmXWaitIdentity and every directory key never change. The requester publishes the live clean-N generation exactly once as an effective grant base -- locally pre-reservation (cluster_pcm_x_local_grant_rebase_publish_exact) and to the master on the INSTALL_READY V2 frame (trailing rebased_own_generation, 104->112). Master and local planes persist it independently (ticket/tag slot grant_base_own_generation) with identical one-shot semantics: same-value replay is DUPLICATE, a different second value is evidence divergence and fails closed. Every exact committed==base+1 proof now runs against the effective grant base under its owning lock (master final-ack prepare/finalize, local final-ack arm, writer-holder authority, bufmgr grant-snapshot binding via the claim copy, image-fetch reservation check); the wire ingress checks relax to a monotonic floor only. Mixed-version safety: PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1 (0x400) is advertised unconditionally, but publication activates only when the whole bound formation advertised it at activation (runtime-lifetime flag, sampled by the formation tick). The V2 frame exists on the wire only to carry a nonzero rebase; every other INSTALL_READY stays the V1 104-byte exact frame, and receivers accept both exact lengths. Without coverage a drifted requester keeps the fail-closed STALE verdict, now with rebase_active in the preflight evidence LOG. Verified: master one-shot/conflict/no-drift and local publish/effective tests RED under neutralized persistence or effective helper, then GREEN; HELLO reference bytes and capability-set tests RED on the new bit, then pinned; ingress dual-length and monotonic-floor arms covered; layout pins updated (ticket slot 384->392, local tag slot 752->760, progress 240->248, writer claim 144->152, shmem header 36504->36512, INSTALL_READY 104->112 with V1-prefix offset). cluster_unit 182 binaries all green; full macOS build clean; check-format clean. --- src/backend/cluster/cluster_gcs_block.c | 126 ++++- src/backend/cluster/cluster_gcs_block_shard.c | 8 +- src/backend/cluster/cluster_ic.c | 3 + src/backend/cluster/cluster_pcm_x_convert.c | 181 ++++++- src/backend/cluster/cluster_sf_dep.c | 16 + src/include/cluster/cluster_gcs_block.h | 24 +- src/include/cluster/cluster_ic.h | 10 + src/include/cluster/cluster_pcm_x_bufmgr.h | 9 +- src/include/cluster/cluster_pcm_x_convert.h | 71 ++- src/include/cluster/cluster_sf_dep.h | 1 + .../cluster_unit/test_cluster_gcs_block.c | 38 +- src/test/cluster_unit/test_cluster_ic.c | 15 +- src/test/cluster_unit/test_cluster_pcm_own.c | 13 + .../cluster_unit/test_cluster_pcm_x_convert.c | 482 +++++++++++++++++- 14 files changed, 929 insertions(+), 68 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index e91f021513..d735e2b142 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -3203,7 +3203,10 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle return queue_result; if (!BufferTagsEqual(&buf->tag, &reservation_base->tag) || !BufferTagsEqual(&leader->identity.tag, &reservation_base->tag) - || reservation_base->generation != leader->identity.base_own_generation) + || reservation_base->generation + != (progress_before.grant_base_own_generation != 0 + ? progress_before.grant_base_own_generation + : leader->identity.base_own_generation)) return PCM_X_QUEUE_STALE; if (!cluster_pcm_x_image_fetch_build_request(&progress_before, cluster_node_id, (int32)MyBackendId, &request)) @@ -8086,6 +8089,7 @@ static int gcs_block_pcm_x_requester_fail_line = 0; * non-N pcm_state vs a live lifecycle flag) without a debugger. */ typedef struct GcsBlockPcmXPreflightEvidence { bool valid; + bool rebase_wire_active; uint8 pcm_state; uint32 own_flags; uint64 live_generation; @@ -8094,6 +8098,40 @@ typedef struct GcsBlockPcmXPreflightEvidence { static GcsBlockPcmXPreflightEvidence gcs_block_pcm_x_requester_preflight_evidence; +/* A' rebase: publication is legal only under an ACTIVE runtime whose bound + * formation had full PCM_X_REBASE_V1 coverage at activation. */ +static bool +gcs_block_pcm_x_rebase_wire_active(const PcmXRuntimeSnapshot *runtime) +{ + return runtime != NULL && runtime->state == PCM_X_RUNTIME_ACTIVE && runtime->rebase_wire_active; +} + +/* The V2 (112-byte) frame exists on the wire only to carry a nonzero rebase; + * every other INSTALL_READY stays the V1 104-byte exact frame, so a formation + * without full V2 coverage never emits a length an old master would refuse + * (a nonzero rebase is impossible there: publication is gated on coverage). */ +static uint16 +gcs_block_pcm_x_install_ready_wire_len(const PcmXInstallReadyPayload *install_ready) +{ + return install_ready->rebased_own_generation != 0 ? (uint16)sizeof(*install_ready) + : (uint16)PCM_X_INSTALL_READY_V1_LEN; +} + +/* Drift is rebase-eligible only as the exact clean-N shape: same tag, idle + * lifecycle flags, pcm_state N and a strictly newer live generation. Any + * other preflight STALE (tag churn, non-N state, live flags) keeps the + * fail-closed verdict, as does a formation without full V2 coverage. */ +static bool +gcs_block_pcm_x_reservation_rebase_eligible(const ClusterPcmOwnSnapshot *live, + const PcmXWaitIdentity *identity, + const PcmXRuntimeSnapshot *runtime) +{ + return gcs_block_pcm_x_rebase_wire_active(runtime) && live != NULL && identity != NULL + && BufferTagsEqual(&live->tag, &identity->tag) && live->pcm_state == (uint8)PCM_STATE_N + && live->flags == 0 && live->generation != UINT64_MAX + && live->generation > identity->base_own_generation; +} + #define GCS_BLOCK_PCM_X_REQUESTER_DONE() \ do { \ gcs_block_pcm_x_requester_fail_line = __LINE__; \ @@ -8867,11 +8905,30 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim result = cluster_gcs_pcm_x_remote_reservation_preflight(&reservation_base, &progress.identity); cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_STALE + && gcs_block_pcm_x_reservation_rebase_eligible( + &reservation_base, &progress.identity, &request_runtime)) { + /* An interleaved revoke consumed the enqueue-time + * base while this request was queued. Publish the + * live clean-N generation as the effective grant + * base (one-shot; replays are DUPLICATE) and carry + * it on the writer claim for the bufmgr cross + * check. The immutable identity never changes. */ + result = cluster_pcm_x_local_grant_rebase_publish_exact( + &handle, reservation_base.generation); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + claim_out->grant_base_own_generation = reservation_base.generation; + result = PCM_X_QUEUE_OK; + } + } if (result != PCM_X_QUEUE_OK) { GcsBlockPcmXPreflightEvidence *evidence = &gcs_block_pcm_x_requester_preflight_evidence; evidence->valid = true; + evidence->rebase_wire_active + = gcs_block_pcm_x_rebase_wire_active(&request_runtime); evidence->pcm_state = reservation_base.pcm_state; evidence->own_flags = reservation_base.flags; evidence->live_generation = reservation_base.generation; @@ -8914,9 +8971,9 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim cluster_pcm_x_stats_note_queue_result(arm_result); result = arm_result; if (arm_result == PCM_X_QUEUE_OK || arm_result == PCM_X_QUEUE_DUPLICATE) - staged = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_INSTALL_READY, - master_node, &install_ready, - sizeof(install_ready)); + staged = cluster_gcs_pcm_x_stage_frame( + PGRAC_IC_MSG_PCM_X_INSTALL_READY, master_node, &install_ready, + gcs_block_pcm_x_install_ready_wire_len(&install_ready)); else { retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, arm_result); @@ -8936,9 +8993,9 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim cluster_pcm_x_stats_note_queue_result(arm_result); result = arm_result; if (arm_result == PCM_X_QUEUE_OK || arm_result == PCM_X_QUEUE_DUPLICATE) - staged = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_INSTALL_READY, - master_node, &install_ready, - sizeof(install_ready)); + staged = cluster_gcs_pcm_x_stage_frame( + PGRAC_IC_MSG_PCM_X_INSTALL_READY, master_node, &install_ready, + gcs_block_pcm_x_install_ready_wire_len(&install_ready)); else { retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, arm_result); @@ -9042,12 +9099,13 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim LOG, (errmsg("PCM-X requester reservation-preflight evidence"), errdetail( - "live_gen=%llu base_gen=%llu pcm_state=%u own_flags=%u", + "live_gen=%llu base_gen=%llu pcm_state=%u own_flags=%u rebase_active=%d", (unsigned long long)gcs_block_pcm_x_requester_preflight_evidence.live_generation, (unsigned long long) gcs_block_pcm_x_requester_preflight_evidence.base_own_generation, (unsigned int)gcs_block_pcm_x_requester_preflight_evidence.pcm_state, - (unsigned int)gcs_block_pcm_x_requester_preflight_evidence.own_flags))); + (unsigned int)gcs_block_pcm_x_requester_preflight_evidence.own_flags, + gcs_block_pcm_x_requester_preflight_evidence.rebase_wire_active ? 1 : 0))); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) result = PCM_X_QUEUE_CORRUPT; @@ -9149,7 +9207,7 @@ gcs_block_pcm_x_source_capable(int32 node_id) */ static bool gcs_block_pcm_x_collect_formation(PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_LIMIT], - uint64 *epoch_out, uint64 *self_session_out) + uint64 *epoch_out, uint64 *self_session_out, bool *rebase_all_out) { ClusterMembershipState membership_after[CLUSTER_MAX_NODES]; ClusterMembershipState membership_before[CLUSTER_MAX_NODES]; @@ -9158,6 +9216,8 @@ gcs_block_pcm_x_collect_formation(PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_L uint64 peer_session; int i; + if (rebase_all_out != NULL) + *rebase_all_out = true; if (bindings == NULL || epoch_out == NULL || self_session_out == NULL || cluster_node_id < 0 || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) return false; @@ -9181,6 +9241,11 @@ gcs_block_pcm_x_collect_formation(PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_L return false; if (i != cluster_node_id && !cluster_sf_peer_supports_pcm_x_convert(i)) return false; + /* Rebase coverage never refuses the base protocol: a member without + * the V2 bit only pins the whole formation to V1 frames. */ + if (rebase_all_out != NULL && i != cluster_node_id + && !cluster_sf_peer_supports_pcm_x_rebase(i)) + *rebase_all_out = false; auth_result = gcs_block_pcm_x_authenticated_session_result(i, epoch_before, &peer_session, NULL); if (auth_result != PCM_X_SESSION_AUTH_OK) @@ -9228,6 +9293,7 @@ cluster_gcs_block_pcm_x_formation_tick(void) uint64 epoch_before; uint64 self_session_after; uint64 self_session_before; + bool rebase_all = false; int i; runtime = cluster_pcm_x_runtime_snapshot(); @@ -9237,10 +9303,11 @@ cluster_gcs_block_pcm_x_formation_tick(void) * runtime. Both complete samples must first agree; any transient read, * including an unrelated membership flip, is a no-op for this tick. */ - if (!gcs_block_pcm_x_collect_formation(bindings_before, &epoch_before, - &self_session_before)) + if (!gcs_block_pcm_x_collect_formation(bindings_before, &epoch_before, &self_session_before, + NULL)) return; - if (!gcs_block_pcm_x_collect_formation(bindings_after, &epoch_after, &self_session_after)) + if (!gcs_block_pcm_x_collect_formation(bindings_after, &epoch_after, &self_session_after, + NULL)) return; if (!cluster_gcs_pcm_x_formation_samples_stable(true, bindings_before, true, bindings_after) || epoch_before != epoch_after || self_session_before != self_session_after) @@ -9268,8 +9335,13 @@ cluster_gcs_block_pcm_x_formation_tick(void) /* ACTIVATING and any post-activation BLOCKED state are non-pristine. */ if (runtime.gate_generation != 0 || runtime.master_session_incarnation != 0) return; - if (!gcs_block_pcm_x_collect_formation(bindings_before, &epoch_before, &self_session_before)) + if (!gcs_block_pcm_x_collect_formation(bindings_before, &epoch_before, &self_session_before, + &rebase_all)) return; + /* Connection-bound capabilities are stable for the runtime's lifetime (a + * peer restart changes its session incarnation and permanently closes the + * steady-state core), so formation-wide V2 coverage is sampled once. */ + cluster_pcm_x_runtime_set_rebase_wire_active(rebase_all); (void)cluster_pcm_x_runtime_activate_bound(self_session_before, bindings_before); return; @@ -11477,7 +11549,7 @@ cluster_gcs_handle_pcm_x_prepare_grant_envelope(const ClusterICEnvelope *env, co static void cluster_gcs_handle_pcm_x_install_ready_envelope(const ClusterICEnvelope *env, const void *payload) { - const PcmXInstallReadyPayload *install_ready; + PcmXInstallReadyPayload frame; PcmXPhasePayload commit; PcmXQueueResult result; uint64 current_epoch; @@ -11485,21 +11557,25 @@ cluster_gcs_handle_pcm_x_install_ready_envelope(const ClusterICEnvelope *env, co int32 source_node; int32 tag_master; - if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXInstallReadyPayload) + /* Both exact lengths are legal: the V1 104-byte frame normalizes to a + * zero rebase, the V2 112-byte frame carries the published grant base. */ + if (env == NULL || payload == NULL + || (env->payload_length != sizeof(PcmXInstallReadyPayload) + && env->payload_length != PCM_X_INSTALL_READY_V1_LEN) || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) return; source_node = (int32)env->source_node_id; - install_ready = (const PcmXInstallReadyPayload *)payload; + memset(&frame, 0, sizeof(frame)); + memcpy(&frame, payload, env->payload_length); current_epoch = cluster_epoch_get_current(); - tag_master = cluster_gcs_lookup_master(install_ready->ref.identity.tag); - if (!cluster_gcs_pcm_x_install_ready_ingress_valid(install_ready, env->payload_length, - source_node, current_epoch, tag_master, - cluster_node_id) - || !gcs_block_pcm_x_transfer_ingress_authorized( - &install_ready->ref.identity.tag, source_node, current_epoch, &source_session)) + tag_master = cluster_gcs_lookup_master(frame.ref.identity.tag); + if (!cluster_gcs_pcm_x_install_ready_ingress_valid(&frame, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_transfer_ingress_authorized(&frame.ref.identity.tag, source_node, + current_epoch, &source_session)) return; - result = cluster_pcm_x_master_install_ready_exact(install_ready, source_node, source_session, - &commit); + result = cluster_pcm_x_master_install_ready_exact(&frame, frame.rebased_own_generation, + source_node, source_session, &commit); cluster_pcm_x_stats_note_queue_result(result); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_COMMIT_X, diff --git a/src/backend/cluster/cluster_gcs_block_shard.c b/src/backend/cluster/cluster_gcs_block_shard.c index 73d1715359..75c374ba22 100644 --- a/src/backend/cluster/cluster_gcs_block_shard.c +++ b/src/backend/cluster/cluster_gcs_block_shard.c @@ -149,7 +149,13 @@ cluster_gcs_block_payload_shard(uint8 msg_type, const void *payload, uint16 payl pcm_x_expected_len = sizeof(PcmXGrantPayload); break; case PGRAC_IC_MSG_PCM_X_INSTALL_READY: - pcm_x_expected_len = sizeof(PcmXInstallReadyPayload); + /* A' rebase: the V1 104-byte and V2 112-byte exact frames are both + * legal; the tag prefix is identical, so the shard key is too. */ + if (payload_len != sizeof(PcmXInstallReadyPayload) + && payload_len != PCM_X_INSTALL_READY_V1_LEN) + return -1; + memcpy(&pcm_x_tag, payload, sizeof(pcm_x_tag)); + tag = &pcm_x_tag; break; case PGRAC_IC_MSG_PCM_X_FINAL_ACK: pcm_x_expected_len = sizeof(PcmXFinalAckPayload); diff --git a/src/backend/cluster/cluster_ic.c b/src/backend/cluster/cluster_ic.c index 738681173e..068b948336 100644 --- a/src/backend/cluster/cluster_ic.c +++ b/src/backend/cluster/cluster_ic.c @@ -663,6 +663,9 @@ cluster_ic_build_hello(uint8 out_buf[PGRAC_IC_HELLO_BYTES], uint16 hello_version /* PCM-X conversion is a protocol capability. Every opcode has a distinct * registered DATA-plane type, so mixed-version senders gate on this bit. */ capabilities |= PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1; + /* A' rebase: V2 INSTALL_READY understanding, same unconditional protocol + * discipline; activation is formation-wide (see cluster_ic.h). */ + capabilities |= PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1; if (capabilities != 0) ic_le_write_uint32(out_buf + PGRAC_IC_HELLO_CAPABILITIES_OFFSET, capabilities); diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index fc5c9b223e..3003df5aa6 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -2564,6 +2564,7 @@ cluster_pcm_x_runtime_snapshot(void) PcmXRuntimeSnapshot snapshot = { 0 }; uint32 gate1; uint32 gate2; + uint32 rebase_wire_active; uint32 state1; uint64 session; @@ -2581,6 +2582,7 @@ cluster_pcm_x_runtime_snapshot(void) pg_read_barrier(); session = ClusterPcmXConvertShmem->master_session_incarnation; + rebase_wire_active = ClusterPcmXConvertShmem->rebase_wire_active; pg_read_barrier(); gate2 = pg_atomic_read_u32(&ClusterPcmXConvertShmem->runtime_gate); if (gate1 != gate2 || session == 0) @@ -2588,10 +2590,20 @@ cluster_pcm_x_runtime_snapshot(void) snapshot.state = PCM_X_RUNTIME_ACTIVE; snapshot.master_session_incarnation = session; + snapshot.rebase_wire_active = rebase_wire_active != 0; return snapshot; } +void +cluster_pcm_x_runtime_set_rebase_wire_active(bool active) +{ + if (ClusterPcmXConvertShmem == NULL) + return; + ClusterPcmXConvertShmem->rebase_wire_active = active ? 1 : 0; +} + + /* * Revalidate one fail-stop acting token at the final owner lock. * @@ -2978,6 +2990,17 @@ pcm_x_local_handle_clear(PcmXLocalHandle *handle) } +/* The exact commit proof runs against the effective grant base: the one-shot + * pre-reservation rebase when an interleaved revoke consumed the enqueue-time + * base on this node, the immutable identity base otherwise. */ +static inline uint64 +pcm_x_local_effective_grant_base(const PcmXLocalTagSlot *tag_slot) +{ + return tag_slot->grant_base_own_generation != 0 ? tag_slot->grant_base_own_generation + : tag_slot->ref.identity.base_own_generation; +} + + static void pcm_x_local_follower_wfg_snapshot_clear(PcmXLocalFollowerWfgSnapshot *snapshot) { @@ -3619,6 +3642,7 @@ pcm_x_master_admit_begin_impl(const PcmXEnqueuePayload *request, PcmXMasterAdmis ticket->blocker_stage_source_node = -1; ticket->blocker_set_source_node = -1; ticket->involved_nodes_bitmap = node_bit; + ticket->grant_base_own_generation = 0; if (new_tag) { directory_result = pcm_x_directory_insert_locked( @@ -7494,10 +7518,21 @@ cluster_pcm_x_master_image_ready_exact(const PcmXGrantPayload *image_ready, } +/* The exact commit proof runs against the effective grant base: the one-shot + * INSTALL_READY rebase when an interleaved revoke consumed the enqueue-time + * base on the requester node, the immutable identity base otherwise. */ +static inline uint64 +pcm_x_master_ticket_effective_grant_base(const PcmXMasterTicketSlot *ticket) +{ + return ticket->grant_base_own_generation != 0 ? ticket->grant_base_own_generation + : ticket->ref.identity.base_own_generation; +} + + PcmXQueueResult cluster_pcm_x_master_install_ready_exact(const PcmXInstallReadyPayload *install_ready, - int32 authenticated_node, uint64 authenticated_session, - PcmXPhasePayload *commit_out) + uint64 rebased_own_generation, int32 authenticated_node, + uint64 authenticated_session, PcmXPhasePayload *commit_out) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; PcmXRuntimeSnapshot runtime; @@ -7514,8 +7549,14 @@ cluster_pcm_x_master_install_ready_exact(const PcmXInstallReadyPayload *install_ || install_ready->phase != PGRAC_IC_MSG_PCM_X_INSTALL_READY || install_ready->flags != 0 || authenticated_node < 0 || authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT || authenticated_session == 0 || install_ready->ref.identity.node_id != authenticated_node - || !pcm_x_wait_identity_valid(&install_ready->ref.identity)) + || !pcm_x_wait_identity_valid(&install_ready->ref.identity) + || rebased_own_generation == UINT64_MAX) return PCM_X_QUEUE_INVALID; + /* A nonzero rebase must name a strictly newer effective grant base than + * the enqueue-time identity base it supersedes. */ + if (rebased_own_generation != 0 + && rebased_own_generation <= install_ready->ref.identity.base_own_generation) + return PCM_X_QUEUE_STALE; runtime = cluster_pcm_x_runtime_snapshot(); if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) return PCM_X_QUEUE_NOT_READY; @@ -7540,12 +7581,22 @@ cluster_pcm_x_master_install_ready_exact(const PcmXInstallReadyPayload *install_ result = PCM_X_QUEUE_STALE; else if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_COMMIT_X, authenticated_node, authenticated_session)) - result = PCM_X_QUEUE_DUPLICATE; - else + /* Replay after COMMIT_X armed: the published grant base is frozen; a + * different value is evidence divergence, never contention. */ + result = ticket->grant_base_own_generation == rebased_own_generation ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_CORRUPT; + else { result = pcm_x_transfer_leg_advance_locked( ticket, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, authenticated_node, authenticated_session, PGRAC_IC_MSG_PCM_X_INSTALL_READY, PGRAC_IC_MSG_PCM_X_COMMIT_X, authenticated_node, authenticated_session); + if (result == PCM_X_QUEUE_OK && rebased_own_generation != 0) { + /* One-shot publish, atomic with arming COMMIT_X under this lock: + * a committed COMMIT_X leg implies the grant base is persisted. */ + ticket->grant_base_own_generation = rebased_own_generation; + pg_write_barrier(); + } + } if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { commit_out->ref = ticket->ref; commit_out->phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; @@ -7580,10 +7631,13 @@ cluster_pcm_x_master_final_ack_prepare_exact(const PcmXFinalAckPayload *final_ac || final_ack->ref.identity.node_id != authenticated_node || !pcm_x_wait_identity_valid(&final_ack->ref.identity)) return PCM_X_QUEUE_INVALID; + /* Monotonic ingress floor only: the exact "+1" proof runs under the ticket + * lock against the effective grant base, which a published INSTALL_READY + * rebase may have moved past the enqueue-time identity base. */ if (!cluster_pcm_x_generation_next(final_ack->ref.identity.base_own_generation, &expected_generation)) return PCM_X_QUEUE_COUNTER_EXHAUSTED; - if (final_ack->committed_own_generation != expected_generation) + if (final_ack->committed_own_generation < expected_generation) return PCM_X_QUEUE_STALE; runtime = cluster_pcm_x_runtime_snapshot(); if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) @@ -7609,6 +7663,11 @@ cluster_pcm_x_master_final_ack_prepare_exact(const PcmXFinalAckPayload *final_ac else if (ticket->image.image_id != final_ack->image_id || !pcm_x_image_token_valid(&ticket->image)) result = PCM_X_QUEUE_STALE; + else if (!cluster_pcm_x_generation_next(pcm_x_master_ticket_effective_grant_base(ticket), + &expected_generation) + || final_ack->committed_own_generation != expected_generation) + /* FINAL_ACK proves exactly one ownership round under this grant. */ + result = PCM_X_QUEUE_STALE; else if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK, authenticated_node, authenticated_session)) result = pcm_x_transfer_peer_exact(ticket, authenticated_node, authenticated_session) @@ -7662,10 +7721,12 @@ cluster_pcm_x_master_final_ack_finalize_exact(const PcmXMasterFinalAckToken *tok || !pcm_x_image_token_valid(&token->image) || token->image.image_id != token->final_ack.image_id) return PCM_X_QUEUE_INVALID; + /* Monotonic floor only; the exact "+1" proof re-runs under the ticket lock + * against the effective grant base. */ if (!cluster_pcm_x_generation_next(token->final_ack.ref.identity.base_own_generation, &expected_generation)) return PCM_X_QUEUE_COUNTER_EXHAUSTED; - if (token->final_ack.committed_own_generation != expected_generation) + if (token->final_ack.committed_own_generation < expected_generation) return PCM_X_QUEUE_STALE; runtime = cluster_pcm_x_runtime_snapshot(); if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0 @@ -7696,6 +7757,10 @@ cluster_pcm_x_master_final_ack_finalize_exact(const PcmXMasterFinalAckToken *tok || !pcm_x_transfer_peer_exact(ticket, token->authenticated_node, token->authenticated_session)) result = PCM_X_QUEUE_STALE; + else if (!cluster_pcm_x_generation_next(pcm_x_master_ticket_effective_grant_base(ticket), + &expected_generation) + || token->final_ack.committed_own_generation != expected_generation) + result = PCM_X_QUEUE_STALE; else if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK, token->authenticated_node, token->authenticated_session)) { if (ticket->reliable.state_sequence == token->state_sequence) @@ -9995,6 +10060,7 @@ pcm_x_local_tag_init_common(PcmXLocalTagSlot *tag_slot, const BufferTag *tag, ui tag_slot->closed_round_member_count = 0; tag_slot->terminal_drain_generation = 0; tag_slot->committed_own_generation = 0; + tag_slot->grant_base_own_generation = 0; memset(&tag_slot->holder_ref, 0, sizeof(tag_slot->holder_ref)); memset(&tag_slot->holder_image, 0, sizeof(tag_slot->holder_image)); memset(&tag_slot->holder_reliable, 0, sizeof(tag_slot->holder_reliable)); @@ -10153,7 +10219,7 @@ pcm_x_local_writer_holder_authority_check(const PcmXLocalHolderKey *key, || tag_slot->ref.grant_generation == 0 || !pcm_x_wait_identity_equal(&tag_slot->ref.identity, &leader->identity) || tag_slot->committed_own_generation == 0 - || !cluster_pcm_x_generation_next(leader->identity.base_own_generation, + || !cluster_pcm_x_generation_next(pcm_x_local_effective_grant_base(tag_slot), &expected_generation) || tag_slot->committed_own_generation != expected_generation) return PCM_X_QUEUE_CORRUPT; @@ -13755,6 +13821,7 @@ cluster_pcm_x_local_progress_exact(const PcmXLocalHandle *handle, PcmXLocalProgr progress_out->phase = tag_slot->reliable.phase; progress_out->master_session_incarnation = tag_slot->master_session_incarnation; progress_out->master_node = tag_slot->master_node; + progress_out->grant_base_own_generation = tag_slot->grant_base_own_generation; result = PCM_X_QUEUE_OK; progress_done: @@ -14378,7 +14445,7 @@ cluster_pcm_x_local_leader_rekey_generation_exact(const PcmXLocalHandle *leader, } memset(&zero_image, 0, sizeof(zero_image)); if (!pcm_x_image_token_equal(&tag_slot->image, &zero_image) - || tag_slot->committed_own_generation != 0) { + || tag_slot->committed_own_generation != 0 || tag_slot->grant_base_own_generation != 0) { result = PCM_X_QUEUE_CORRUPT; fail_closed = true; goto rekey_local_done; @@ -14779,6 +14846,7 @@ cluster_pcm_x_local_writer_claim_exact(const PcmXLocalHandle *writer, claim_out->claim_generation = next_claim_generation; claim_out->local_round = member->admitted_round; claim_out->role = member->role; + claim_out->grant_base_own_generation = 0; result = PCM_X_QUEUE_OK; claim_release_gate: @@ -15983,6 +16051,7 @@ cluster_pcm_x_local_prepare_grant_exact(const PcmXLocalHandle *leader, tag_slot->ref = prepare->ref; tag_slot->image = prepare->image; tag_slot->committed_own_generation = 0; + tag_slot->grant_base_own_generation = 0; tag_slot->reliable.last_responder_node = (uint32)authenticated_node; tag_slot->reliable.last_response_opcode = PGRAC_IC_MSG_PCM_X_PREPARE_GRANT; pg_write_barrier(); @@ -16036,10 +16105,12 @@ pcm_x_local_transfer_arm_exact(const PcmXLocalHandle *leader, uint16 opcode, || !pcm_x_wait_identity_valid(&leader->identity)) return PCM_X_QUEUE_INVALID; if (opcode == PGRAC_IC_MSG_PCM_X_FINAL_ACK) { + /* Monotonic floor only; the exact "+1" proof re-runs under the local + * lock against the effective grant base. */ if (!cluster_pcm_x_generation_next(leader->identity.base_own_generation, &expected_generation)) return PCM_X_QUEUE_COUNTER_EXHAUSTED; - if (committed_own_generation != expected_generation) + if (committed_own_generation < expected_generation) return PCM_X_QUEUE_STALE; } runtime = cluster_pcm_x_runtime_snapshot(); @@ -16109,6 +16180,19 @@ pcm_x_local_transfer_arm_exact(const PcmXLocalHandle *leader, uint16 opcode, result = PCM_X_QUEUE_BAD_STATE; goto arm_done; } + if (opcode == PGRAC_IC_MSG_PCM_X_FINAL_ACK) { + if (!cluster_pcm_x_generation_next(pcm_x_local_effective_grant_base(tag_slot), + &expected_generation)) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + fail_closed = true; + goto arm_done; + } + /* FINAL_ACK proves exactly one ownership round under this grant. */ + if (committed_own_generation != expected_generation) { + result = PCM_X_QUEUE_STALE; + goto arm_done; + } + } if (!cluster_pcm_x_generation_next(tag_slot->reliable.state_sequence, &next_sequence)) { result = PCM_X_QUEUE_COUNTER_EXHAUSTED; fail_closed = true; @@ -16137,6 +16221,7 @@ pcm_x_local_transfer_arm_exact(const PcmXLocalHandle *leader, uint16 opcode, install_ready_out->image_id = tag_slot->image.image_id; install_ready_out->result = PCM_X_QUEUE_OK; install_ready_out->phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; + install_ready_out->rebased_own_generation = tag_slot->grant_base_own_generation; } else { final_ack_out->ref = tag_slot->ref; final_ack_out->image_id = tag_slot->image.image_id; @@ -16153,6 +16238,80 @@ pcm_x_local_transfer_arm_exact(const PcmXLocalHandle *leader, uint16 opcode, } +/* + * One-shot pre-reservation publication of the effective grant base for a + * prepared transfer whose enqueue-time base was consumed by an interleaved + * revoke. Runs strictly between PREPARE_GRANT apply and the INSTALL_READY + * arm: the reliable leg must be clear with PREPARE_GRANT as its last + * response. A same-value replay is DUPLICATE; a different second value is + * evidence divergence and fails the runtime closed. The immutable identity + * (directory key) is never touched. + */ +PcmXQueueResult +cluster_pcm_x_local_grant_rebase_publish_exact(const PcmXLocalHandle *leader, + uint64 rebased_own_generation) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint32 partition; + bool fail_closed = false; + + if (header == NULL || leader == NULL || rebased_own_generation == 0 + || rebased_own_generation == UINT64_MAX || !pcm_x_wait_identity_valid(&leader->identity)) + return PCM_X_QUEUE_INVALID; + if (rebased_own_generation <= leader->identity.base_own_generation) + return PCM_X_QUEUE_STALE; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(leader, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + result = pcm_x_local_transfer_slots_exact(leader, tag_ref, member_ref, &tag_slot, &member); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto rebase_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto rebase_done; + } + /* Only a prepared, not-yet-armed transfer may move its grant base. */ + if (tag_slot->ref.grant_generation == 0 || !pcm_x_image_token_valid(&tag_slot->image) + || pcm_x_slot_state_read(&member->slot) != PCM_XL_REMOTE_WAIT + || !pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) + || tag_slot->reliable.last_response_opcode != PGRAC_IC_MSG_PCM_X_PREPARE_GRANT) { + result = PCM_X_QUEUE_BAD_STATE; + goto rebase_done; + } + if (tag_slot->grant_base_own_generation == rebased_own_generation) { + result = PCM_X_QUEUE_DUPLICATE; + goto rebase_done; + } + if (tag_slot->grant_base_own_generation != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rebase_done; + } + tag_slot->grant_base_own_generation = rebased_own_generation; + pg_write_barrier(); + result = PCM_X_QUEUE_OK; + +rebase_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + PcmXQueueResult cluster_pcm_x_local_install_ready_arm_exact(const PcmXLocalHandle *leader, const PcmXTicketRef *expected_ref, @@ -18624,6 +18783,7 @@ pcm_x_local_reset_holder_only_queue(PcmXLocalTagSlot *tag_slot) tag_slot->active_writer_slot_generation = 0; tag_slot->closed_round_member_count = 0; tag_slot->committed_own_generation = 0; + tag_slot->grant_base_own_generation = 0; } @@ -18869,6 +19029,7 @@ pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_pr * and its generation rekey correctly classifies the tag as corrupt. */ memset(&tag_slot->image, 0, sizeof(tag_slot->image)); tag_slot->committed_own_generation = 0; + tag_slot->grant_base_own_generation = 0; /* DRAIN closes the old round by leaving its response opcode as a replay * tombstone. RETIRE consumes that tombstone with the membership: a * promoted successor must see a pristine application leg or the requester diff --git a/src/backend/cluster/cluster_sf_dep.c b/src/backend/cluster/cluster_sf_dep.c index f52774731c..1d1ae2557c 100644 --- a/src/backend/cluster/cluster_sf_dep.c +++ b/src/backend/cluster/cluster_sf_dep.c @@ -371,6 +371,22 @@ cluster_sf_peer_supports_pcm_x_convert(int32 peer_id) return (capabilities & PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1) != 0; } +/* A' rebase: the V2 INSTALL_READY frame is legal on this connection only when + * its verified HELLO advertised the rebase protocol capability. */ +bool +cluster_sf_peer_supports_pcm_x_rebase(int32 peer_id) +{ + uint32 capabilities; + + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return false; + + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + capabilities = cluster_sf_peer_cap_bits(&ClusterSfDep->peer_capabilities[peer_id]); + LWLockRelease(&ClusterSfDep->lock); + return (capabilities & PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1) != 0; +} + /* Return one lock-coherent snapshot of the capability-record generation whose * verified HELLO advertised PCM-X. Tier1 owns this record on CONTROL and can * sample it around another authority read to reject reconnect ABA; RDMA keeps diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index e3d2e8e7b4..5b66ae30c5 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -984,16 +984,25 @@ cluster_gcs_pcm_x_prepare_grant_ingress_valid(const PcmXGrantPayload *grant, Siz && cluster_gcs_pcm_x_image_token_wire_valid(&grant->image, authenticated_node); } -/* INSTALL_READY is one canonical requester -> master application ACK. */ +/* INSTALL_READY is one canonical requester -> master application ACK. Both + * the V1 (104-byte, no rebase) and V2 (112-byte) exact frames are legal; the + * caller normalizes a V1 frame to rebased_own_generation == 0 before this + * check. A V1 frame carrying a nonzero rebase is impossible by construction + * and a V2 rebase must be strictly newer than the immutable identity base. */ static inline bool cluster_gcs_pcm_x_install_ready_ingress_valid(const PcmXInstallReadyPayload *ready, Size payload_length, int32 authenticated_node, uint64 current_epoch, int32 tag_master, int32 local_node) { - return ready != NULL && payload_length == sizeof(*ready) && authenticated_node >= 0 - && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 - && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == local_node + return ready != NULL + && (payload_length == sizeof(*ready) || payload_length == PCM_X_INSTALL_READY_V1_LEN) + && (payload_length == sizeof(*ready) || ready->rebased_own_generation == 0) + && (ready->rebased_own_generation == 0 + || (ready->rebased_own_generation != UINT64_MAX + && ready->rebased_own_generation > ready->ref.identity.base_own_generation)) + && authenticated_node >= 0 && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT + && local_node >= 0 && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == local_node && cluster_gcs_pcm_x_transfer_ref_wire_valid(&ready->ref, current_epoch) && ready->ref.identity.node_id == authenticated_node && cluster_gcs_pcm_x_image_id_master_wire_valid(ready->image_id, local_node) @@ -1015,7 +1024,10 @@ cluster_gcs_pcm_x_commit_x_ingress_valid(const PcmXPhasePayload *commit, Size pa && commit->phase == PGRAC_IC_MSG_PCM_X_COMMIT_X && commit->flags == 0; } -/* FINAL_ACK proves the requester committed exactly one ownership generation. */ +/* FINAL_ACK proves the requester committed exactly one ownership generation. + * The wire check is only a monotonic floor against the immutable identity + * base: a published INSTALL_READY rebase may have moved the effective grant + * base, and the exact "+1" proof runs under the master ticket lock. */ static inline bool cluster_gcs_pcm_x_final_ack_ingress_valid(const PcmXFinalAckPayload *ack, Size payload_length, int32 authenticated_node, uint64 current_epoch, @@ -1028,7 +1040,7 @@ cluster_gcs_pcm_x_final_ack_ingress_valid(const PcmXFinalAckPayload *ack, Size p && ack->ref.identity.node_id == authenticated_node && cluster_gcs_pcm_x_image_id_master_wire_valid(ack->image_id, local_node) && ack->ref.identity.base_own_generation != UINT64_MAX - && ack->committed_own_generation == ack->ref.identity.base_own_generation + 1; + && ack->committed_own_generation > ack->ref.identity.base_own_generation; } /* FINAL_COMMIT_ACK is the canonical master -> requester application ACK. */ diff --git a/src/include/cluster/cluster_ic.h b/src/include/cluster/cluster_ic.h index 4f10bf143f..ed626ff601 100644 --- a/src/include/cluster/cluster_ic.h +++ b/src/include/cluster/cluster_ic.h @@ -368,6 +368,16 @@ typedef enum ClusterICPlane { * family. A sender must not transmit any PCM-X frame to a peer that did not * advertise this bit; an old peer would treat the type as unregistered. */ #define PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 ((uint32)0x00000200U) +/* PGRAC: A' rebase (t/400 item 2) — this binary understands the V2 (112-byte) + * INSTALL_READY frame whose trailing rebased_own_generation publishes the + * effective grant base after an interleaved revoke consumed the enqueue-time + * base. A PROTOCOL capability, advertised unconditionally. Activation is + * formation-wide: the PCM-X runtime samples every MEMBER's bit at activation + * and only then may a requester publish a rebase; without full coverage the + * V2 frame is never sent (a drifted requester keeps the fail-closed STALE + * verdict) and every INSTALL_READY stays the V1 104-byte exact frame, so an + * old master never sees a length its byte-exact table would refuse. */ +#define PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1 ((uint32)0x00000400U) /* * PGRAC: spec-7.2 D2 — plane + connection-epoch ride the documented-zero * pad region (capabilities precedent: occupy pad bytes, do not resize V1). diff --git a/src/include/cluster/cluster_pcm_x_bufmgr.h b/src/include/cluster/cluster_pcm_x_bufmgr.h index 6bd548d26c..fbb639caa2 100644 --- a/src/include/cluster/cluster_pcm_x_bufmgr.h +++ b/src/include/cluster/cluster_pcm_x_bufmgr.h @@ -131,7 +131,14 @@ cluster_pcm_x_writer_grant_snapshot_exact(const PcmXLocalWriterClaim *claim, return claim != NULL && granted != NULL && live != NULL && claim->flags == 0 && claim->writer.flags == 0 && claim->claim_generation != 0 && claim->writer.identity.base_own_generation != UINT64_MAX - && granted->generation == claim->writer.identity.base_own_generation + 1 + && claim->grant_base_own_generation != UINT64_MAX + /* A' rebase: the requester copies the published effective grant + * base into the claim; zero keeps the enqueue-time identity base. */ + && granted->generation + == (claim->grant_base_own_generation != 0 + ? claim->grant_base_own_generation + : claim->writer.identity.base_own_generation) + + 1 && granted->reservation_token != 0 && granted->flags == 0 && granted->pcm_state == (uint8)PCM_STATE_X && BufferTagsEqual(&granted->tag, &claim->writer.identity.tag) diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h index 7a498b6ea0..e10030bc8f 100644 --- a/src/include/cluster/cluster_pcm_x_convert.h +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -204,8 +204,16 @@ typedef struct PcmXInstallReadyPayload { uint32 result; uint16 phase; uint16 flags; + /* A' rebase: nonzero publishes the effective grant base exactly once (an + * interleaved revoke consumed ref.identity.base_own_generation while the + * request was queued). ref stays the immutable admission-time locator. + * Only sent as the V2 (112-byte) frame when the whole formation + * advertises PCM_X_REBASE_V1; the V1 frame is the first 104 bytes. */ + uint64 rebased_own_generation; } PcmXInstallReadyPayload; +#define PCM_X_INSTALL_READY_V1_LEN ((Size)104) + typedef struct PcmXFinalAckPayload { PcmXTicketRef ref; uint64 image_id; @@ -251,7 +259,10 @@ StaticAssertDecl(sizeof(PcmXAdmitAckPayload) == 112, "PCM-X ADMIT_ACK ABI"); StaticAssertDecl(sizeof(PcmXPhasePayload) == 96, "PCM-X phase ABI"); StaticAssertDecl(sizeof(PcmXRevokePayload) == 96, "PCM-X revoke ABI"); StaticAssertDecl(sizeof(PcmXGrantPayload) == 128, "PCM-X grant ABI"); -StaticAssertDecl(sizeof(PcmXInstallReadyPayload) == 104, "PCM-X INSTALL_READY ABI"); +StaticAssertDecl(sizeof(PcmXInstallReadyPayload) == 112, "PCM-X INSTALL_READY ABI"); +StaticAssertDecl(offsetof(PcmXInstallReadyPayload, rebased_own_generation) + == PCM_X_INSTALL_READY_V1_LEN, + "PCM-X INSTALL_READY V1 prefix ABI"); StaticAssertDecl(sizeof(PcmXFinalAckPayload) == 104, "PCM-X FINAL_ACK ABI"); StaticAssertDecl(sizeof(PcmXBlockerSetHeaderPayload) == 104, "PCM-X blocker header ABI"); StaticAssertDecl(sizeof(PcmXBlockerChunkPayload) == 128, "PCM-X blocker chunk ABI"); @@ -328,9 +339,11 @@ typedef struct PcmXRuntimeSnapshot { uint64 master_session_incarnation; uint32 gate_generation; PcmXRuntimeState state; + /* A' rebase: formation-wide PCM_X_REBASE_V1 coverage at activation. */ + bool rebase_wire_active; } PcmXRuntimeSnapshot; -StaticAssertDecl(sizeof(PcmXRuntimeSnapshot) == 16, "PCM-X runtime snapshot ABI"); +StaticAssertDecl(sizeof(PcmXRuntimeSnapshot) == 24, "PCM-X runtime snapshot ABI"); typedef enum PcmXMasterTicketState { PCM_XT_FREE = PCM_X_SLOT_FREE, @@ -563,6 +576,9 @@ typedef struct PcmXLocalWriterClaim { uint32 local_round; uint16 role; uint16 flags; + /* A' rebase: the requester copies the published effective grant base here + * (0 = no drift) so the bufmgr grant-snapshot cross-check stays exact. */ + uint64 grant_base_own_generation; } PcmXLocalWriterClaim; /* @@ -586,9 +602,11 @@ typedef struct PcmXLocalProgress { uint64 master_session_incarnation; int32 master_node; uint32 reserved; + /* A' rebase: the tag's published effective grant base (0 = no drift). */ + uint64 grant_base_own_generation; } PcmXLocalProgress; -StaticAssertDecl(sizeof(PcmXLocalProgress) == 240, "PCM-X local progress ABI"); +StaticAssertDecl(sizeof(PcmXLocalProgress) == 248, "PCM-X local progress ABI"); /* Read-only process-local view of the independent holder transfer lane. */ typedef struct PcmXLocalHolderProgress { @@ -750,7 +768,7 @@ StaticAssertDecl(sizeof(PcmXLocalFollowerWfgSnapshot) == 248, "PCM-X local follower WFG snapshot process-local ABI"); StaticAssertDecl(offsetof(PcmXLocalFollowerWfgSnapshot, waiter_graph_generation) == 192, "PCM-X local follower sampled graph offset"); -StaticAssertDecl(sizeof(PcmXLocalWriterClaim) == 144, "PCM-X local writer claim process-local ABI"); +StaticAssertDecl(sizeof(PcmXLocalWriterClaim) == 152, "PCM-X local writer claim process-local ABI"); StaticAssertDecl(offsetof(PcmXLocalWriterClaim, claim_generation) == 128, "PCM-X local writer claim generation offset"); StaticAssertDecl(sizeof(PcmXLocalCutoff) == 48, "PCM-X local cutoff process-local ABI"); @@ -920,6 +938,12 @@ typedef struct PcmXMasterTicketSlot { uint32 involved_nodes_bitmap; uint32 drained_nodes_bitmap; uint32 retire_acked_nodes_bitmap; + /* A' rebase (t/400 item 2): effective grant base published exactly once by + * INSTALL_READY when an interleaved revoke consumed the enqueue-time + * base_own_generation on the requester node. Zero = no drift; the wire + * ref identity stays immutable and every exact committed==base+1 check + * runs against the effective base instead. */ + uint64 grant_base_own_generation; } PcmXMasterTicketSlot; typedef struct PcmXBlockerSlot { @@ -971,6 +995,11 @@ typedef struct PcmXLocalTagSlot { uint64 holder_terminal_drain_generation; PcmXTicketRef blocker_snapshot_ref; PcmXReliableLegState blocker_snapshot_reliable; + /* A' rebase (t/400 item 2): effective grant base published exactly once + * (pre-reservation) when an interleaved revoke consumed the enqueue-time + * base_own_generation. Zero = no drift. Writer-round evidence: cleared + * with image/committed_own_generation, never with the holder lane. */ + uint64 grant_base_own_generation; } PcmXLocalTagSlot; typedef struct PcmXLocalMembershipSlot { @@ -1002,9 +1031,13 @@ StaticAssertDecl(sizeof(PcmXReliableLegState) == 56, "PCM-X reliable leg ABI"); StaticAssertDecl(sizeof(PcmXMasterTagSlot) == 120, "PCM-X master tag slot ABI"); StaticAssertDecl(offsetof(PcmXMasterTagSlot, outstanding_ticket_count) == 112, "PCM-X master outstanding-ticket count offset"); -StaticAssertDecl(sizeof(PcmXMasterTicketSlot) == 384, "PCM-X master ticket slot ABI"); +StaticAssertDecl(sizeof(PcmXMasterTicketSlot) == 392, "PCM-X master ticket slot ABI"); +StaticAssertDecl(offsetof(PcmXMasterTicketSlot, grant_base_own_generation) == 384, + "PCM-X master ticket grant-base offset"); StaticAssertDecl(sizeof(PcmXBlockerSlot) == 128, "PCM-X blocker slot ABI"); -StaticAssertDecl(sizeof(PcmXLocalTagSlot) == 752, "PCM-X local tag slot ABI"); +StaticAssertDecl(sizeof(PcmXLocalTagSlot) == 760, "PCM-X local tag slot ABI"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, grant_base_own_generation) == 752, + "PCM-X local tag grant-base offset"); StaticAssertDecl(offsetof(PcmXLocalTagSlot, membership_count) == 384, "PCM-X local membership count offset"); StaticAssertDecl(offsetof(PcmXLocalTagSlot, closed_round_member_count) == 392, @@ -1215,6 +1248,14 @@ typedef struct PcmXShmemHeader { uint32 terminal_note_count; uint32 terminal_note_reserved; uint64 terminal_note_ticket; + /* A' rebase: nonzero only when EVERY member of the bound formation + * advertised PCM_X_REBASE_V1 at activation. Written by the activating + * formation tick strictly before the ACTIVE gate publish and immutable + * for the runtime's lifetime; readers consume it through the snapshot's + * double-gate sample. Gates rebase publication and the V2 INSTALL_READY + * frame, never the base protocol. */ + uint32 rebase_wire_active; + uint32 rebase_reserved; } PcmXShmemHeader; /* Terminal-kick note arms (diagnostic identifiers, not protocol state). */ @@ -1252,7 +1293,7 @@ StaticAssertDecl(offsetof(PcmXShmemHeader, peer_frontiers) == 33664, StaticAssertDecl(offsetof(PcmXShmemHeader, stats) == 35200, "PCM-X stats offset"); StaticAssertDecl(offsetof(PcmXShmemHeader, outbound_targets) == 35376, "PCM-X outbound target frontier array offset"); -StaticAssertDecl(sizeof(PcmXShmemHeader) == 36504, "PCM-X shmem header ABI"); +StaticAssertDecl(sizeof(PcmXShmemHeader) == 36512, "PCM-X shmem header ABI"); typedef enum PcmXAttachResult { PCM_X_ATTACH_OK = 0, @@ -1303,6 +1344,9 @@ extern void cluster_pcm_x_stats_note_own_abort(void); extern void cluster_pcm_x_stats_note_own_busy(void); extern void cluster_pcm_x_stats_note_own_corrupt(void); extern bool cluster_pcm_x_runtime_activate(uint64 master_session_incarnation); +/* A' rebase: publish formation-wide PCM_X_REBASE_V1 coverage. Legal only + * from the activating formation tick, strictly before activate_bound(). */ +extern void cluster_pcm_x_runtime_set_rebase_wire_active(bool active); extern bool cluster_pcm_x_runtime_activate_bound(uint64 master_session_incarnation, const PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_LIMIT]); @@ -1423,10 +1467,9 @@ extern PcmXQueueResult cluster_pcm_x_master_image_ready_exact(const PcmXGrantPay int32 authenticated_node, uint64 authenticated_session, PcmXGrantPayload *prepare_out); -extern PcmXQueueResult -cluster_pcm_x_master_install_ready_exact(const PcmXInstallReadyPayload *install_ready, - int32 authenticated_node, uint64 authenticated_session, - PcmXPhasePayload *commit_out); +extern PcmXQueueResult cluster_pcm_x_master_install_ready_exact( + const PcmXInstallReadyPayload *install_ready, uint64 rebased_own_generation, + int32 authenticated_node, uint64 authenticated_session, PcmXPhasePayload *commit_out); extern PcmXQueueResult cluster_pcm_x_master_final_ack_exact(const PcmXFinalAckPayload *final_ack, int32 authenticated_node, uint64 authenticated_session, @@ -1556,6 +1599,12 @@ extern PcmXQueueResult cluster_pcm_x_local_prepare_grant_exact(const PcmXLocalHa const PcmXGrantPayload *prepare, int32 authenticated_node, uint64 authenticated_session); +/* A' rebase: one-shot pre-reservation publication of the effective grant base + * for the prepared transfer. Same value replays as DUPLICATE; a different + * second value is evidence divergence and fails closed. */ +extern PcmXQueueResult +cluster_pcm_x_local_grant_rebase_publish_exact(const PcmXLocalHandle *leader, + uint64 rebased_own_generation); extern PcmXQueueResult cluster_pcm_x_local_install_ready_arm_exact( const PcmXLocalHandle *leader, const PcmXTicketRef *expected_ref, const PcmXImageToken *expected_image, PcmXInstallReadyPayload *install_ready_out, diff --git a/src/include/cluster/cluster_sf_dep.h b/src/include/cluster/cluster_sf_dep.h index a02d0e4ef3..50403b81c2 100644 --- a/src/include/cluster/cluster_sf_dep.h +++ b/src/include/cluster/cluster_sf_dep.h @@ -236,6 +236,7 @@ extern bool cluster_sf_peer_supports_xid_native_disable(int32 peer_id); extern bool cluster_sf_peer_supports_xid_authority_flock(int32 peer_id); extern bool cluster_sf_peer_supports_gcs_inval_busy(int32 peer_id); extern bool cluster_sf_peer_supports_pcm_x_convert(int32 peer_id); +extern bool cluster_sf_peer_supports_pcm_x_rebase(int32 peer_id); extern bool cluster_sf_peer_pcm_x_connection_generation(int32 peer_id, uint32 *generation); extern void cluster_sf_note_peer_disconnected_gen(int32 peer_id, uint32 generation); extern void cluster_sf_note_peer_disconnected(int32 peer_id); diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 90cc0a35a7..76d3af3e08 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -1312,6 +1312,29 @@ UT_TEST(test_pcm_x_install_ready_ingress_is_canonical_requester_ack) UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); UT_ASSERT(cluster_pcm_x_image_id_encode(3, 37, &ready.image_id)); UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + + /* A' rebase: both exact frame lengths are legal, nothing in between; a + * V1-length frame must carry a zero rebase and a V2 rebase must be + * strictly newer than the immutable identity base and not the exhausted + * sentinel. */ + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 37, &ready.image_id)); + UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, PCM_X_INSTALL_READY_V1_LEN, 1, + 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, PCM_X_INSTALL_READY_V1_LEN + 4, + 1, 11, 2, 2)); + ready.ref.identity.base_own_generation = 5; + ready.rebased_own_generation = 8; + UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, PCM_X_INSTALL_READY_V1_LEN, 1, + 11, 2, 2)); + ready.rebased_own_generation = 5; + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + ready.rebased_own_generation = 4; + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + ready.rebased_own_generation = UINT64_MAX; + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + ready.rebased_own_generation = 0; + ready.ref.identity.base_own_generation = 0; } @@ -1334,7 +1357,7 @@ UT_TEST(test_pcm_x_commit_x_ingress_is_canonical_master_phase) } -UT_TEST(test_pcm_x_final_ack_ingress_binds_exact_committed_generation) +UT_TEST(test_pcm_x_final_ack_ingress_binds_monotonic_committed_floor) { PcmXFinalAckPayload ack; uint64 image_id; @@ -1348,8 +1371,19 @@ UT_TEST(test_pcm_x_final_ack_ingress_binds_exact_committed_generation) UT_ASSERT(cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); UT_ASSERT(!cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 3, 11, 2, 2)); + /* A' rebase: the wire check is only a monotonic floor (committed > base); + * a published rebase legally lifts committed past base+1 and the exact + * "+1" proof runs under the master ticket lock against the effective + * grant base. At or below the base stays refused. */ ack.committed_own_generation = 2; + UT_ASSERT(cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); + ack.ref.identity.base_own_generation = 5; + ack.committed_own_generation = 5; UT_ASSERT(!cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); + ack.committed_own_generation = 4; + UT_ASSERT(!cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); + ack.committed_own_generation = 9; + UT_ASSERT(cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); ack.ref.identity.base_own_generation = UINT64_MAX; ack.committed_own_generation = 0; UT_ASSERT(!cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); @@ -3488,7 +3522,7 @@ main(void) UT_RUN(test_pcm_x_prepare_grant_ingress_binds_master_to_requester); UT_RUN(test_pcm_x_install_ready_ingress_is_canonical_requester_ack); UT_RUN(test_pcm_x_commit_x_ingress_is_canonical_master_phase); - UT_RUN(test_pcm_x_final_ack_ingress_binds_exact_committed_generation); + UT_RUN(test_pcm_x_final_ack_ingress_binds_monotonic_committed_floor); UT_RUN(test_pcm_x_final_commit_ack_ingress_is_canonical_master_phase); UT_RUN(test_pcm_x_final_confirm_ingress_is_canonical_requester_phase); UT_RUN(test_pcm_x_master_drive_selects_exact_authority_and_next_holder); diff --git a/src/test/cluster_unit/test_cluster_ic.c b/src/test/cluster_unit/test_cluster_ic.c index 71877a8c7f..1f02607bdc 100644 --- a/src/test/cluster_unit/test_cluster_ic.c +++ b/src/test/cluster_unit/test_cluster_ic.c @@ -597,9 +597,10 @@ UT_TEST(test_hello_wire_reference_bytes) * (0x10) + round-3 P0-1 xid wrap barrier (0x20) + round-4 P0-1 * authority flock (0x40) + ownership-gen ruling② invalidate BUSY * (0x80) + TT-lane undo-horizon idle sentinel (0x100) + PCM-X - * conversion (0x200) (smart-fusion is off in this fixture) */ + * conversion (0x200) + A' rebase V2 INSTALL_READY (0x400) + * (smart-fusion is off in this fixture) */ UT_ASSERT_EQ(wire[36], 0xFE); - UT_ASSERT_EQ(wire[37], 0x03); + UT_ASSERT_EQ(wire[37], 0x07); UT_ASSERT_EQ(wire[38], 0x00); UT_ASSERT_EQ(wire[39], 0x00); /* remaining _pad must be all zero: a CONTROL-plane HELLO with @@ -698,11 +699,11 @@ UT_TEST(test_hello_smart_fusion_capability_gate) | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 - | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1); + | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1); /* Keep the aggregate word byte-exact as well as symbolically composed: * parallel protocol lanes have collided while preserving the same symbolic * expectation, so the literal catches accidental bit reuse. */ - UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), (uint32)0x000003FEU); + UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), (uint32)0x000007FEU); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_2; @@ -716,7 +717,7 @@ UT_TEST(test_hello_smart_fusion_capability_gate) | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 - | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1); + | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_3; @@ -730,7 +731,8 @@ UT_TEST(test_hello_smart_fusion_capability_gate) | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1 | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 - | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1); + | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 + | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1); cluster_smart_fusion = false; cluster_interconnect_tier = CLUSTER_IC_TIER_STUB; @@ -790,6 +792,7 @@ UT_TEST(test_hello_gcs_done_and_wrap_barrier_gates) UT_ASSERT_EQ((int)PGRAC_IC_MSG_XID_NATIVE_DISABLE, 39); UT_ASSERT_EQ((int)PGRAC_IC_MSG_XID_NATIVE_DISABLE_ACK, 40); UT_ASSERT_EQ(PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, (uint32)0x00000200U); + UT_ASSERT_EQ(PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, (uint32)0x00000400U); UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_ENQUEUE, 41); UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_ADMIT_ACK, 42); UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM, 43); diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index 7a9a9345e7..88358572c5 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -1822,6 +1822,19 @@ UT_TEST(test_queue_writer_grant_snapshot_is_claim_and_generation_exact) live.pcm_state = (uint8)PCM_STATE_S; UT_ASSERT(!cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); live = granted; + /* A' rebase: a published effective grant base supersedes the enqueue-time + * identity base in the exact granted-generation binding. */ + claim.grant_base_own_generation = 14; + UT_ASSERT(!cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); + granted.generation = 15; + live = granted; + UT_ASSERT(cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); + claim.grant_base_own_generation = UINT64_MAX; + UT_ASSERT(!cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); + claim.grant_base_own_generation = 0; + granted.generation = 11; + live = granted; + UT_ASSERT(cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); claim.active_slot.slot_generation++; UT_ASSERT(!cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 6b2fd23e99..23fdf2e045 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -1528,6 +1528,25 @@ init_active_pcm_x(uint64 master_session_incarnation) UT_ASSERT(cluster_pcm_x_runtime_activate(master_session_incarnation)); } +/* + * A' rebase: the formation-wide V2 coverage flag is written before activation + * and immutable for the runtime's lifetime; an ACTIVE snapshot exposes it and + * an activation without the flag (a formation with any old member) leaves + * rebase publication disabled. + */ +UT_TEST(test_runtime_rebase_wire_active_is_activation_bound) +{ + init_active_pcm_x(UINT64_C(77)); + UT_ASSERT(!cluster_pcm_x_runtime_snapshot().rebase_wire_active); + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + cluster_pcm_x_runtime_set_rebase_wire_active(true); + UT_ASSERT(cluster_pcm_x_runtime_activate(UINT64_C(77))); + UT_ASSERT(cluster_pcm_x_runtime_snapshot().rebase_wire_active); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + static bool ticket_refs_equal(const PcmXTicketRef *left, const PcmXTicketRef *right) { @@ -2096,7 +2115,9 @@ UT_TEST(test_wire_abi_sizes_are_exact) UT_ASSERT_EQ(sizeof(PcmXPhasePayload), 96); UT_ASSERT_EQ(sizeof(PcmXRevokePayload), 96); UT_ASSERT_EQ(sizeof(PcmXGrantPayload), 128); - UT_ASSERT_EQ(sizeof(PcmXInstallReadyPayload), 104); + UT_ASSERT_EQ(sizeof(PcmXInstallReadyPayload), 112); + UT_ASSERT_EQ(offsetof(PcmXInstallReadyPayload, rebased_own_generation), + PCM_X_INSTALL_READY_V1_LEN); UT_ASSERT_EQ(sizeof(PcmXFinalAckPayload), 104); UT_ASSERT_EQ(sizeof(PcmXBlockerSetHeaderPayload), 104); UT_ASSERT_EQ(sizeof(PcmXBlockerChunkPayload), 128); @@ -2162,7 +2183,7 @@ UT_TEST(test_runtime_layout_abi_and_offsets_are_exact) UT_ASSERT_EQ(offsetof(PcmXSlotHeader, slot_generation_hi), 16); UT_ASSERT_EQ(offsetof(PcmXSlotHeader, state_flags), 20); UT_ASSERT_EQ(sizeof(PcmXReliableLegState), 56); - UT_ASSERT_EQ(sizeof(PcmXLocalProgress), 240); + UT_ASSERT_EQ(sizeof(PcmXLocalProgress), 248); UT_ASSERT_EQ(offsetof(PcmXLocalProgress, master_session_incarnation), 224); UT_ASSERT_EQ(offsetof(PcmXLocalProgress, master_node), 232); UT_ASSERT_EQ(sizeof(PcmXLocalHolderProgress), 160); @@ -2170,9 +2191,11 @@ UT_TEST(test_runtime_layout_abi_and_offsets_are_exact) UT_ASSERT_EQ(offsetof(PcmXLocalHolderProgress, master_node), 152); UT_ASSERT_EQ(sizeof(PcmXLocalBlockerSnapshot), 112); UT_ASSERT_EQ(sizeof(PcmXMasterTagSlot), 120); - UT_ASSERT_EQ(sizeof(PcmXMasterTicketSlot), 384); + UT_ASSERT_EQ(sizeof(PcmXMasterTicketSlot), 392); + UT_ASSERT_EQ(offsetof(PcmXMasterTicketSlot, grant_base_own_generation), 384); UT_ASSERT_EQ(sizeof(PcmXBlockerSlot), 128); - UT_ASSERT_EQ(sizeof(PcmXLocalTagSlot), 752); + UT_ASSERT_EQ(sizeof(PcmXLocalTagSlot), 760); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, grant_base_own_generation), 752); UT_ASSERT_EQ(sizeof(PcmXLocalMembershipSlot), 168); UT_ASSERT_EQ(sizeof(PcmXPeerFrontier), 48); UT_ASSERT_EQ(sizeof(PcmXPeerBinding), 16); @@ -2215,7 +2238,7 @@ UT_TEST(test_runtime_layout_abi_and_offsets_are_exact) UT_ASSERT_EQ(offsetof(PcmXShmemHeader, peer_frontiers), 33664); UT_ASSERT_EQ(offsetof(PcmXShmemHeader, stats), 35200); UT_ASSERT_EQ(offsetof(PcmXShmemHeader, outbound_targets), 35376); - UT_ASSERT_EQ(sizeof(PcmXShmemHeader), 36504); + UT_ASSERT_EQ(sizeof(PcmXShmemHeader), 36512); } UT_TEST(test_lwlock_held_limit_is_shared_200) @@ -6275,7 +6298,7 @@ UT_TEST(test_master_transfer_wire_49_56_is_generation_exact) install_ready.result = PCM_X_QUEUE_OK; install_ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; UT_ASSERT_EQ( - cluster_pcm_x_master_install_ready_exact(&install_ready, 0, requester_session, &commit), + cluster_pcm_x_master_install_ready_exact(&install_ready, 0, 0, requester_session, &commit), PCM_X_QUEUE_OK); UT_ASSERT_EQ(commit.phase, PGRAC_IC_MSG_PCM_X_COMMIT_X); @@ -6327,6 +6350,252 @@ UT_TEST(test_master_transfer_wire_49_56_is_generation_exact) PCM_X_QUEUE_DUPLICATE); } +/* Admit with an explicit nonzero enqueue-time base_own_generation, so the + * strictly-newer rebase arms have a real lower bound to violate. */ +static void +admit_active_probe_with_base(BlockNumber block, int32 node_id, uint32 procno, uint64 request_id, + uint64 sender_session, uint64 prehandle_sequence, + uint64 base_own_generation, PcmXMasterAdmission *admission_out) +{ + PcmXWaitIdentity identity = make_wait_identity(block, node_id, procno, request_id); + PcmXEnqueuePayload request; + PcmXTicketRef active; + + identity.base_own_generation = base_own_generation; + request = make_enqueue(identity, sender_session, prehandle_sequence); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, admission_out), PCM_X_QUEUE_OK); + UT_ASSERT_NOT_NULL(admission_out); + UT_ASSERT_EQ( + cluster_pcm_x_master_admit_confirm_exact(&admission_out->ref, UINT64_C(9000) + request_id), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_promote_head_exact(&identity.tag, identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&active, &admission_out->ref)); +} + +/* Drive an admitted ACTIVE_PROBE ticket through blocker commit, transfer, + * revoke and IMAGE_READY, leaving its reliable leg armed at PREPARE_GRANT. */ +static void +drive_master_ticket_to_prepared(PcmXMasterAdmission *admission, uint64 source_session, + uint64 graph_generation, PcmXTicketRef *transfer_out, + uint64 *image_id_out) +{ + PcmXMasterBlockerSnapshot snapshot; + PcmXBlockerSetHeaderPayload blocker_commit; + PcmXRevokePayload revoke; + PcmXGrantPayload image_ready; + PcmXGrantPayload prepare; + + arm_blocker_probe(&admission->ref, 2, source_session); + blocker_commit = make_blocker_header(&admission->ref, 1, NULL, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 2, source_session, + NULL, 0, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, graph_generation), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_probe_complete_exact(&admission->ref, 1, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission->ref, graph_generation, transfer_out), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(transfer_out, 2, source_session, &revoke), + PCM_X_QUEUE_OK); + memset(&image_ready, 0, sizeof(image_ready)); + image_ready.ref = *transfer_out; + image_ready.image.image_id = revoke.image_id; + image_ready.image.source_own_generation = UINT64_C(44); + image_ready.image.page_scn = UINT64_C(55); + image_ready.image.page_lsn = UINT64_C(66); + image_ready.image.source_node = 2; + image_ready.image.page_checksum = UINT32_C(77); + UT_ASSERT_EQ(cluster_pcm_x_master_image_ready_exact(&image_ready, 2, source_session, &prepare), + PCM_X_QUEUE_OK); + *image_id_out = revoke.image_id; +} + +/* + * A' rebase (t/400 item 2): an interleaved revoke on the requester node + * consumes the enqueue-time base_own_generation while the request is queued. + * INSTALL_READY may publish a strictly-newer effective grant base exactly + * once; the ticket persists it and every later exact "+1" check runs against + * the effective base while the wire ref identity stays immutable. + */ +UT_TEST(test_master_install_ready_rebase_grant_base_is_one_shot) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef transfer; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXMasterFinalAckToken final_ack_token; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + uint64 image_id; + const uint64 requester_session = UINT64_C(7281); + const uint64 source_session = UINT64_C(8281); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe_with_base(810, 0, 20, UINT64_C(8281), requester_session, 1, UINT64_C(5), + &admission); + drive_master_ticket_to_prepared(&admission, source_session, UINT64_C(9281), &transfer, + &image_id); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(ticket->grant_base_own_generation, 0); + + memset(&install_ready, 0, sizeof(install_ready)); + install_ready.ref = transfer; + install_ready.image_id = image_id; + install_ready.result = PCM_X_QUEUE_OK; + install_ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; + + /* Shape violation: the exhausted sentinel can never be a grant base. */ + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, UINT64_MAX, 0, + requester_session, &commit), + PCM_X_QUEUE_INVALID); + /* Not strictly newer than the enqueue-time base: lower, then equal. */ + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, UINT64_C(4), 0, + requester_session, &commit), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, UINT64_C(5), 0, + requester_session, &commit), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(ticket->grant_base_own_generation, 0); + UT_ASSERT_EQ(ticket->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT); + + /* One-shot publish of a >1 drift (5 -> 8), then a same-value replay. */ + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, UINT64_C(8), 0, + requester_session, &commit), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(commit.phase, PGRAC_IC_MSG_PCM_X_COMMIT_X); + UT_ASSERT_EQ(ticket->grant_base_own_generation, UINT64_C(8)); + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, UINT64_C(8), 0, + requester_session, &commit), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(ticket->grant_base_own_generation, UINT64_C(8)); + + /* The effective grant base now drives the exact final-ack "+1" math: the + * enqueue-time math (5+1) is refused, the rebased math (8+1) commits. */ + memset(&final_ack, 0, sizeof(final_ack)); + final_ack.ref = transfer; + final_ack.image_id = image_id; + final_ack.committed_own_generation = UINT64_C(6); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session, + &final_ack_token), + PCM_X_QUEUE_STALE); + final_ack.committed_own_generation = UINT64_C(9); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session, + &final_ack_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_finalize_exact(&final_ack_token, &final_commit), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(final_commit.phase, PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK); + memset(&final_confirm, 0, sizeof(final_confirm)); + final_confirm.ref = transfer; + final_confirm.phase = PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM; + UT_ASSERT_EQ(cluster_pcm_x_master_final_confirm_exact(&final_confirm, 0, requester_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_COMPLETE); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +/* A second, different rebase value is evidence divergence, never contention. */ +UT_TEST(test_master_install_ready_rebase_conflict_fails_closed) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef transfer; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + uint64 image_id; + const uint64 requester_session = UINT64_C(7282); + const uint64 source_session = UINT64_C(8282); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe_with_base(811, 0, 21, UINT64_C(8282), requester_session, 1, UINT64_C(5), + &admission); + drive_master_ticket_to_prepared(&admission, source_session, UINT64_C(9282), &transfer, + &image_id); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + + memset(&install_ready, 0, sizeof(install_ready)); + install_ready.ref = transfer; + install_ready.image_id = image_id; + install_ready.result = PCM_X_QUEUE_OK; + install_ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, UINT64_C(8), 0, + requester_session, &commit), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, UINT64_C(9), 0, + requester_session, &commit), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(ticket->grant_base_own_generation, UINT64_C(8)); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +/* Without a rebase the ticket keeps the enqueue-time math bit for bit, and + * the locked exact check still refuses a wrong committed generation. */ +UT_TEST(test_master_install_ready_no_drift_keeps_identity_base_math) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef transfer; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXMasterFinalAckToken final_ack_token; + PcmXPhasePayload final_commit; + uint64 image_id; + const uint64 requester_session = UINT64_C(7283); + const uint64 source_session = UINT64_C(8283); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe_with_base(812, 0, 22, UINT64_C(8283), requester_session, 1, UINT64_C(5), + &admission); + drive_master_ticket_to_prepared(&admission, source_session, UINT64_C(9283), &transfer, + &image_id); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + + memset(&install_ready, 0, sizeof(install_ready)); + install_ready.ref = transfer; + install_ready.image_id = image_id; + install_ready.result = PCM_X_QUEUE_OK; + install_ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; + UT_ASSERT_EQ( + cluster_pcm_x_master_install_ready_exact(&install_ready, 0, 0, requester_session, &commit), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ticket->grant_base_own_generation, 0); + + memset(&final_ack, 0, sizeof(final_ack)); + final_ack.ref = transfer; + final_ack.image_id = image_id; + /* Monotonic but not exactly base+1: passes ingress sanity, refused by the + * locked exact check. */ + final_ack.committed_own_generation = UINT64_C(7); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session, + &final_ack_token), + PCM_X_QUEUE_STALE); + final_ack.committed_own_generation = UINT64_C(6); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session, + &final_ack_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_finalize_exact(&final_ack_token, &final_commit), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + UT_TEST(test_master_grant_generation_exhaustion_never_wraps) { PcmXShmemHeader *header; @@ -10067,6 +10336,201 @@ UT_TEST(test_local_cross_lane_retire_exemption_requires_distinct_ticket) UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); } +typedef struct TestLocalRebaseFixture { + PcmXLocalHandle writer; + PcmXLocalWriterClaim writer_claim; + PcmXAdmitAckPayload admit_ack; + PcmXGrantPayload prepare; + PcmXLocalTagSlot *tag_slot; + uint64 master_session; +} TestLocalRebaseFixture; + +/* Drive a local leader with a nonzero enqueue-time base (=5) through the + * production chain to the exact PREPARE_GRANT-applied, pre-INSTALL window + * where an interleaved revoke would have moved the own generation. */ +static void +prepare_local_rebase_fixture(BlockNumber block, uint64 master_session, + TestLocalRebaseFixture *fixture) +{ + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXWaitIdentity identity; + PcmXEnqueuePayload enqueue; + PcmXPhasePayload admit_confirm; + PcmXLocalCutoff cutoff; + PcmXLocalReliableToken token; + + UT_ASSERT_NOT_NULL(fixture); + memset(fixture, 0, sizeof(*fixture)); + fixture->master_session = master_session; + init_active_pcm_x(UINT64_C(77)); + bind_local_master(1, UINT64_C(9), master_session); + holder_key = make_local_holder_key(block, 0, 2, master_session + UINT64_C(1), 3); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + identity = make_wait_identity(block, 0, 3, master_session + UINT64_C(2)); + identity.base_own_generation = UINT64_C(5); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &fixture->writer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&fixture->writer, &fixture->writer_claim), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(fixture->writer_claim.grant_base_own_generation, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&fixture->writer, &cutoff), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&fixture->writer, &enqueue, &token), + PCM_X_QUEUE_OK); + memset(&fixture->admit_ack, 0, sizeof(fixture->admit_ack)); + fixture->admit_ack.ref.identity = identity; + fixture->admit_ack.ref.handle.ticket_id = master_session + UINT64_C(100); + fixture->admit_ack.ref.handle.queue_generation = UINT64_C(1); + fixture->admit_ack.prehandle = enqueue.prehandle; + fixture->admit_ack.result = PCM_X_QUEUE_OK; + fixture->admit_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&fixture->writer, &fixture->admit_ack, 1, + master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_admit_confirm_arm_exact(&fixture->writer, &admit_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&fixture->writer, &admit_confirm, 1, + master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact( + &fixture->admit_ack.ref, 1, master_session, &holder_copy, 1, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact( + &fixture->admit_ack.ref, 1, master_session, &holder_snapshot, &holder_copy, 1, + NULL, 0, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact( + &fixture->admit_ack.ref, blocker_snapshot.set_generation, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&holder), PCM_X_QUEUE_OK); + + memset(&fixture->prepare, 0, sizeof(fixture->prepare)); + fixture->prepare.ref = fixture->admit_ack.ref; + fixture->prepare.ref.grant_generation = UINT64_C(1); + UT_ASSERT(cluster_pcm_x_image_id_encode(1, master_session + UINT64_C(200), + &fixture->prepare.image.image_id)); + fixture->prepare.image.source_own_generation = UINT64_C(9); + fixture->prepare.image.page_scn = UINT64_C(10); + fixture->prepare.image.page_lsn = UINT64_C(11); + fixture->prepare.image.source_node = 2; + fixture->prepare.image.page_checksum = UINT32_C(12); + UT_ASSERT_EQ(cluster_pcm_x_local_prepare_grant_exact(&fixture->writer, &fixture->prepare, 1, + master_session), + PCM_X_QUEUE_OK); + fixture->tag_slot + = &local_tag_slots(ClusterPcmXConvertShmem)[fixture->writer.tag_slot.slot_index]; + UT_ASSERT_EQ(fixture->tag_slot->grant_base_own_generation, 0); +} + +/* + * A' rebase, local plane: the one-shot pre-reservation publication of the + * effective grant base and the exact final-ack math that follows it. The + * wire V2 field is emitted from the persisted value, never from a caller + * argument, and the enqueue-time math is refused once a rebase is live. + */ +UT_TEST(test_local_grant_rebase_publish_is_one_shot_and_effective) +{ + TestLocalRebaseFixture fixture; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + PcmXLocalReliableToken token; + PcmXDrainPollPayload poll; + PcmXRetirePayload retire; + PcmXLocalProgress progress; + const uint64 master_session = UINT64_C(1811); + + prepare_local_rebase_fixture(7120, master_session, &fixture); + + /* Shape and strictly-newer gates. */ + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, 0), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_MAX), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(4)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(5)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(fixture.tag_slot->grant_base_own_generation, 0); + + /* One-shot publish of a >1 drift (5 -> 8), then a same-value replay. */ + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(8)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(fixture.tag_slot->grant_base_own_generation, UINT64_C(8)); + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(8)), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&fixture.writer, &progress), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.grant_base_own_generation, UINT64_C(8)); + + /* The armed INSTALL_READY emits the persisted value on the V2 field. */ + UT_ASSERT_EQ(cluster_pcm_x_local_install_ready_arm_exact(&fixture.writer, &fixture.prepare.ref, + &fixture.prepare.image, &install_ready, + &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(install_ready.rebased_own_generation, UINT64_C(8)); + /* Once the leg is armed the publish window is closed. */ + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(8)), + PCM_X_QUEUE_BAD_STATE); + + memset(&commit, 0, sizeof(commit)); + commit.ref = fixture.prepare.ref; + commit.phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; + UT_ASSERT_EQ(cluster_pcm_x_local_commit_x_exact(&fixture.writer, &commit, 1, master_session), + PCM_X_QUEUE_OK); + + /* The enqueue-time math (5+1) is refused; the rebased math (8+1) arms. */ + UT_ASSERT_EQ( + cluster_pcm_x_local_final_ack_arm_exact(&fixture.writer, UINT64_C(6), &final_ack, &token), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ( + cluster_pcm_x_local_final_ack_arm_exact(&fixture.writer, UINT64_C(9), &final_ack, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(final_ack.committed_own_generation, UINT64_C(9)); + memset(&final_commit, 0, sizeof(final_commit)); + final_commit.ref = fixture.prepare.ref; + final_commit.phase = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; + UT_ASSERT_EQ(cluster_pcm_x_local_final_commit_ack_exact(&fixture.writer, &final_commit, 1, + master_session, &final_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&fixture.writer_claim), + PCM_X_QUEUE_OK); + memset(&poll, 0, sizeof(poll)); + poll.ref = fixture.prepare.ref; + poll.drain_generation = UINT64_C(43); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.prepare.ref.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = fixture.prepare.ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + assert_local_queue_baseline(ClusterPcmXConvertShmem); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +/* A second, different local rebase value is evidence divergence. */ +UT_TEST(test_local_grant_rebase_conflict_fails_closed) +{ + TestLocalRebaseFixture fixture; + const uint64 master_session = UINT64_C(1813); + + prepare_local_rebase_fixture(7121, master_session, &fixture); + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(8)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(9)), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(fixture.tag_slot->grant_base_own_generation, UINT64_C(8)); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + UT_TEST(test_local_cancelled_non_source_participant_gen0_drains_and_retires_exactly) { PcmXShmemHeader *header; @@ -14575,6 +15039,7 @@ main(void) UT_RUN(test_wire_abi_sizes_are_exact); UT_RUN(test_wire_abi_offsets_are_exact); UT_RUN(test_runtime_layout_abi_and_offsets_are_exact); + UT_RUN(test_runtime_rebase_wire_active_is_activation_bound); UT_RUN(test_lwlock_held_limit_is_shared_200); UT_RUN(test_default_capacity_formulas_are_exact); UT_RUN(test_exactly_five_pools_and_bounded_directories); @@ -14689,6 +15154,9 @@ main(void) UT_RUN(test_master_drive_bitmap_replace_rejects_reliable_leg_drift); UT_RUN(test_master_drive_snapshot_fails_closed_on_bitmap_corruption); UT_RUN(test_master_transfer_wire_49_56_is_generation_exact); + UT_RUN(test_master_install_ready_rebase_grant_base_is_one_shot); + UT_RUN(test_master_install_ready_rebase_conflict_fails_closed); + UT_RUN(test_master_install_ready_no_drift_keeps_identity_base_math); UT_RUN(test_master_grant_generation_exhaustion_never_wraps); UT_RUN(test_master_image_id_allocator_encodes_node31_and_never_wraps); UT_RUN(test_master_blocker_wire_stage_preserves_old_set_until_commit); @@ -14742,6 +15210,8 @@ main(void) UT_RUN(test_local_non_source_blocker_participant_drains_and_retires_exactly); UT_RUN(test_local_cross_lane_holder_terminal_retires_under_revoke_barrier); UT_RUN(test_local_cross_lane_retire_exemption_requires_distinct_ticket); + UT_RUN(test_local_grant_rebase_publish_is_one_shot_and_effective); + UT_RUN(test_local_grant_rebase_conflict_fails_closed); UT_RUN(test_local_cancelled_non_source_participant_gen0_drains_and_retires_exactly); UT_RUN(test_local_cancelled_participant_gen0_drain_requires_frozen_round); UT_RUN(test_local_holder_drain_validates_frozen_round_before_terminal_publish); From 03ada812caab9a650670a55a7c6be74d724ec84a Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 01:34:34 -0700 Subject: [PATCH 18/56] fix(cluster): unwind nested-guard BARRIER_CLOSED to the heap caller instead of a client ERROR t/400 L3 item 3. A writer holding a heap content lock whose tag sits under a frozen REVOKE_BARRIER hits the nested-wait guard when its pre-crit VM LockBuffer enters the PCM-X queue (heap_update -> LockBuffer(vm) -> fail_line 8555, captured live via backtrace_functions in t/400): the refusal escaped as an OBJECT_IN_USE ERROR, killing a pgbench client and violating the S3 errors=0 gate. The refusal is now caller-owned: - bufmgr: cluster_bufmgr_pcm_x_writer_prepare() grows a barrier_refused out-param; LockBuffer body becomes LockBufferInternal() and the new ClusterLockBufferExclusiveBarrierAware() returns false with nothing held on BARRIER_CLOSED (all other paths keep the historical error surface). bufmgr never releases the outer foreign content lock itself. - heapam: the reachable pre-crit VM sites unwind their OWN heap lock(s), resolve the map-page conversion via cluster_heap_vm_barrier_warm() while holding no content lock (blocking legal, node X retained by hold-until-revoked), then re-enter a proven point: heap_update requalifies at l2 when the old tuple carries no temporary lock (ITL pick idempotent, unstamped undo record unreachable), re-enters the stock page-reacquire loop when the lock-only xmax already shields it (foreign newbuf unlocked and unpinned so RelationGetBufferForTuple re-picks), and heap_delete requalifies at l1. Bounded by CLUSTER_HEAP_VM_BARRIER_MAX_RETRIES, after which the plain LockBuffer restores the fail-closed ERROR. heap_insert/heap_multi_insert/heap_lock_tuple keep the explicit fail-closed ERROR: their pre-crit VM shape is unreachable under the t/400 and S3 workloads (no all-visible insert targets without vacuum, no FOR UPDATE); follow-up may extend the same arm. - observability: pcm_x_queue_barrier_unwind_count (pg_cluster_state pcm row 60); PcmXStats grows 176->184, header 36512->36520, and PCM_X_SHMEM_LAYOUT_VERSION bumps 12->13 covering the A' slot growth that had landed without a version bump. TDD: source-scan pin test_precrit_vm_barrier_refusal_unwinds_to_caller RED (not ok 18) before the arms existed, GREEN after; full cluster_unit suite RC=0 with zero not-ok; check-format 0 violations. e2e t/400 errors=0 verification follows with the queued review-gap fixes. --- src/backend/access/heap/heapam.c | 193 +++++++++++++++++- src/backend/cluster/cluster_debug.c | 2 + src/backend/cluster/cluster_pcm_x_convert.c | 14 ++ src/backend/storage/buffer/bufmgr.c | 61 +++++- src/include/cluster/cluster_pcm_x_convert.h | 16 +- src/include/storage/bufmgr.h | 3 + src/test/cluster_tap/t/024_pcm_lock.pl | 6 +- .../cluster_tap/t/108_pcm_state_machine.pl | 4 +- src/test/cluster_unit/Makefile | 1 + src/test/cluster_unit/test_cluster_debug.c | 2 +- .../test_cluster_pcm_direct_init.c | 69 ++++++- src/test/cluster_unit/test_cluster_pcm_own.c | 14 +- .../cluster_unit/test_cluster_pcm_x_convert.c | 18 +- 13 files changed, 359 insertions(+), 44 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 9a6ffef55a..ebfdeab2eb 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -3438,6 +3438,31 @@ cluster_xwait_has_remote_evidence(Page page, HeapTupleHeader tuple, TransactionI } #endif /* USE_PGRAC_CLUSTER */ +/* + * PGRAC (t/400 L3 item 3): bound for the BARRIER_CLOSED caller-owned unwind + * at the pre-crit VM lock sites. The all-visible clear race is one-shot per + * page (whoever clears first removes the need for every later contender), so + * the retry converges; the bound only guards the no-node-cache configuration + * where the warmed conversion can be revoked before the re-attempt. Once + * exhausted, the plain LockBuffer restores the historical fail-closed ERROR. + */ +#define CLUSTER_HEAP_VM_BARRIER_MAX_RETRIES 16 + +/* + * PGRAC (t/400 L3 item 3): resolve a refused map-page conversion while the + * caller holds NO content lock. Blocking is legal here (the nested guard + * only refuses a wait entered while a frozen-tag content lock is held); + * hold-until-revoked retains the node X after the local release, so the + * caller's re-attempt is normally served by the cached-cover fast path + * without consulting the guard again. + */ +static void +cluster_heap_vm_barrier_warm(Buffer vmbuf) +{ + LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE); + LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK); +} + TM_Result heap_delete(Relation relation, ItemPointer tid, @@ -3453,6 +3478,7 @@ heap_delete(Relation relation, ItemPointer tid, Buffer buffer; Buffer vmbuffer = InvalidBuffer; bool vm_locked; /* PGRAC: pre-crit VM content lock held */ + int vm_barrier_retries = 0; /* PGRAC: t/400 L3 item 3 */ TransactionId new_xmax; uint16 new_infomask, new_infomask2; @@ -3855,7 +3881,25 @@ heap_delete(Relation relation, ItemPointer tid, vm_locked = false; if (PageIsAllVisible(page) && BufferIsValid(vmbuffer)) { - LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); + if (vm_barrier_retries >= CLUSTER_HEAP_VM_BARRIER_MAX_RETRIES) + LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); + else if (!ClusterLockBufferExclusiveBarrierAware(vmbuffer)) + { + /* + * PGRAC: vm barrier unwind (delete requalify) — this backend + * holds the heap page whose tag sits under a frozen revoke + * barrier, so it may not sleep on the map-page conversion. + * Release our own lock, resolve the conversion unlocked, and + * requalify from scratch: the ITL slot pick is idempotent and an + * unstamped undo record is unreachable garbage, exactly as after + * any pre-crit ERROR. + */ + vm_barrier_retries++; + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + cluster_heap_vm_barrier_warm(vmbuffer); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + goto l1; + } vm_locked = true; } @@ -4147,6 +4191,8 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, vmbuffer_new = InvalidBuffer; bool vm_locked; /* PGRAC: pre-crit VM content lock held */ bool vm_locked_new; + bool old_tuple_temp_locked = false; /* PGRAC: t/400 L3 item 3 */ + int vm_barrier_retries = 0; /* PGRAC: t/400 L3 item 3 */ bool need_toast; Size newtupsize, pagefree; @@ -4816,7 +4862,22 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, vm_locked = false; if (PageIsAllVisible(page) && BufferIsValid(vmbuffer)) { - LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); + if (vm_barrier_retries >= CLUSTER_HEAP_VM_BARRIER_MAX_RETRIES) + LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); + else if (!ClusterLockBufferExclusiveBarrierAware(vmbuffer)) + { + /* + * PGRAC: vm barrier unwind (update pre-toast) — the old + * tuple carries no temporary lock yet and nothing + * irreversible has happened in this pass, so a full + * requalify is required and sufficient. + */ + vm_barrier_retries++; + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + cluster_heap_vm_barrier_warm(vmbuffer); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + goto l2; + } vm_locked = true; } @@ -4870,6 +4931,11 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, END_CRIT_SECTION(); + /* PGRAC (t/400 L3 item 3): the lock-only xmax now shields the old + * tuple from concurrent updaters, so later unlocked windows (the + * page-reacquire loop below) do not require requalification. */ + old_tuple_temp_locked = true; + if (vm_locked) LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK); @@ -4914,6 +4980,16 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, * pins; but if we don't, we have to handle that here. Hence we need * a loop. */ + /* + * PGRAC (t/400 L3 item 3): the VM-barrier unwind re-enters here with + * both content locks released (and a foreign newbuf unpinned) after + * resolving the refused map-page conversion. The lock-only xmax + * stamped above makes requalification unnecessary, exactly as for + * the stock unlocked TOAST window; everything below — page choice, + * serializable-conflict check, HOT decision, replica identity, ITL + * pick and undo emit — is re-derived on the way back down. + */ +l_pgrac_reacquire: for (;;) { if (newtupsize > pagefree) @@ -5140,6 +5216,8 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, bool need_new = newbuf != buffer && PageIsAllVisible(BufferGetPage(newbuf)) && BufferIsValid(vmbuffer_new); + bool barrier_aware = vm_barrier_retries < CLUSTER_HEAP_VM_BARRIER_MAX_RETRIES; + Buffer refused_vm = InvalidBuffer; if (need_old && need_new && vmbuffer_new == vmbuffer) need_new = false; /* same map page: one lock covers both */ @@ -5147,24 +5225,117 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, if (need_old && need_new && vmbuffer_new < vmbuffer) { - LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE); - vm_locked_new = true; - LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); - vm_locked = true; + if (!barrier_aware) + { + LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE); + vm_locked_new = true; + LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); + vm_locked = true; + } + else + { + if (ClusterLockBufferExclusiveBarrierAware(vmbuffer_new)) + vm_locked_new = true; + else + refused_vm = vmbuffer_new; + if (refused_vm == InvalidBuffer) + { + if (ClusterLockBufferExclusiveBarrierAware(vmbuffer)) + vm_locked = true; + else + refused_vm = vmbuffer; + } + } } else { if (need_old) { - LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); - vm_locked = true; + if (!barrier_aware) + { + LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); + vm_locked = true; + } + else if (ClusterLockBufferExclusiveBarrierAware(vmbuffer)) + vm_locked = true; + else + refused_vm = vmbuffer; } - if (need_new) + if (need_new && refused_vm == InvalidBuffer) { - LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE); - vm_locked_new = true; + if (!barrier_aware) + { + LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE); + vm_locked_new = true; + } + else if (ClusterLockBufferExclusiveBarrierAware(vmbuffer_new)) + vm_locked_new = true; + else + refused_vm = vmbuffer_new; } } + + if (refused_vm != InvalidBuffer) + { + /* + * PGRAC: BARRIER_CLOSED caller-owned unwind. This backend holds + * heap content lock(s) whose tag sits under a frozen revoke + * barrier; sleeping on the map-page conversion here could + * deadlock across locks, and bufmgr must never release the + * outer lock itself. Release our OWN locks, resolve the + * conversion while holding no content lock, then re-enter a + * proven point. + */ + vm_barrier_retries++; + if (vm_locked_new) + { + LockBuffer(vmbuffer_new, BUFFER_LOCK_UNLOCK); + vm_locked_new = false; + } + if (vm_locked) + { + LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK); + vm_locked = false; + } + if (old_key_tuple != NULL && old_key_copied) + { + heap_freetuple(old_key_tuple); + old_key_tuple = NULL; + old_key_copied = false; + } + if (!old_tuple_temp_locked) + { + /* + * PGRAC: vm barrier unwind (update requalify) — no + * temporary lock protects the old tuple yet (same-page + * non-TOAST path, so newbuf == buffer), hence a concurrent + * updater may win the race while we are unlocked: requalify + * from scratch. ITL pick and undo emit of this pass are + * abandoned safely (idempotent pick, unreachable record). + */ + Assert(newbuf == buffer); + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + cluster_heap_vm_barrier_warm(refused_vm); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + goto l2; + } + + /* + * PGRAC: vm barrier unwind (update reacquire) — the lock-only + * xmax shields the old tuple, so re-entering the page-reacquire + * loop (stock unlocked-TOAST idiom) is sufficient. A distinct + * newbuf is unlocked AND unpinned: the loop re-picks the target + * page through RelationGetBufferForTuple. + */ + if (newbuf != buffer) + { + LockBuffer(newbuf, BUFFER_LOCK_UNLOCK); + ReleaseBuffer(newbuf); + } + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + cluster_heap_vm_barrier_warm(refused_vm); + goto l_pgrac_reacquire; + } } /* NO EREPORT(ERROR) from here till changes are logged */ diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index a73b0aa22b..1e86fe5d8d 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -2088,6 +2088,8 @@ dump_pcm(ReturnSetInfo *rsinfo) emit_row(rsinfo, "pcm", "pcm_x_own_busy_count", fmt_int64((int64)stats.own_busy_count)); emit_row(rsinfo, "pcm", "pcm_x_own_corrupt_count", fmt_int64((int64)stats.own_corrupt_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_barrier_unwind_count", + fmt_int64((int64)stats.barrier_unwind_count)); } } diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index 3003df5aa6..449af318d7 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -838,6 +838,7 @@ pcm_x_init_stats(PcmXStats *stats) pg_atomic_init_u64(&stats->own_abort_count, 0); pg_atomic_init_u64(&stats->own_busy_count, 0); pg_atomic_init_u64(&stats->own_corrupt_count, 0); + pg_atomic_init_u64(&stats->barrier_unwind_count, 0); } @@ -940,6 +941,7 @@ cluster_pcm_x_stats_snapshot(PcmXStatsSnapshot *snapshot_out) snapshot_out->own_abort_count = pg_atomic_read_u64(&header->stats.own_abort_count); snapshot_out->own_busy_count = pg_atomic_read_u64(&header->stats.own_busy_count); snapshot_out->own_corrupt_count = pg_atomic_read_u64(&header->stats.own_corrupt_count); + snapshot_out->barrier_unwind_count = pg_atomic_read_u64(&header->stats.barrier_unwind_count); snapshot_out->active_tags = (uint64)header->allocator[PCM_X_ALLOC_MASTER_TAG].used; snapshot_out->live_tickets = (uint64)header->allocator[PCM_X_ALLOC_MASTER_TICKET].used; snapshot_out->local_retire_gate = (uint64)pg_atomic_read_u32(&header->local_retire_gate); @@ -1053,6 +1055,18 @@ cluster_pcm_x_stats_note_own_corrupt(void) } +/* t/400 L3 item 3: a BARRIER_CLOSED refusal consumed by a barrier-aware + * LockBuffer caller (unwound in place of a client ERROR). */ +void +cluster_pcm_x_stats_note_barrier_unwind(void) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (header != NULL) + pcm_x_stats_increment(&header->stats.barrier_unwind_count); +} + + static void pcm_x_runtime_fail_closed_impl(const char *site_file, int site_line) { diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 3a06cd8541..7b2275c4ac 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -1877,7 +1877,7 @@ cluster_bufmgr_pcm_x_writer_drain_deferred_nowait(void) } static ClusterPcmXWriterLedgerEntry * -cluster_bufmgr_pcm_x_writer_prepare(BufferDesc *buf, PcmLockMode mode) +cluster_bufmgr_pcm_x_writer_prepare(BufferDesc *buf, PcmLockMode mode, bool *barrier_refused) { ClusterPcmXWriterLedgerEntry *entry; ClusterPcmDirectInitSnapshot observed; @@ -1932,6 +1932,21 @@ cluster_bufmgr_pcm_x_writer_prepare(BufferDesc *buf, PcmLockMode mode) "failed claim handoff"); } cluster_bufmgr_pcm_x_writer_clear(entry); + + /* + * PGRAC (t/400 L3 item 3): a nested-guard BARRIER_CLOSED means this + * backend already holds another content lock whose tag sits under a + * frozen revoke barrier; the requester unwound cleanly (no claim, no + * wait identity). A barrier-aware caller owns the unwind: hand the + * refusal back instead of escalating to a client ERROR. bufmgr must + * never release the foreign content lock itself. + */ + if (result == PCM_X_QUEUE_BARRIER_CLOSED && barrier_refused != NULL) + { + *barrier_refused = true; + cluster_pcm_x_stats_note_barrier_unwind(); + return NULL; + } cluster_bufmgr_pcm_x_writer_report_failure(result, buf, "queue acquire"); } if (!entry->claim_handed_off) { @@ -7927,9 +7942,16 @@ UnlockBuffers(void) * PGRAC: spec-2.31 wires the PCM state machine as the outer lock for * shared-buffer content locks. PCM acquisition happens before the local * content_lock; release happens after the local content_lock is released. + * + * PGRAC (t/400 L3 item 3): pcm_barrier_refused, when non-NULL, arms the + * barrier-aware mode used by ClusterLockBufferExclusiveBarrierAware(): a + * nested-guard BARRIER_CLOSED from the PCM-X queue acquire sets the flag and + * returns WITHOUT taking the content lock, leaving the unwind to the caller + * that owns the outer (frozen-tag) content lock. A NULL pointer keeps the + * historical behavior (the refusal escalates to a client ERROR). */ -void -LockBuffer(Buffer buffer, int mode) +static void +LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) { BufferDesc *buf; #ifdef USE_PGRAC_CLUSTER @@ -7982,7 +8004,10 @@ LockBuffer(Buffer buffer, int mode) } if (!pcm_covered) - pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode); + pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode, + pcm_barrier_refused); + if (pcm_barrier_refused != NULL && *pcm_barrier_refused) + return; /* barrier-aware caller owns the unwind */ if (pcm_x_writer == NULL && !pcm_covered) { /* * PGRAC: spec-4.7a D2 — hold-until-revoked acquire gate. @@ -8339,6 +8364,34 @@ LockBuffer(Buffer buffer, int mode) #endif } +void +LockBuffer(Buffer buffer, int mode) +{ + LockBufferInternal(buffer, mode, NULL); +} + +/* + * PGRAC (t/400 L3 item 3): EXCLUSIVE LockBuffer that reports a nested-guard + * BARRIER_CLOSED refusal instead of raising a client ERROR. + * + * Returns true with the content lock held (every non-barrier path behaves + * exactly like LockBuffer, including its error surface). Returns false with + * NOTHING held when the PCM-X queue acquire refused because this backend + * already holds another content lock whose tag sits under a frozen revoke + * barrier: sleeping here could deadlock across locks and the frozen barrier + * snapshot cannot see the nested edge. The caller — the owner of that outer + * content lock — must release its own lock(s), resolve the map-page + * conversion while holding no content lock, and re-enter a requalify point. + */ +bool +ClusterLockBufferExclusiveBarrierAware(Buffer buffer) +{ + bool barrier_refused = false; + + LockBufferInternal(buffer, BUFFER_LOCK_EXCLUSIVE, &barrier_refused); + return !barrier_refused; +} + /* VM/FSM pages read with RBM_ZERO_ON_ERROR become BM_VALID before their * PageIsNew initialization. Dedicated wrappers are the provenance: a plain * LockBuffer(X) on the same N buffer remains queue-only. */ diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h index e10030bc8f..ff7b7a9c18 100644 --- a/src/include/cluster/cluster_pcm_x_convert.h +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -32,7 +32,8 @@ #define PCM_X_PROTOCOL_NODE_LIMIT 32 #define PCM_X_SHMEM_REGION_NAME "pgrac cluster pcm convert queue" #define PCM_X_SHMEM_MAGIC ((uint32)0x50435851) /* "PCXQ" */ -#define PCM_X_SHMEM_LAYOUT_VERSION ((uint32)12) +/* 13: A' rebase slot growth (ticket 392 / local tag 760 / header stats). */ +#define PCM_X_SHMEM_LAYOUT_VERSION ((uint32)13) #define PCM_X_INVALID_SLOT_INDEX ((Size) - 1) #define PCM_X_LOCK_PARTITIONS NUM_BUFFER_PARTITIONS #define PCM_X_LWLOCK_COUNT (1 + 2 * PCM_X_LOCK_PARTITIONS) @@ -1153,6 +1154,9 @@ typedef struct PcmXStats { pg_atomic_uint64 own_abort_count; pg_atomic_uint64 own_busy_count; pg_atomic_uint64 own_corrupt_count; + /* t/400 L3 item 3: BARRIER_CLOSED refusals handed back to a + * barrier-aware LockBuffer caller instead of escaping as ERROR. */ + pg_atomic_uint64 barrier_unwind_count; } PcmXStats; /* Process-local copy used by debug views and acceptance gates. */ @@ -1188,6 +1192,7 @@ typedef struct PcmXStatsSnapshot { uint64 own_abort_count; uint64 own_busy_count; uint64 own_corrupt_count; + uint64 barrier_unwind_count; } PcmXStatsSnapshot; /* Optional peer authority installed while the runtime gate is ACTIVATING. */ @@ -1272,8 +1277,8 @@ typedef struct PcmXShmemHeader { StaticAssertDecl(sizeof(PcmXShmemLayout) == 440, "PCM-X shmem layout ABI"); StaticAssertDecl(sizeof(PcmXAllocatorState) == 32, "PCM-X allocator state ABI"); StaticAssertDecl(sizeof(PcmXPeerFrontier) == 48, "PCM-X peer frontier ABI"); -StaticAssertDecl(sizeof(PcmXStats) == 176, "PCM-X stats ABI"); -StaticAssertDecl(sizeof(PcmXStatsSnapshot) == 224, "PCM-X stats snapshot ABI"); +StaticAssertDecl(sizeof(PcmXStats) == 184, "PCM-X stats ABI"); +StaticAssertDecl(sizeof(PcmXStatsSnapshot) == 232, "PCM-X stats snapshot ABI"); StaticAssertDecl(sizeof(PcmXPeerBinding) == 16, "PCM-X peer binding ABI"); StaticAssertDecl(sizeof(PcmXOutboundTargetFrontier) == 32, "PCM-X outbound target frontier ABI"); StaticAssertDecl(offsetof(PcmXOutboundTargetFrontier, mint_gate) == 0, @@ -1291,9 +1296,9 @@ StaticAssertDecl(offsetof(PcmXShmemHeader, local_locks) == 17280, "PCM-X local l StaticAssertDecl(offsetof(PcmXShmemHeader, peer_frontiers) == 33664, "PCM-X peer frontier array offset"); StaticAssertDecl(offsetof(PcmXShmemHeader, stats) == 35200, "PCM-X stats offset"); -StaticAssertDecl(offsetof(PcmXShmemHeader, outbound_targets) == 35376, +StaticAssertDecl(offsetof(PcmXShmemHeader, outbound_targets) == 35384, "PCM-X outbound target frontier array offset"); -StaticAssertDecl(sizeof(PcmXShmemHeader) == 36512, "PCM-X shmem header ABI"); +StaticAssertDecl(sizeof(PcmXShmemHeader) == 36520, "PCM-X shmem header ABI"); typedef enum PcmXAttachResult { PCM_X_ATTACH_OK = 0, @@ -1343,6 +1348,7 @@ extern void cluster_pcm_x_stats_note_own_commit(void); extern void cluster_pcm_x_stats_note_own_abort(void); extern void cluster_pcm_x_stats_note_own_busy(void); extern void cluster_pcm_x_stats_note_own_corrupt(void); +extern void cluster_pcm_x_stats_note_barrier_unwind(void); extern bool cluster_pcm_x_runtime_activate(uint64 master_session_incarnation); /* A' rebase: publish formation-wide PCM_X_REBASE_V1 coverage. Legal only * from the activating formation tick, strictly before activate_bound(). */ diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index 2a0a63e2e7..6fecd6fbf2 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -243,6 +243,9 @@ extern void MarkBufferDirtyHint(Buffer buffer, bool buffer_std); extern void UnlockBuffers(void); extern void LockBuffer(Buffer buffer, int mode); +/* PGRAC: EXCLUSIVE lock that hands a nested-guard BARRIER_CLOSED refusal + * back to the caller (false, nothing held) instead of raising an ERROR. */ +extern bool ClusterLockBufferExclusiveBarrierAware(Buffer buffer); /* PGRAC: operation-scoped PCM-X direct-init entrances for zero VM/FSM pages. */ extern void LockBufferForVisibilityMapPageInit(Buffer buffer); extern void LockBufferForFreeSpaceMapPageInit(Buffer buffer); diff --git a/src/test/cluster_tap/t/024_pcm_lock.pl b/src/test/cluster_tap/t/024_pcm_lock.pl index 01adc6236c..1a26f2b098 100644 --- a/src/test/cluster_tap/t/024_pcm_lock.pl +++ b/src/test/cluster_tap/t/024_pcm_lock.pl @@ -55,14 +55,14 @@ # ---------- -# L1: pg_cluster_state.pcm category has 59 keys, including the 31-key PCM-X FIFO surface. +# L1: pg_cluster_state.pcm category has 60 keys, including the 32-key PCM-X FIFO surface. # activates the state-machine diagnostics. # ---------- is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='pcm'}), - '59', - 'L1 pg_cluster_state.pcm category has 59 keys (existing 28 + PCM-X FIFO/ownership/runtime 30 + fail-closed site)'); + '60', + 'L1 pg_cluster_state.pcm category has 60 keys (existing 28 + PCM-X FIFO/ownership/runtime 31 + fail-closed site)'); # ---------- diff --git a/src/test/cluster_tap/t/108_pcm_state_machine.pl b/src/test/cluster_tap/t/108_pcm_state_machine.pl index 9f7241696c..e0a885f1ae 100644 --- a/src/test/cluster_tap/t/108_pcm_state_machine.pl +++ b/src/test/cluster_tap/t/108_pcm_state_machine.pl @@ -48,8 +48,8 @@ my $pcm_category_rows = $node_default->safe_psql( 'postgres', "SELECT count(*) FROM pg_cluster_state WHERE category = 'pcm'"); -is($pcm_category_rows, '59', - 'L1 pg_cluster_state pcm category has 59 rows (existing 28 + PCM-X FIFO/ownership/runtime 30 + fail-closed site)'); +is($pcm_category_rows, '60', + 'L1 pg_cluster_state pcm category has 60 rows (existing 28 + PCM-X FIFO/ownership/runtime 31 + fail-closed site)'); # L3 — api_state shows "active" when GUC=-1 default my $api_state_default = $node_default->safe_psql( diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 5ffd59af30..f9f6c1f97d 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -1378,6 +1378,7 @@ test_cluster_pcm_direct_init: test_cluster_pcm_direct_init.c unit_test.h \ -DBUFMGR_SOURCE_PATH='"$(top_srcdir)/src/backend/storage/buffer/bufmgr.c"' \ -DVM_SOURCE_PATH='"$(top_srcdir)/src/backend/access/heap/visibilitymap.c"' \ -DFSM_SOURCE_PATH='"$(top_srcdir)/src/backend/storage/freespace/freespace.c"' \ + -DHEAPAM_SOURCE_PATH='"$(top_srcdir)/src/backend/access/heap/heapam.c"' \ $< \ $(CLUSTER_PCM_DIRECT_INIT_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 486e1d082f..2d0a59d3d6 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -4328,7 +4328,7 @@ UT_TEST(test_debug_dump_exposes_exact_pcm_x_lmd_and_gcs_key_sets) fcinfo->resultinfo = (fmNodePtr)&rsinfo; (void)cluster_dump_state(fcinfo); - UT_ASSERT_EQ(captured_dump_count("pcm", NULL), 59); + UT_ASSERT_EQ(captured_dump_count("pcm", NULL), 60); UT_ASSERT_EQ(captured_dump_count("lmd", NULL), 51); UT_ASSERT_EQ(captured_dump_count("gcs", NULL), 119); for (i = 0; i < (int)lengthof(pcm_keys); i++) diff --git a/src/test/cluster_unit/test_cluster_pcm_direct_init.c b/src/test/cluster_unit/test_cluster_pcm_direct_init.c index 2eeb07bec7..61b826f32e 100644 --- a/src/test/cluster_unit/test_cluster_pcm_direct_init.c +++ b/src/test/cluster_unit/test_cluster_pcm_direct_init.c @@ -23,6 +23,9 @@ UT_DEFINE_GLOBALS(); #ifndef FSM_SOURCE_PATH #error "FSM_SOURCE_PATH must identify production freespace.c" #endif +#ifndef HEAPAM_SOURCE_PATH +#error "HEAPAM_SOURCE_PATH must identify production heapam.c" +#endif void ExceptionalCondition(const char *conditionName pg_attribute_unused(), @@ -369,7 +372,7 @@ UT_TEST(test_valid_n_s_x_without_proof_enters_queue_before_legacy_wire) { char *source = read_source(BUFMGR_SOURCE_PATH); static const char *const order[] - = { "pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode)", + = { "pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode,", "cluster_pcm_own_begin_grant_reservation(buf, &pcm_pending_base", "cluster_pcm_lock_acquire_buffer(buf, pcm_mode)" }; @@ -409,10 +412,71 @@ UT_TEST(test_wire_throw_exact_aborts_reservation_before_rethrow) } } +/* t/400 L3 item 3 — a nested-guard BARRIER_CLOSED at the pre-crit VM lock + * must unwind to the caller that owns the outer heap content lock instead of + * escaping as a client ERROR. bufmgr exposes the refusal (never releasing + * the foreign lock itself); heapam releases its own lock(s), warms the map + * page's node X while holding no content lock, and re-enters a proven + * requalify/reacquire point. */ +UT_TEST(test_precrit_vm_barrier_refusal_unwinds_to_caller) +{ + char *bufmgr = read_source(BUFMGR_SOURCE_PATH); + char *heapam = read_source(HEAPAM_SOURCE_PATH); + + UT_ASSERT(bufmgr != NULL); + UT_ASSERT(heapam != NULL); + if (bufmgr != NULL) { + /* The refusal arm sits between the queue acquire and the ERROR + * report, and only the barrier-aware entry can consume it. */ + static const char *const refusal_order[] + = { "cluster_gcs_pcm_x_acquire_writer(buf, &entry->claim", + "PCM_X_QUEUE_BARRIER_CLOSED && barrier_refused != NULL", + "cluster_bufmgr_pcm_x_writer_report_failure(result, buf, \"queue acquire\")" }; + + UT_ASSERT(strstr(bufmgr, "ClusterLockBufferExclusiveBarrierAware(Buffer buffer)") != NULL); + assert_ordered(bufmgr, refusal_order, lengthof(refusal_order)); + free(bufmgr); + } + if (heapam != NULL) { + static const char *const pretoast_order[] + = { "PGRAC: vm barrier unwind (update pre-toast)", + "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", "cluster_heap_vm_barrier_warm", + "LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE)", "goto l2;" }; + static const char *const requalify_order[] + = { "PGRAC: vm barrier unwind (update requalify)", + "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", "cluster_heap_vm_barrier_warm", + "LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE)", "goto l2;" }; + static const char *const reacquire_order[] + = { "PGRAC: vm barrier unwind (update reacquire)", + "LockBuffer(newbuf, BUFFER_LOCK_UNLOCK)", + "ReleaseBuffer(newbuf)", + "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", + "cluster_heap_vm_barrier_warm", + "goto l_pgrac_reacquire;" }; + static const char *const delete_order[] + = { "PGRAC: vm barrier unwind (delete requalify)", + "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", "cluster_heap_vm_barrier_warm", + "LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE)", "goto l1;" }; + + /* The warm helper itself may only run with no content lock held: + * it must take and drop the map-page lock, nothing else. */ + static const char *const warm_order[] = { "cluster_heap_vm_barrier_warm(Buffer vmbuf)", + "LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE)", + "LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK)" }; + + assert_ordered(heapam, warm_order, lengthof(warm_order)); + assert_ordered(heapam, pretoast_order, lengthof(pretoast_order)); + assert_ordered(heapam, requalify_order, lengthof(requalify_order)); + assert_ordered(heapam, reacquire_order, lengthof(reacquire_order)); + assert_ordered(heapam, delete_order, lengthof(delete_order)); + free(heapam); + } +} + int main(void) { - UT_PLAN(17); + UT_PLAN(18); UT_RUN(test_valid_read_miss_proof); UT_RUN(test_valid_extend_proof); UT_RUN(test_valid_vm_and_fsm_proofs); @@ -430,6 +494,7 @@ main(void) UT_RUN(test_valid_n_s_x_without_proof_enters_queue_before_legacy_wire); UT_RUN(test_direct_init_one_shot_image_cannot_return_without_x); UT_RUN(test_wire_throw_exact_aborts_reservation_before_rethrow); + UT_RUN(test_precrit_vm_barrier_refusal_unwinds_to_caller); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index 88358572c5..bcc2b935e0 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -740,7 +740,7 @@ UT_TEST(test_lockbuffer_content_error_uses_post_master_rollback_contract) assert_ordered_in_function(source, "\ncluster_pcm_own_rollback_grant_after_error_and_rethrow(", "\nstatic ", rethrow_contract, lengthof(rethrow_contract)); assert_ordered_in_function( - source, "\nLockBuffer(Buffer buffer, int mode)", + source, "\nLockBufferInternal(Buffer buffer, int mode", "\n/*\n * Acquire the content_lock for the buffer, but only if we don't have to wait.", content_error_contract, lengthof(content_error_contract)); free(source); @@ -1161,7 +1161,7 @@ UT_TEST(test_retained_image_release_and_writeback_gates_are_exact) flush = strstr(source, "\nFlushBuffer("); dirty = strstr(source, "\nMarkBufferDirty(Buffer buffer)"); hint = strstr(source, "\nMarkBufferDirtyHint(Buffer buffer, bool buffer_std)"); - lockbuffer = strstr(source, "\nLockBuffer(Buffer buffer, int mode)"); + lockbuffer = strstr(source, "\nLockBufferInternal(Buffer buffer, int mode"); conditional = strstr(source, "\nConditionalLockBuffer(Buffer buffer)"); resident_stamp = strstr(source, "\ncluster_bufmgr_lock_resident_for_stamp("); storage_refresh = strstr(source, "\ncluster_bufmgr_refresh_block_from_storage_for_gcs("); @@ -1327,11 +1327,11 @@ UT_TEST(test_lockbuffer_pcm_x_holder_ledger_brackets_both_content_acquires) * published across the W1 release/reacquire fallback. Normal release * publishes RELEASING before unlocking and unlinks only afterwards. */ assert_ordered_in_function( - source, "\nLockBuffer(Buffer buffer, int mode)", + source, "\nLockBufferInternal(Buffer buffer, int mode", "\n/*\n * Acquire the content_lock for the buffer, but only if we don't have to wait.", unlock_contract, lengthof(unlock_contract)); assert_ordered_in_function( - source, "\nLockBuffer(Buffer buffer, int mode)", + source, "\nLockBufferInternal(Buffer buffer, int mode", "\n/*\n * Acquire the content_lock for the buffer, but only if we don't have to wait.", acquire_contract, lengthof(acquire_contract)); free(source); @@ -1872,7 +1872,7 @@ UT_TEST(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_aut "cluster_pcm_lock_unlock_content_buffer(buf, old_mode)" }; static const char *const acquire_contract[] = { "pcm_covered = cluster_pcm_x_cached_cover_bypasses_queue(", - "pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode)", + "pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode,", "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf)", "LWLockAcquire(BufferDescriptorGetContentLock(buf)", "cluster_bufmgr_pcm_x_writer_activate(pcm_x_writer)" }; @@ -1977,11 +1977,11 @@ UT_TEST(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_aut writer_holder_runtime_contract, lengthof(writer_holder_runtime_contract)); assert_ordered_in_function( - source, "\nLockBuffer(Buffer buffer, int mode)", + source, "\nLockBufferInternal(Buffer buffer, int mode", "\n/*\n * Acquire the content_lock for the buffer, but only if we don't have to wait.", acquire_contract, lengthof(acquire_contract)); assert_ordered_in_function( - source, "\nLockBuffer(Buffer buffer, int mode)", + source, "\nLockBufferInternal(Buffer buffer, int mode", "\n/*\n * Acquire the content_lock for the buffer, but only if we don't have to wait.", unlock_contract, lengthof(unlock_contract)); assert_ordered_in_function(source, "\nUnlockBuffers(void)", diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 23fdf2e045..aff688a5f1 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -2169,13 +2169,13 @@ UT_TEST(test_wire_abi_offsets_are_exact) UT_TEST(test_runtime_layout_abi_and_offsets_are_exact) { - UT_ASSERT_EQ(PCM_X_SHMEM_LAYOUT_VERSION, 12); + UT_ASSERT_EQ(PCM_X_SHMEM_LAYOUT_VERSION, 13); UT_ASSERT_EQ(PCM_X_LOCK_PARTITIONS, NUM_BUFFER_PARTITIONS); UT_ASSERT_EQ(PCM_X_LWLOCK_COUNT, 257); UT_ASSERT_EQ(sizeof(PcmXShmemLayout), 440); UT_ASSERT_EQ(sizeof(PcmXAllocatorState), 32); - UT_ASSERT_EQ(sizeof(PcmXStats), 176); - UT_ASSERT_EQ(sizeof(PcmXStatsSnapshot), 224); + UT_ASSERT_EQ(sizeof(PcmXStats), 184); + UT_ASSERT_EQ(sizeof(PcmXStatsSnapshot), 232); UT_ASSERT_EQ(sizeof(PcmXSlotHeader), 24); UT_ASSERT_EQ(offsetof(PcmXSlotHeader, next_free), 0); UT_ASSERT_EQ(offsetof(PcmXSlotHeader, generation_change_seq), 8); @@ -2237,8 +2237,8 @@ UT_TEST(test_runtime_layout_abi_and_offsets_are_exact) UT_ASSERT_EQ(offsetof(PcmXShmemHeader, local_locks), 17280); UT_ASSERT_EQ(offsetof(PcmXShmemHeader, peer_frontiers), 33664); UT_ASSERT_EQ(offsetof(PcmXShmemHeader, stats), 35200); - UT_ASSERT_EQ(offsetof(PcmXShmemHeader, outbound_targets), 35376); - UT_ASSERT_EQ(sizeof(PcmXShmemHeader), 36512); + UT_ASSERT_EQ(offsetof(PcmXShmemHeader, outbound_targets), 35384); + UT_ASSERT_EQ(sizeof(PcmXShmemHeader), 36520); } UT_TEST(test_lwlock_held_limit_is_shared_200) @@ -2286,19 +2286,19 @@ UT_TEST(test_exactly_five_pools_and_bounded_directories) UT_ASSERT_EQ(layout.local_holder.directory_capacity, layout.local_holder.capacity * 2); } -UT_TEST(test_layout_v12_records_transfer_and_terminal_frontiers) +UT_TEST(test_layout_v13_records_transfer_and_terminal_frontiers) { PcmXShmemLayout layout; cluster_pcm_x_layout_compute(122, 25, 16384, 1024, &layout); - UT_ASSERT_EQ(layout.version, 12); + UT_ASSERT_EQ(layout.version, 13); UT_ASSERT_EQ(layout.lock_partition_count, PCM_X_LOCK_PARTITIONS); UT_ASSERT_EQ(layout.lwlock_count, PCM_X_LWLOCK_COUNT); UT_ASSERT_EQ(sizeof(PcmXPeerFrontier), 48); UT_ASSERT_EQ(sizeof(PcmXOutboundTargetFrontier), 32); UT_ASSERT_EQ(offsetof(PcmXShmemHeader, peer_frontiers), 33664); UT_ASSERT_EQ(offsetof(PcmXShmemHeader, stats), 35200); - UT_ASSERT_EQ(offsetof(PcmXShmemHeader, outbound_targets), 35376); + UT_ASSERT_EQ(offsetof(PcmXShmemHeader, outbound_targets), 35384); } UT_TEST(test_offsets_are_aligned_ordered_and_bounded) @@ -15043,7 +15043,7 @@ main(void) UT_RUN(test_lwlock_held_limit_is_shared_200); UT_RUN(test_default_capacity_formulas_are_exact); UT_RUN(test_exactly_five_pools_and_bounded_directories); - UT_RUN(test_layout_v12_records_transfer_and_terminal_frontiers); + UT_RUN(test_layout_v13_records_transfer_and_terminal_frontiers); UT_RUN(test_offsets_are_aligned_ordered_and_bounded); UT_RUN(test_membership_wait_and_holder_partitions_do_not_overlap); UT_RUN(test_generation_zero_advances_without_being_a_sentinel); From fbd7f8be24fa3ca6b02c9570f363e5085d2b80a9 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 01:40:33 -0700 Subject: [PATCH 19/56] fix(cluster): inherit the published effective grant base into every same-round writer claim t/400 L3 review P0-1 + P1-1. cluster_pcm_x_local_writer_claim_exact() unconditionally zeroed claim_out->grant_base_own_generation, so only the leader (whose claim is backfilled after the preflight rebase publish in the gcs requester) carried the effective base. A same-round FIFO follower claiming after the leader's one-shot rebase kept its enqueue-time identity base and the bufmgr grant-snapshot proof then verified the rebased X against stale base+1 -- a deterministic fail-closed ERROR for every follower of a rebased round (direct hit under multi-client-per-node load). The claim now inherits tag_slot->grant_base_own_generation under the same partition lock; zero (no rebase live) keeps the enqueue-time math bit for bit. TDD: test_local_follower_claim_inherits_published_rebase drives the production chain leader join->claim->same-round follower join (pre-cutoff) ->rebase publish (5->8)->leader INSTALL/COMMIT/FINAL on 8+1->claim release ->follower claim. RED on the pre-fix code: claim OK but inherited base 0 != 8 and cluster_pcm_x_writer_grant_snapshot_exact refused rebased+1; GREEN after (257/257). Also repairs the unit TAP plans that had drifted from the executed test count (review P1-1): pcm_x_convert 247->257, gcs_block 76->78 (78/78). --- src/backend/cluster/cluster_pcm_x_convert.c | 6 +- .../cluster_unit/test_cluster_gcs_block.c | 2 +- .../cluster_unit/test_cluster_pcm_x_convert.c | 89 ++++++++++++++++++- 3 files changed, 92 insertions(+), 5 deletions(-) diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index 449af318d7..dac13fc296 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -14860,7 +14860,11 @@ cluster_pcm_x_local_writer_claim_exact(const PcmXLocalHandle *writer, claim_out->claim_generation = next_claim_generation; claim_out->local_round = member->admitted_round; claim_out->role = member->role; - claim_out->grant_base_own_generation = 0; + /* A' rebase: every same-round claim inherits the published effective + * grant base under this partition lock; zero (no rebase) keeps the + * enqueue-time identity math. Without this a FIFO follower would verify + * the rebased X against its stale base+1 and fail closed. */ + claim_out->grant_base_own_generation = tag_slot->grant_base_own_generation; result = PCM_X_QUEUE_OK; claim_release_gate: diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 76d3af3e08..71d80d71a2 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -3473,7 +3473,7 @@ UT_TEST(test_pcm_x_role_refresh_accepts_only_same_member_promotion) int main(void) { - UT_PLAN(76); + UT_PLAN(78); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index aff688a5f1..f0b0d571f1 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -25,6 +25,7 @@ #include #include "cluster/cluster_ic_envelope.h" +#include "cluster/cluster_pcm_x_bufmgr.h" #include "cluster/cluster_pcm_x_convert.h" #include "cluster/cluster_shmem.h" #include "storage/buf_internals.h" @@ -10348,9 +10349,13 @@ typedef struct TestLocalRebaseFixture { /* Drive a local leader with a nonzero enqueue-time base (=5) through the * production chain to the exact PREPARE_GRANT-applied, pre-INSTALL window * where an interleaved revoke would have moved the own generation. */ +static void prepare_local_rebase_fixture(BlockNumber block, uint64 master_session, + TestLocalRebaseFixture *fixture); + static void -prepare_local_rebase_fixture(BlockNumber block, uint64 master_session, - TestLocalRebaseFixture *fixture) +prepare_local_rebase_fixture_with_follower(BlockNumber block, uint64 master_session, + TestLocalRebaseFixture *fixture, + PcmXLocalHandle *same_round_follower_out) { PcmXLocalHolderKey holder_key; PcmXLocalHolderHandle holder; @@ -10377,6 +10382,18 @@ prepare_local_rebase_fixture(BlockNumber block, uint64 master_session, UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&fixture->writer, &fixture->writer_claim), PCM_X_QUEUE_OK); UT_ASSERT_EQ(fixture->writer_claim.grant_base_own_generation, 0); + if (same_round_follower_out != NULL) { + /* Joined before the cutoff freezes the round: a same-round FIFO + * follower with its own enqueue-time identity base. */ + PcmXWaitIdentity follower_identity + = make_wait_identity(block, 0, 4, master_session + UINT64_C(3)); + + follower_identity.base_own_generation = UINT64_C(5); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&follower_identity, 1, master_session, + same_round_follower_out), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(same_round_follower_out->role, PCM_X_LOCAL_ROLE_FOLLOWER); + } UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&fixture->writer, &cutoff), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&fixture->writer, &enqueue, &token), @@ -10427,6 +10444,13 @@ prepare_local_rebase_fixture(BlockNumber block, uint64 master_session, UT_ASSERT_EQ(fixture->tag_slot->grant_base_own_generation, 0); } +static void +prepare_local_rebase_fixture(BlockNumber block, uint64 master_session, + TestLocalRebaseFixture *fixture) +{ + prepare_local_rebase_fixture_with_follower(block, master_session, fixture, NULL); +} + /* * A' rebase, local plane: the one-shot pre-reservation publication of the * effective grant base and the exact final-ack math that follows it. The @@ -10531,6 +10555,64 @@ UT_TEST(test_local_grant_rebase_conflict_fails_closed) UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); } +/* A' rebase, same-node FIFO: a follower claiming after the leader's one-shot + * publish must inherit the published effective grant base from the tag slot; + * with its enqueue-time identity base it would verify the new X against the + * stale base+1 and fail closed (t/400 L3 review P0-1). */ +UT_TEST(test_local_follower_claim_inherits_published_rebase) +{ + TestLocalRebaseFixture fixture; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + PcmXLocalReliableToken token; + PcmXLocalHandle follower; + PcmXLocalWriterClaim follower_claim; + ClusterPcmOwnSnapshot granted; + const uint64 master_session = UINT64_C(1815); + + prepare_local_rebase_fixture_with_follower(7122, master_session, &fixture, &follower); + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(8)), + PCM_X_QUEUE_OK); + + /* Leader finishes its grant on the rebased math and releases the claim. */ + UT_ASSERT_EQ(cluster_pcm_x_local_install_ready_arm_exact(&fixture.writer, &fixture.prepare.ref, + &fixture.prepare.image, &install_ready, + &token), + PCM_X_QUEUE_OK); + memset(&commit, 0, sizeof(commit)); + commit.ref = fixture.prepare.ref; + commit.phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; + UT_ASSERT_EQ(cluster_pcm_x_local_commit_x_exact(&fixture.writer, &commit, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_final_ack_arm_exact(&fixture.writer, UINT64_C(9), &final_ack, &token), + PCM_X_QUEUE_OK); + memset(&final_commit, 0, sizeof(final_commit)); + final_commit.ref = fixture.prepare.ref; + final_commit.phase = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; + UT_ASSERT_EQ(cluster_pcm_x_local_final_commit_ack_exact(&fixture.writer, &final_commit, 1, + master_session, &final_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&fixture.writer_claim), + PCM_X_QUEUE_OK); + + /* The follower's claim carries the effective grant base, and the bufmgr + * grant-snapshot proof accepts exactly rebased+1. */ + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &follower_claim), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(follower_claim.grant_base_own_generation, UINT64_C(8)); + memset(&granted, 0, sizeof(granted)); + granted.tag = follower_claim.writer.identity.tag; + granted.generation = UINT64_C(9); + granted.reservation_token = UINT64_C(77); + granted.flags = 0; + granted.pcm_state = (uint8)PCM_STATE_X; + UT_ASSERT(cluster_pcm_x_writer_grant_snapshot_exact(&follower_claim, &granted, &granted)); +} + UT_TEST(test_local_cancelled_non_source_participant_gen0_drains_and_retires_exactly) { PcmXShmemHeader *header; @@ -15034,7 +15116,7 @@ UT_TEST(test_local_retire_episode_lock_errors_fail_closed) int main(void) { - UT_PLAN(247); + UT_PLAN(257); UT_RUN(test_image_id_domain_is_canonical_and_bounded); UT_RUN(test_wire_abi_sizes_are_exact); UT_RUN(test_wire_abi_offsets_are_exact); @@ -15212,6 +15294,7 @@ main(void) UT_RUN(test_local_cross_lane_retire_exemption_requires_distinct_ticket); UT_RUN(test_local_grant_rebase_publish_is_one_shot_and_effective); UT_RUN(test_local_grant_rebase_conflict_fails_closed); + UT_RUN(test_local_follower_claim_inherits_published_rebase); UT_RUN(test_local_cancelled_non_source_participant_gen0_drains_and_retires_exactly); UT_RUN(test_local_cancelled_participant_gen0_drain_requires_frozen_round); UT_RUN(test_local_holder_drain_validates_frozen_round_before_terminal_publish); From e5c73450407ac279b2aec47d9387b8e60b194f97 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 02:15:38 -0700 Subject: [PATCH 20/56] fix(cluster): sample the PCM-X capability family atomically and enforce V2 admission at ingress t/400 L3 review P0-2 + P1-3. P0-2: the formation collector read cluster_sf_peer_supports_pcm_x_rebase() in a separate lock acquisition from the session authentication, so a peer reconnect inside one collection could bind the OLD connection's REBASE bit to the NEW connection's session and wrongly activate formation-wide V2 coverage. The collector now takes one lock-coherent (CONVERT, REBASE, capability-record generation) triple per member -- new pure cluster_sf_peer_cap_family_sample() + locked cluster_sf_peer_pcm_x_capability_sample() -- and re-samples after the membership/epoch after-pass: any reconnect moves the record generation and the tick is refused instead of activated. P1-3: the INSTALL_READY handler accepted a 112-byte V2 frame on length alone; the sender-side coverage gate is not a receiver invariant. cluster_gcs_pcm_x_install_ready_ingress_valid() now requires, for the V2 length, BOTH the local activated formation's rebase_wire_active and the source connection's REBASE HELLO; V1 frames stay independent of the pair. TDD: gcs_block truth-table rows for the V2 admission pair RED (3 bites) against the ungated validator, GREEN after (79/79 incl. the new collector source pin test_pcm_x_formation_samples_capability_family_atomically); sf_dep family-sample record-coherence matrix 12/12. --- src/backend/cluster/cluster_gcs_block.c | 50 +++++++++-- src/backend/cluster/cluster_sf_dep.c | 32 +++++++ src/include/cluster/cluster_gcs_block.h | 5 +- src/include/cluster/cluster_sf_dep.h | 26 ++++++ .../cluster_unit/test_cluster_gcs_block.c | 90 +++++++++++++++---- src/test/cluster_unit/test_cluster_sf_dep.c | 55 ++++++++++++ 6 files changed, 234 insertions(+), 24 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index d735e2b142..3ef26894e9 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -9211,6 +9211,8 @@ gcs_block_pcm_x_collect_formation(PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_L { ClusterMembershipState membership_after[CLUSTER_MAX_NODES]; ClusterMembershipState membership_before[CLUSTER_MAX_NODES]; + uint32 cap_generation_before[CLUSTER_MAX_NODES]; + bool cap_rebase_before[CLUSTER_MAX_NODES]; uint64 epoch_after; uint64 epoch_before; uint64 peer_session; @@ -9222,6 +9224,8 @@ gcs_block_pcm_x_collect_formation(PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_L || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) return false; memset(bindings, 0, sizeof(PcmXPeerBinding) * PCM_X_PROTOCOL_NODE_LIMIT); + memset(cap_generation_before, 0, sizeof(cap_generation_before)); + memset(cap_rebase_before, 0, sizeof(cap_rebase_before)); *epoch_out = 0; *self_session_out = 0; epoch_before = cluster_epoch_get_current(); @@ -9239,13 +9243,19 @@ gcs_block_pcm_x_collect_formation(PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_L continue; if (i >= PCM_X_PROTOCOL_NODE_LIMIT) return false; - if (i != cluster_node_id && !cluster_sf_peer_supports_pcm_x_convert(i)) - return false; - /* Rebase coverage never refuses the base protocol: a member without - * the V2 bit only pins the whole formation to V1 frames. */ - if (rebase_all_out != NULL && i != cluster_node_id - && !cluster_sf_peer_supports_pcm_x_rebase(i)) - *rebase_all_out = false; + /* review P0-2: the CONVERT requirement, the REBASE coverage bit and + * the capability-record generation are one lock-coherent sample, so + * the after-pass below can prove the peer connection that advertised + * them is the SAME one the session binding names. Rebase coverage + * never refuses the base protocol: a member without the V2 bit only + * pins the whole formation to V1 frames. */ + if (i != cluster_node_id) { + if (!cluster_sf_peer_pcm_x_capability_sample(i, &cap_rebase_before[i], + &cap_generation_before[i])) + return false; + if (rebase_all_out != NULL && !cap_rebase_before[i]) + *rebase_all_out = false; + } auth_result = gcs_block_pcm_x_authenticated_session_result(i, epoch_before, &peer_session, NULL); if (auth_result != PCM_X_SESSION_AUTH_OK) @@ -9261,6 +9271,20 @@ gcs_block_pcm_x_collect_formation(PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_L if (epoch_before != epoch_after || !cluster_qvotec_in_quorum() || memcmp(membership_before, membership_after, sizeof(membership_before)) != 0) return false; + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + bool cap_rebase_after; + uint32 cap_generation_after; + + if (membership_before[i] != CLUSTER_MEMBER_MEMBER || i == cluster_node_id) + continue; + /* review P0-2 after-pass: any reconnect inside this collection moved + * the capability-record generation; refuse the tick rather than bind + * a stale connection's capability word to the fresh session. */ + if (!cluster_sf_peer_pcm_x_capability_sample(i, &cap_rebase_after, &cap_generation_after) + || cap_rebase_after != cap_rebase_before[i] + || cap_generation_after != cap_generation_before[i]) + return false; + } *epoch_out = epoch_before; *self_session_out = bindings[cluster_node_id].peer_session_incarnation; @@ -11556,6 +11580,8 @@ cluster_gcs_handle_pcm_x_install_ready_envelope(const ClusterICEnvelope *env, co uint64 source_session; int32 source_node; int32 tag_master; + bool rebase_wire_active = false; + bool source_rebase_capable = false; /* Both exact lengths are legal: the V1 104-byte frame normalizes to a * zero rebase, the V2 112-byte frame carries the published grant base. */ @@ -11569,8 +11595,16 @@ cluster_gcs_handle_pcm_x_install_ready_envelope(const ClusterICEnvelope *env, co memcpy(&frame, payload, env->payload_length); current_epoch = cluster_epoch_get_current(); tag_master = cluster_gcs_lookup_master(frame.ref.identity.tag); + /* Receiver-side V2 admission: the sender-side coverage gate is not an + * ingress invariant, so a V2 frame re-proves BOTH the activated + * formation-wide coverage and the source connection's REBASE HELLO. */ + if (env->payload_length == sizeof(frame)) { + rebase_wire_active = cluster_pcm_x_runtime_snapshot().rebase_wire_active; + source_rebase_capable = cluster_sf_peer_supports_pcm_x_rebase(source_node); + } if (!cluster_gcs_pcm_x_install_ready_ingress_valid(&frame, env->payload_length, source_node, - current_epoch, tag_master, cluster_node_id) + current_epoch, tag_master, cluster_node_id, + rebase_wire_active, source_rebase_capable) || !gcs_block_pcm_x_transfer_ingress_authorized(&frame.ref.identity.tag, source_node, current_epoch, &source_session)) return; diff --git a/src/backend/cluster/cluster_sf_dep.c b/src/backend/cluster/cluster_sf_dep.c index 1d1ae2557c..cebc1ee2dd 100644 --- a/src/backend/cluster/cluster_sf_dep.c +++ b/src/backend/cluster/cluster_sf_dep.c @@ -387,6 +387,38 @@ cluster_sf_peer_supports_pcm_x_rebase(int32 peer_id) return (capabilities & PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1) != 0; } +/* + * cluster_sf_peer_pcm_x_capability_sample + * + * review P0-2: one lock-coherent sample of the PCM-X capability family on + * the peer's CURRENT connection. True iff the verified HELLO advertised + * PCM_X_CONVERT; the REBASE bit and the capability-record generation come + * from the SAME locked read. The formation collector samples this on both + * sides of its authority pass: a reconnect between the samples changes the + * generation and rejects the tick, so a stale connection's REBASE bit can + * never be bound to a fresh session/binding. + */ +bool +cluster_sf_peer_pcm_x_capability_sample(int32 peer_id, bool *rebase_out, uint32 *generation_out) +{ + bool supported; + + if (rebase_out != NULL) + *rebase_out = false; + if (generation_out != NULL) + *generation_out = 0; + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return false; + + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + supported = cluster_sf_peer_cap_family_sample(&ClusterSfDep->peer_capabilities[peer_id], + PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, rebase_out, + generation_out); + LWLockRelease(&ClusterSfDep->lock); + return supported; +} + /* Return one lock-coherent snapshot of the capability-record generation whose * verified HELLO advertised PCM-X. Tier1 owns this record on CONTROL and can * sample it around another authority read to reject reconnect ABA; RDMA keeps diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index 5b66ae30c5..69d091a420 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -993,11 +993,14 @@ static inline bool cluster_gcs_pcm_x_install_ready_ingress_valid(const PcmXInstallReadyPayload *ready, Size payload_length, int32 authenticated_node, uint64 current_epoch, int32 tag_master, - int32 local_node) + int32 local_node, bool rebase_wire_active, + bool source_supports_rebase) { return ready != NULL && (payload_length == sizeof(*ready) || payload_length == PCM_X_INSTALL_READY_V1_LEN) && (payload_length == sizeof(*ready) || ready->rebased_own_generation == 0) + && (payload_length != sizeof(*ready) + || (rebase_wire_active && source_supports_rebase)) && (ready->rebased_own_generation == 0 || (ready->rebased_own_generation != UINT64_MAX && ready->rebased_own_generation > ready->ref.identity.base_own_generation)) diff --git a/src/include/cluster/cluster_sf_dep.h b/src/include/cluster/cluster_sf_dep.h index 50403b81c2..606441dfde 100644 --- a/src/include/cluster/cluster_sf_dep.h +++ b/src/include/cluster/cluster_sf_dep.h @@ -103,6 +103,28 @@ cluster_sf_peer_cap_generation_for_bits(const ClusterSfPeerCap *cap, uint32 requ return true; } +/* review P0-2: one record-coherent sample of a capability family. True iff + * every required bit is present on the CURRENT connection's record; the + * optional-bit presence and the record generation come from the SAME read, + * so a caller sampling this on both sides of an authority pass rejects any + * reconnect in between (the generation moves) instead of pairing a stale + * connection's optional bit with a fresh session. False zeroes both + * outputs. */ +static inline bool +cluster_sf_peer_cap_family_sample(const ClusterSfPeerCap *cap, uint32 required_bits, + uint32 optional_bits, bool *optional_out, + uint32 *generation_out) +{ + if (optional_out != NULL) + *optional_out = false; + if (!cluster_sf_peer_cap_generation_for_bits(cap, required_bits, generation_out)) + return false; + if (optional_out != NULL) + *optional_out = (cluster_sf_peer_cap_bits(cap) & optional_bits) == optional_bits + && optional_bits != 0; + return true; +} + /* Returns true iff the record was live for exactly this generation and got * cleared; a mismatch (older/newer generation, already invalid) is a no-op. */ static inline bool @@ -237,6 +259,10 @@ extern bool cluster_sf_peer_supports_xid_authority_flock(int32 peer_id); extern bool cluster_sf_peer_supports_gcs_inval_busy(int32 peer_id); extern bool cluster_sf_peer_supports_pcm_x_convert(int32 peer_id); extern bool cluster_sf_peer_supports_pcm_x_rebase(int32 peer_id); +/* review P0-2: lock-coherent (CONVERT supported, REBASE bit, record + * generation) triple for the formation collector's double sample. */ +extern bool cluster_sf_peer_pcm_x_capability_sample(int32 peer_id, bool *rebase_out, + uint32 *generation_out); extern bool cluster_sf_peer_pcm_x_connection_generation(int32 peer_id, uint32 *generation); extern void cluster_sf_note_peer_disconnected_gen(int32 peer_id, uint32 generation); extern void cluster_sf_note_peer_disconnected(int32 peer_id); diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 71d80d71a2..d024e912f9 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -1172,6 +1172,50 @@ UT_TEST(test_pcm_x_formation_transient_or_inconsistent_sample_is_tick_noop) } +/* review P0-2: the collector binds capability bits to the peer session only + * through one lock-coherent (bits, generation) sample taken on BOTH sides of + * the authority pass; a reconnect between the samples changes the record + * generation and rejects the tick, so a stale connection's REBASE bit can + * never activate V2 against a fresh session. */ +UT_TEST(test_pcm_x_formation_samples_capability_family_atomically) +{ + char *source = read_gcs_block_source(); + const char *collect; + const char *first_sample; + const char *barrier; + const char *second_sample; + const char *end; + const char *stray; + + UT_ASSERT_NOT_NULL(source); + if (source != NULL) { + collect = strstr(source, "\ngcs_block_pcm_x_collect_formation("); + UT_ASSERT_NOT_NULL(collect); + end = collect != NULL ? strstr(collect, "\n#define PCM_X_MASTER_DRIVE_SCAN_BUDGET") : NULL; + UT_ASSERT_NOT_NULL(end); + first_sample + = collect != NULL ? strstr(collect, "cluster_sf_peer_pcm_x_capability_sample(") : NULL; + UT_ASSERT_NOT_NULL(first_sample); + barrier = first_sample != NULL ? strstr(first_sample, "pg_read_barrier();") : NULL; + UT_ASSERT_NOT_NULL(barrier); + second_sample + = barrier != NULL ? strstr(barrier, "cluster_sf_peer_pcm_x_capability_sample(") : NULL; + UT_ASSERT_NOT_NULL(second_sample); + if (second_sample != NULL && end != NULL) + UT_ASSERT(second_sample < end); + /* The separate single-shot capability reads are gone from the + * collector: only the atomic family sample may feed it. */ + if (collect != NULL && end != NULL) { + stray = strstr(collect, "cluster_sf_peer_supports_pcm_x_convert("); + UT_ASSERT(stray == NULL || stray > end); + stray = strstr(collect, "cluster_sf_peer_supports_pcm_x_rebase("); + UT_ASSERT(stray == NULL || stray > end); + } + free(source); + } +} + + UT_TEST(test_pcm_x_confirm_publish_then_stale_requires_exact_graph_close) { UT_ASSERT(cluster_gcs_pcm_x_confirm_compensation_required(UINT64_C(7001), PCM_X_QUEUE_STALE)); @@ -1297,21 +1341,21 @@ UT_TEST(test_pcm_x_install_ready_ingress_is_canonical_requester_ack) ready.result = PCM_X_QUEUE_OK; ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; - UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 3, 11, 2, 2)); + UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 3, 11, 2, 2, true, true)); ready.result = PCM_X_QUEUE_DUPLICATE; - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); ready.result = PCM_X_QUEUE_OK; ready.phase = PGRAC_IC_MSG_PCM_X_PREPARE_GRANT; - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; ready.flags = 1; - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); ready.flags = 0; ready.image_id = UINT64CONST(0xf000000000000025); - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); UT_ASSERT(cluster_pcm_x_image_id_encode(3, 37, &ready.image_id)); - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); /* A' rebase: both exact frame lengths are legal, nothing in between; a * V1-length frame must carry a zero rebase and a V2 rebase must be @@ -1319,22 +1363,37 @@ UT_TEST(test_pcm_x_install_ready_ingress_is_canonical_requester_ack) * sentinel. */ UT_ASSERT(cluster_pcm_x_image_id_encode(2, 37, &ready.image_id)); UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, PCM_X_INSTALL_READY_V1_LEN, 1, - 11, 2, 2)); + 11, 2, 2, true, true)); UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, PCM_X_INSTALL_READY_V1_LEN + 4, - 1, 11, 2, 2)); + 1, 11, 2, 2, true, true)); ready.ref.identity.base_own_generation = 5; ready.rebased_own_generation = 8; - UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, PCM_X_INSTALL_READY_V1_LEN, 1, - 11, 2, 2)); + 11, 2, 2, true, true)); ready.rebased_own_generation = 5; - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); ready.rebased_own_generation = 4; - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); ready.rebased_own_generation = UINT64_MAX; - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); + + /* Receiver-side V2 admission (review P1-3): a 112-byte frame is refused + * unless BOTH this node's activated formation has full V2 coverage and + * the source's current connection advertised the REBASE capability. The + * sender-side coverage gate is not a receiver invariant. */ + ready.rebased_own_generation = 8; + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + false, true)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, false)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + false, false)); ready.rebased_own_generation = 0; ready.ref.identity.base_own_generation = 0; + /* V1 frames stay independent of the rebase capability pair. */ + UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, PCM_X_INSTALL_READY_V1_LEN, 1, + 11, 2, 2, false, false)); } @@ -3473,7 +3532,7 @@ UT_TEST(test_pcm_x_role_refresh_accepts_only_same_member_promotion) int main(void) { - UT_PLAN(78); + UT_PLAN(79); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -3512,6 +3571,7 @@ main(void) UT_RUN(test_pcm_x_blocker_ack_carries_full_generation_and_binds_master_source); UT_RUN(test_pcm_x_formation_identical_complete_samples_may_revalidate); UT_RUN(test_pcm_x_formation_transient_or_inconsistent_sample_is_tick_noop); + UT_RUN(test_pcm_x_formation_samples_capability_family_atomically); UT_RUN(test_pcm_x_confirm_publish_then_stale_requires_exact_graph_close); UT_RUN(test_pcm_x_drain_poll_binds_exact_master_and_generation); UT_RUN(test_pcm_x_drain_ack_binds_participant_and_canonical_payload); diff --git a/src/test/cluster_unit/test_cluster_sf_dep.c b/src/test/cluster_unit/test_cluster_sf_dep.c index 148ffe4bb0..886a384d42 100644 --- a/src/test/cluster_unit/test_cluster_sf_dep.c +++ b/src/test/cluster_unit/test_cluster_sf_dep.c @@ -260,6 +260,60 @@ UT_TEST(test_pcm_x_capability_generation_snapshot_is_exact) UT_ASSERT_EQ(generation, (uint32)0); } +/* review P0-2: the family sample returns required-bit support, the optional + * bit and the record generation from one record read; a reconnect renote is + * visible as a generation move together with its bits. */ +UT_TEST(test_pcm_x_capability_family_sample_is_record_coherent) +{ + ClusterSfPeerCap cap; + uint32 generation = UINT32_MAX; + bool rebase = true; + + memset(&cap, 0, sizeof(cap)); + UT_ASSERT(!cluster_sf_peer_cap_family_sample(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, &rebase, + &generation)); + UT_ASSERT(!rebase); + UT_ASSERT_EQ(generation, (uint32)0); + + /* CONVERT without REBASE: supported, optional bit false. */ + cluster_sf_peer_cap_note(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, 7); + rebase = true; + UT_ASSERT(cluster_sf_peer_cap_family_sample(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, &rebase, + &generation)); + UT_ASSERT(!rebase); + UT_ASSERT_EQ(generation, (uint32)7); + + /* Reconnect renote with both bits: the generation moves with the bits. */ + cluster_sf_peer_cap_note( + &cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, 8); + UT_ASSERT(cluster_sf_peer_cap_family_sample(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, &rebase, + &generation)); + UT_ASSERT(rebase); + UT_ASSERT_EQ(generation, (uint32)8); + + /* REBASE alone never satisfies the family requirement. */ + cluster_sf_peer_cap_note(&cap, PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, 9); + rebase = true; + UT_ASSERT(!cluster_sf_peer_cap_family_sample(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, &rebase, + &generation)); + UT_ASSERT(!rebase); + UT_ASSERT_EQ(generation, (uint32)0); + + /* Disconnect invalidation clears the whole family. */ + cluster_sf_peer_cap_note( + &cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, 10); + UT_ASSERT(cluster_sf_peer_cap_invalidate_gen(&cap, 10)); + rebase = true; + UT_ASSERT(!cluster_sf_peer_cap_family_sample(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, &rebase, + &generation)); + UT_ASSERT(!rebase); +} + int main(void) { @@ -274,5 +328,6 @@ main(void) UT_RUN(test_peer_cap_gen_renote_after_reconnect); UT_RUN(test_pcm_x_capability_does_not_alias_idle_horizon); UT_RUN(test_pcm_x_capability_generation_snapshot_is_exact); + UT_RUN(test_pcm_x_capability_family_sample_is_record_coherent); UT_DONE(); } From e39bca792618b0352c5b97d24e1e4b154f19b738 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 02:19:08 -0700 Subject: [PATCH 21/56] test(cluster): execute the preflight BUSY wait arm over the real ownership object t/400 L3 review P1-4. The transient-BUSY preflight fix shipped with a classification table, preflight truth rows and a driver source pin, but no test EXECUTED the arm end to end. The new pcm_own case drives the real reservation object: a live GRANT_PENDING (then REVOKING) lifecycle preflights BUSY and the RESERVATION_PREFLIGHT site classifies WAIT; after the lifecycle clears, the SAME identity's re-snapshot preflights OK and the real grant reservation begins on the idle tuple (token monotonic, GRANT_PENDING re-armed). Formation/session drift across the wait fails the exact runtime proof on gate generation, master session and runtime state, refusing the re-entry before any reservation mutation. 46/46. --- src/test/cluster_unit/test_cluster_pcm_own.c | 88 +++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index bcc2b935e0..d5f0de6811 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -15,6 +15,7 @@ */ #include "postgres.h" +#include "cluster/cluster_gcs_block.h" #include "cluster/cluster_pcm_own.h" #include "cluster/cluster_pcm_x_bufmgr.h" #include "cluster/cluster_shmem.h" @@ -2007,10 +2008,94 @@ UT_TEST(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_aut free(source); } +/* review P1-4: deterministic execution of the preflight transient-BUSY arm + * over the REAL ownership object. A live GRANT_PENDING (then REVOKING) + * lifecycle preflights BUSY and the requester site classifies WAIT -- never + * fail-closed; once the lifecycle clears, the SAME identity's re-snapshot + * preflights OK and the real grant reservation begins on the idle tuple. + * Formation/session drift observed across the wait fails the exact runtime + * proof, so the driver exits before touching the reservation. */ +UT_TEST(test_preflight_busy_waits_then_clean_resnapshot_begins_reservation) +{ + ClusterPcmOwnSnapshot live; + PcmXWaitIdentity identity; + PcmXRuntimeSnapshot current; + PcmXRuntimeSnapshot start; + uint64 blocker_token = 0; + uint64 token = 0; + + reset_fixture(); + memset(&identity, 0, sizeof(identity)); + identity.tag.relNumber = 20001; + identity.tag.blockNum = 3; + identity.base_own_generation = 0; + memset(&live, 0, sizeof(live)); + live.tag = identity.tag; + live.pcm_state = (uint8)PCM_STATE_N; + + /* A concurrent grant lifecycle holds GRANT_PENDING: BUSY -> WAIT. */ + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_GRANT_PENDING, + &blocker_token), + CLUSTER_PCM_OWN_OK); + live.generation = pg_atomic_read_u64(&ClusterPcmOwnArray[0].generation); + live.reservation_token = pg_atomic_read_u64(&ClusterPcmOwnArray[0].reservation_token); + live.flags = pg_atomic_read_u32(&ClusterPcmOwnArray[0].flags); + UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, PCM_X_QUEUE_BUSY), + GCS_BLOCK_PCM_X_RETRY_WAIT); + UT_ASSERT_EQ(cluster_pcm_own_reservation_abort_exact(0, 0, blocker_token, + PCM_OWN_FLAG_GRANT_PENDING), + CLUSTER_PCM_OWN_OK); + + /* A live revoke lifecycle classifies exactly the same way. */ + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_REVOKING, + &blocker_token), + CLUSTER_PCM_OWN_OK); + live.reservation_token = pg_atomic_read_u64(&ClusterPcmOwnArray[0].reservation_token); + live.flags = pg_atomic_read_u32(&ClusterPcmOwnArray[0].flags); + UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, PCM_X_QUEUE_BUSY), + GCS_BLOCK_PCM_X_RETRY_WAIT); + UT_ASSERT_EQ( + cluster_pcm_own_reservation_abort_exact(0, 0, blocker_token, PCM_OWN_FLAG_REVOKING), + CLUSTER_PCM_OWN_OK); + + /* The wait ends: the clean-N re-snapshot (idle nonzero token is legal) + * preflights OK and the REAL reservation begins on the same tuple. */ + live.reservation_token = pg_atomic_read_u64(&ClusterPcmOwnArray[0].reservation_token); + live.flags = pg_atomic_read_u32(&ClusterPcmOwnArray[0].flags); + UT_ASSERT_EQ(live.flags, (uint32)0); + UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_GRANT_PENDING, &token), + CLUSTER_PCM_OWN_OK); + UT_ASSERT(token > blocker_token); + assert_entry(0, token, PCM_OWN_FLAG_GRANT_PENDING); + + /* Formation/session drift across the wait refuses the re-entry. */ + memset(&start, 0, sizeof(start)); + start.state = PCM_X_RUNTIME_ACTIVE; + start.gate_generation = 3; + start.master_session_incarnation = 9; + current = start; + UT_ASSERT(cluster_gcs_pcm_x_requester_runtime_exact(&start, ¤t)); + current.gate_generation = 4; + UT_ASSERT(!cluster_gcs_pcm_x_requester_runtime_exact(&start, ¤t)); + current = start; + current.master_session_incarnation = 10; + UT_ASSERT(!cluster_gcs_pcm_x_requester_runtime_exact(&start, ¤t)); + current = start; + current.state = PCM_X_RUNTIME_RECOVERY_BLOCKED; + UT_ASSERT(!cluster_gcs_pcm_x_requester_runtime_exact(&start, ¤t)); +} + int main(void) { - UT_PLAN(45); + UT_PLAN(46); UT_RUN(test_shmem_initializes_complete_entry); UT_RUN(test_begin_abort_is_exact_and_monotonic); UT_RUN(test_invalid_live_flag_shapes_are_corrupt_not_busy); @@ -2056,6 +2141,7 @@ main(void) UT_RUN(test_queue_passive_n_mirror_is_never_gcs_ship_authority); UT_RUN(test_queue_writer_grant_snapshot_is_claim_and_generation_exact); UT_RUN(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_authority); + UT_RUN(test_preflight_busy_waits_then_clean_resnapshot_begins_reservation); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From e6134eb4afd4484718afb59b9edcf9d1be4d4251 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 02:20:22 -0700 Subject: [PATCH 22/56] style(cluster): clang-format the review-fix surfaces --- src/backend/cluster/cluster_sf_dep.c | 7 ++-- src/include/cluster/cluster_gcs_block.h | 3 +- src/include/cluster/cluster_sf_dep.h | 3 +- .../cluster_unit/test_cluster_gcs_block.c | 33 ++++++++++++------- src/test/cluster_unit/test_cluster_pcm_own.c | 18 +++++----- .../cluster_unit/test_cluster_pcm_x_convert.c | 2 +- 6 files changed, 37 insertions(+), 29 deletions(-) diff --git a/src/backend/cluster/cluster_sf_dep.c b/src/backend/cluster/cluster_sf_dep.c index cebc1ee2dd..90d2d0d1d0 100644 --- a/src/backend/cluster/cluster_sf_dep.c +++ b/src/backend/cluster/cluster_sf_dep.c @@ -411,10 +411,9 @@ cluster_sf_peer_pcm_x_capability_sample(int32 peer_id, bool *rebase_out, uint32 return false; LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); - supported = cluster_sf_peer_cap_family_sample(&ClusterSfDep->peer_capabilities[peer_id], - PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, - PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, rebase_out, - generation_out); + supported = cluster_sf_peer_cap_family_sample( + &ClusterSfDep->peer_capabilities[peer_id], PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, rebase_out, generation_out); LWLockRelease(&ClusterSfDep->lock); return supported; } diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index 69d091a420..11ec756f62 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -999,8 +999,7 @@ cluster_gcs_pcm_x_install_ready_ingress_valid(const PcmXInstallReadyPayload *rea return ready != NULL && (payload_length == sizeof(*ready) || payload_length == PCM_X_INSTALL_READY_V1_LEN) && (payload_length == sizeof(*ready) || ready->rebased_own_generation == 0) - && (payload_length != sizeof(*ready) - || (rebase_wire_active && source_supports_rebase)) + && (payload_length != sizeof(*ready) || (rebase_wire_active && source_supports_rebase)) && (ready->rebased_own_generation == 0 || (ready->rebased_own_generation != UINT64_MAX && ready->rebased_own_generation > ready->ref.identity.base_own_generation)) diff --git a/src/include/cluster/cluster_sf_dep.h b/src/include/cluster/cluster_sf_dep.h index 606441dfde..5d0f61d4a0 100644 --- a/src/include/cluster/cluster_sf_dep.h +++ b/src/include/cluster/cluster_sf_dep.h @@ -112,8 +112,7 @@ cluster_sf_peer_cap_generation_for_bits(const ClusterSfPeerCap *cap, uint32 requ * outputs. */ static inline bool cluster_sf_peer_cap_family_sample(const ClusterSfPeerCap *cap, uint32 required_bits, - uint32 optional_bits, bool *optional_out, - uint32 *generation_out) + uint32 optional_bits, bool *optional_out, uint32 *generation_out) { if (optional_out != NULL) *optional_out = false; diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index d024e912f9..5e86082312 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -1341,21 +1341,28 @@ UT_TEST(test_pcm_x_install_ready_ingress_is_canonical_requester_ack) ready.result = PCM_X_QUEUE_OK; ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; - UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 3, 11, 2, 2, true, true)); + UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 3, 11, 2, 2, + true, true)); ready.result = PCM_X_QUEUE_DUPLICATE; - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); ready.result = PCM_X_QUEUE_OK; ready.phase = PGRAC_IC_MSG_PCM_X_PREPARE_GRANT; - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; ready.flags = 1; - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); ready.flags = 0; ready.image_id = UINT64CONST(0xf000000000000025); - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); UT_ASSERT(cluster_pcm_x_image_id_encode(3, 37, &ready.image_id)); - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); /* A' rebase: both exact frame lengths are legal, nothing in between; a * V1-length frame must carry a zero rebase and a V2 rebase must be @@ -1368,15 +1375,19 @@ UT_TEST(test_pcm_x_install_ready_ingress_is_canonical_requester_ack) 1, 11, 2, 2, true, true)); ready.ref.identity.base_own_generation = 5; ready.rebased_own_generation = 8; - UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); + UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, PCM_X_INSTALL_READY_V1_LEN, 1, 11, 2, 2, true, true)); ready.rebased_own_generation = 5; - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); ready.rebased_own_generation = 4; - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); ready.rebased_own_generation = UINT64_MAX; - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, true, true)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); /* Receiver-side V2 admission (review P1-3): a 112-byte frame is refused * unless BOTH this node's activated formation has full V2 coverage and diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index d5f0de6811..aeceb6b128 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -2034,9 +2034,9 @@ UT_TEST(test_preflight_busy_waits_then_clean_resnapshot_begins_reservation) live.pcm_state = (uint8)PCM_STATE_N; /* A concurrent grant lifecycle holds GRANT_PENDING: BUSY -> WAIT. */ - UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_GRANT_PENDING, - &blocker_token), - CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ( + cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_GRANT_PENDING, &blocker_token), + CLUSTER_PCM_OWN_OK); live.generation = pg_atomic_read_u64(&ClusterPcmOwnArray[0].generation); live.reservation_token = pg_atomic_read_u64(&ClusterPcmOwnArray[0].reservation_token); live.flags = pg_atomic_read_u32(&ClusterPcmOwnArray[0].flags); @@ -2045,14 +2045,14 @@ UT_TEST(test_preflight_busy_waits_then_clean_resnapshot_begins_reservation) UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, PCM_X_QUEUE_BUSY), GCS_BLOCK_PCM_X_RETRY_WAIT); - UT_ASSERT_EQ(cluster_pcm_own_reservation_abort_exact(0, 0, blocker_token, - PCM_OWN_FLAG_GRANT_PENDING), - CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ( + cluster_pcm_own_reservation_abort_exact(0, 0, blocker_token, PCM_OWN_FLAG_GRANT_PENDING), + CLUSTER_PCM_OWN_OK); /* A live revoke lifecycle classifies exactly the same way. */ - UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_REVOKING, - &blocker_token), - CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ( + cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_REVOKING, &blocker_token), + CLUSTER_PCM_OWN_OK); live.reservation_token = pg_atomic_read_u64(&ClusterPcmOwnArray[0].reservation_token); live.flags = pg_atomic_read_u32(&ClusterPcmOwnArray[0].flags); UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index f0b0d571f1..4dc639c781 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -10350,7 +10350,7 @@ typedef struct TestLocalRebaseFixture { * production chain to the exact PREPARE_GRANT-applied, pre-INSTALL window * where an interleaved revoke would have moved the own generation. */ static void prepare_local_rebase_fixture(BlockNumber block, uint64 master_session, - TestLocalRebaseFixture *fixture); + TestLocalRebaseFixture *fixture); static void prepare_local_rebase_fixture_with_follower(BlockNumber block, uint64 master_session, From 776fdb838f02a95820bdb9acd1f64d815c2b10cc Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 02:52:02 -0700 Subject: [PATCH 23/56] fix(cluster): admit the self-loopback V2 INSTALL_READY at ingress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit t/400 L3 review R2 P0-1 — the deterministic liveness regression behind the current t/400 RED. The receiver-side V2 admission consulted cluster_sf_peer_supports_pcm_x_rebase() for every 112-byte frame, but a SELF-loopback frame (master == requester node) has no HELLO capability record, so a legal self V2 INSTALL_READY was silently dropped. The master ticket froze in ACTIVE_TRANSFER at the PREPARE_GRANT leg, the remote retained source (REVOKING descriptor) was never released, and every legacy S/X entry on the source node then surfaced 'conflicts with an active reservation' client ERRORs (t/400 runs 1-2 on this branch; base commit shows zero such conflicts — bisect-confirmed regression of this branch). Mirror gcs_block_pcm_x_source_capable: self IS rebase-capable (the local binary), while the activation-bound formation coverage flag still applies. Pinned by test_pcm_x_install_ready_v2_self_loopback_is_admissible (80/80); behavioral proof is the t/400 e2e that exercises self V2 on every drift. --- src/backend/cluster/cluster_gcs_block.c | 8 +++-- .../cluster_unit/test_cluster_gcs_block.c | 36 ++++++++++++++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 3ef26894e9..8288b96fba 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -11597,10 +11597,14 @@ cluster_gcs_handle_pcm_x_install_ready_envelope(const ClusterICEnvelope *env, co tag_master = cluster_gcs_lookup_master(frame.ref.identity.tag); /* Receiver-side V2 admission: the sender-side coverage gate is not an * ingress invariant, so a V2 frame re-proves BOTH the activated - * formation-wide coverage and the source connection's REBASE HELLO. */ + * formation-wide coverage and the source's REBASE capability. A + * SELF-loopback frame (master == requester node) has no HELLO record — + * exactly like gcs_block_pcm_x_source_capable, its capability is the + * local binary itself; the coverage flag still applies. */ if (env->payload_length == sizeof(frame)) { rebase_wire_active = cluster_pcm_x_runtime_snapshot().rebase_wire_active; - source_rebase_capable = cluster_sf_peer_supports_pcm_x_rebase(source_node); + source_rebase_capable + = source_node == cluster_node_id || cluster_sf_peer_supports_pcm_x_rebase(source_node); } if (!cluster_gcs_pcm_x_install_ready_ingress_valid(&frame, env->payload_length, source_node, current_epoch, tag_master, cluster_node_id, diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 5e86082312..033f472252 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -1172,6 +1172,39 @@ UT_TEST(test_pcm_x_formation_transient_or_inconsistent_sample_is_tick_noop) } +/* review R2 P0-1: a SELF-loopback V2 INSTALL_READY (master == requester + * node) has no HELLO capability record; the handler must treat self as + * rebase-capable (the local binary) while still requiring the + * activation-bound coverage flag. The regression silently dropped the + * self V2 frame, froze the PREPARE->INSTALL chain on the master ticket and + * leaked the remote retained source. */ +UT_TEST(test_pcm_x_install_ready_v2_self_loopback_is_admissible) +{ + char *source = read_gcs_block_source(); + const char *handler; + const char *self_arm; + const char *peer_arm; + const char *valid_call; + + UT_ASSERT_NOT_NULL(source); + if (source != NULL) { + handler = strstr(source, "\ncluster_gcs_handle_pcm_x_install_ready_envelope("); + UT_ASSERT_NOT_NULL(handler); + self_arm = handler != NULL ? strstr(handler, "source_node == cluster_node_id") : NULL; + UT_ASSERT_NOT_NULL(self_arm); + peer_arm = self_arm != NULL + ? strstr(self_arm, "cluster_sf_peer_supports_pcm_x_rebase(source_node)") + : NULL; + UT_ASSERT_NOT_NULL(peer_arm); + valid_call = peer_arm != NULL + ? strstr(peer_arm, "cluster_gcs_pcm_x_install_ready_ingress_valid(") + : NULL; + UT_ASSERT_NOT_NULL(valid_call); + free(source); + } +} + + /* review P0-2: the collector binds capability bits to the peer session only * through one lock-coherent (bits, generation) sample taken on BOTH sides of * the authority pass; a reconnect between the samples changes the record @@ -3543,7 +3576,7 @@ UT_TEST(test_pcm_x_role_refresh_accepts_only_same_member_promotion) int main(void) { - UT_PLAN(79); + UT_PLAN(80); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -3582,6 +3615,7 @@ main(void) UT_RUN(test_pcm_x_blocker_ack_carries_full_generation_and_binds_master_source); UT_RUN(test_pcm_x_formation_identical_complete_samples_may_revalidate); UT_RUN(test_pcm_x_formation_transient_or_inconsistent_sample_is_tick_noop); + UT_RUN(test_pcm_x_install_ready_v2_self_loopback_is_admissible); UT_RUN(test_pcm_x_formation_samples_capability_family_atomically); UT_RUN(test_pcm_x_confirm_publish_then_stale_requires_exact_graph_close); UT_RUN(test_pcm_x_drain_poll_binds_exact_master_and_generation); From cb7ab02a82f379254fe221d45790bd9fdfa8151f Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 02:53:08 -0700 Subject: [PATCH 24/56] fix(heap): make every barrier-re-entered update/delete decision pass-local Review closure for the BARRIER_CLOSED caller-owned unwind (opus CRITICAL + codex R2 P0-3 minimal set): a re-entered pass must not consume attempt-local state from the abandoned pass. - ITL slot picks: both pairs reset before derivation, so a cross-page pass's new-page slot index can never be stamped onto a different page chosen by the retry (the stale-active false-visible class). - HOT / summarized-update verdicts: reset before the same-page check, so a cross-page retry can never carry a same-page HOT verdict and skip index entries. - cid/iscombo: the entry cid is restored on every requalify arm, so a pass-1 combo cmax neither feeds HeapTupleSatisfiesUpdate nor recombines into a combo-of-combo; the temp-locked reacquire arm deliberately keeps the pass-1 combo it already stamped. - heap_delete now frees the abandoned replica-identity copy like heap_update. Behavioral repeat-barrier tests are deferred with the rest of the non-blocker review queue until after the S3 gate (user scope ruling). --- src/backend/access/heap/heapam.c | 39 ++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index ebfdeab2eb..4cd6a60003 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -3479,6 +3479,7 @@ heap_delete(Relation relation, ItemPointer tid, Buffer vmbuffer = InvalidBuffer; bool vm_locked; /* PGRAC: pre-crit VM content lock held */ int vm_barrier_retries = 0; /* PGRAC: t/400 L3 item 3 */ + CommandId pgrac_entry_cid = cid; /* PGRAC: restored on barrier requalify */ TransactionId new_xmax; uint16 new_infomask, new_infomask2; @@ -3892,9 +3893,18 @@ heap_delete(Relation relation, ItemPointer tid, * Release our own lock, resolve the conversion unlocked, and * requalify from scratch: the ITL slot pick is idempotent and an * unstamped undo record is unreachable garbage, exactly as after - * any pre-crit ERROR. + * any pre-crit ERROR. The entry cid is restored (no + * combo-of-combo) and the abandoned replica-identity copy freed. */ vm_barrier_retries++; + cid = pgrac_entry_cid; + iscombo = false; + if (old_key_tuple != NULL && old_key_copied) + { + heap_freetuple(old_key_tuple); + old_key_tuple = NULL; + old_key_copied = false; + } LockBuffer(buffer, BUFFER_LOCK_UNLOCK); cluster_heap_vm_barrier_warm(vmbuffer); LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); @@ -4193,6 +4203,7 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, bool vm_locked_new; bool old_tuple_temp_locked = false; /* PGRAC: t/400 L3 item 3 */ int vm_barrier_retries = 0; /* PGRAC: t/400 L3 item 3 */ + CommandId pgrac_entry_cid = cid; /* PGRAC: restored on barrier requalify */ bool need_toast; Size newtupsize, pagefree; @@ -4870,9 +4881,13 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, * PGRAC: vm barrier unwind (update pre-toast) — the old * tuple carries no temporary lock yet and nothing * irreversible has happened in this pass, so a full - * requalify is required and sufficient. + * requalify is required and sufficient. Restore the entry + * cid: a pass-1 combo cmax must not feed requalification or + * be recombined into a combo-of-combo. */ vm_barrier_retries++; + cid = pgrac_entry_cid; + iscombo = false; LockBuffer(buffer, BUFFER_LOCK_UNLOCK); cluster_heap_vm_barrier_warm(vmbuffer); LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); @@ -5057,6 +5072,13 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, * one pin is held. */ + /* + * PGRAC (t/400 L3 item 3 review): pass-local decision — the VM-barrier + * reacquire can flip same-page <-> cross-page between passes, and a + * stale HOT verdict on a cross-page retry would skip index entries. + */ + use_hot_update = false; + summarized_update = false; if (newbuf == buffer) { /* @@ -5104,6 +5126,17 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, * EXCLUSIVE content locks already held; OVERFLOW raises ERROR * before the critical section. */ + /* + * PGRAC (t/400 L3 item 3 review): the VM-barrier unwind can abandon a + * pass and re-enter this block with a DIFFERENT page choice (a cross-page + * pass may retry same-page once the old page regains space). Every pick + * below is pass-local: reset both pairs so a stale cross-page slot index + * can never be stamped onto a page it was not chosen on. + */ + cluster_itl_old_slot = CLUSTER_ITL_SLOT_UNALLOCATED; + cluster_itl_new_slot = CLUSTER_ITL_SLOT_UNALLOCATED; + cluster_itl_old_active = false; + cluster_itl_new_active = false; if (cluster_itl_write_path_enabled(relation) && PageHasItl(BufferGetPage(buffer))) { /* spec-3.4b D5: single xact-local TT binding shared by both stamps (F11). */ @@ -5314,6 +5347,8 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, * abandoned safely (idempotent pick, unreachable record). */ Assert(newbuf == buffer); + cid = pgrac_entry_cid; + iscombo = false; LockBuffer(buffer, BUFFER_LOCK_UNLOCK); cluster_heap_vm_barrier_warm(refused_vm); LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); From d7cac6a83592f99a87942c5696e19c71ba8611ab Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 02:53:08 -0700 Subject: [PATCH 25/56] fix(cluster): wait out a live ownership lifecycle at the legacy grant reservation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit t/400 L3 review R2 P0-2 — the second live t/400 blocker. A reader (or the rare un-valid-page writer) entering the legacy LockBuffer reservation while the tuple carries a live GRANT_PENDING/REVOKING lifecycle got an immediate 'conflicts with an active reservation' client ERROR, even though a normal revoke/grant window is a ms-scale transient (t/400: three node0 client errors against a retained-source window). cluster_bufmgr_pcm_begin_grant_reservation_wait() re-begins against the re-sampled complete ownership tuple with the holder backoff ladder and the PCM convert wait event; it never touches another lifecycle's token/flags. Sleeping is refused under the nested-wait guard (a held frozen-tag content lock) and bounded at 64 waits — both fall back to the historical fail-closed report. BUSY observations surface through pcm_x_own_busy_count. Both legacy begin sites (master reservation + W1 revalidate) route through the helper; the source-scan contracts follow the new call. --- src/backend/storage/buffer/bufmgr.c | 46 +++++++++++++++++-- .../test_cluster_pcm_direct_init.c | 2 +- src/test/cluster_unit/test_cluster_pcm_own.c | 13 ++++-- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 7b2275c4ac..77a2ad138d 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -1239,6 +1239,46 @@ cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, CHECK_FOR_INTERRUPTS(); } +/* + * PGRAC (t/400 L3 review R2 P0-2): a legacy grant reservation that observes + * a live GRANT_PENDING/REVOKING lifecycle sits in a normal ms-scale revoke or + * grant window — for a reader (or a rare un-valid-page writer) that is a + * transient to wait out off-lock, not a client ERROR. The retry is only a + * fresh begin against the re-sampled complete ownership tuple; another + * lifecycle's token/flags are never touched. No content lock is held at + * either call site, and a backend holding OTHER frozen-tag content locks + * must not sleep (nested-guard discipline) — both the guard refusal and the + * bounded-wait exhaustion fall back to the historical fail-closed report in + * the caller. + */ +#define CLUSTER_BUFMGR_PCM_RESERVATION_MAX_WAITS 64 + +static ClusterPcmOwnResult +cluster_bufmgr_pcm_begin_grant_reservation_wait(BufferDesc *buf, + ClusterPcmOwnSnapshot *base_out, + uint64 *token_out) +{ + ClusterPcmOwnResult result; + uint32 waits = 0; + + for (;;) + { + result = cluster_pcm_own_begin_grant_reservation(buf, base_out, token_out); + if (result != CLUSTER_PCM_OWN_BUSY + || waits >= CLUSTER_BUFMGR_PCM_RESERVATION_MAX_WAITS) + return result; + if (cluster_pcm_x_nested_wait_guard_before_block() != PCM_X_QUEUE_OK) + return result; + cluster_pcm_x_stats_note_own_busy(); + CHECK_FOR_INTERRUPTS(); + (void) WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + cluster_pcm_x_holder_retry_delay_ms(waits), + WAIT_EVENT_PCM_BLOCK_CONVERT_WAIT); + CHECK_FOR_INTERRUPTS(); + waits++; + } +} + static ClusterPcmXHolderLedgerEntry * cluster_bufmgr_pcm_x_holder_find(BufferDesc *buf) { @@ -8074,8 +8114,8 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) * JOIN/WAIT and call begin_x_reservation only from * ACTIVE_TRANSFER/PREPARE. */ - pcm_pending_result = cluster_pcm_own_begin_grant_reservation(buf, &pcm_pending_base, - &pcm_pending_token); + pcm_pending_result = cluster_bufmgr_pcm_begin_grant_reservation_wait( + buf, &pcm_pending_base, &pcm_pending_token); if (pcm_pending_result != CLUSTER_PCM_OWN_OK) cluster_pcm_own_report_bump_failure( buf, pcm_pending_result, pcm_pending_base.generation, @@ -8188,7 +8228,7 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) cluster_pcm_note_writer_cover_stale_detected(); pcm_covered = false; LWLockRelease(BufferDescriptorGetContentLock(buf)); - pcm_pending_result = cluster_pcm_own_begin_grant_reservation( + pcm_pending_result = cluster_bufmgr_pcm_begin_grant_reservation_wait( buf, &pcm_pending_base, &pcm_pending_token); if (pcm_pending_result != CLUSTER_PCM_OWN_OK) cluster_pcm_own_report_bump_failure( diff --git a/src/test/cluster_unit/test_cluster_pcm_direct_init.c b/src/test/cluster_unit/test_cluster_pcm_direct_init.c index 61b826f32e..cfb8052722 100644 --- a/src/test/cluster_unit/test_cluster_pcm_direct_init.c +++ b/src/test/cluster_unit/test_cluster_pcm_direct_init.c @@ -373,7 +373,7 @@ UT_TEST(test_valid_n_s_x_without_proof_enters_queue_before_legacy_wire) char *source = read_source(BUFMGR_SOURCE_PATH); static const char *const order[] = { "pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode,", - "cluster_pcm_own_begin_grant_reservation(buf, &pcm_pending_base", + "cluster_bufmgr_pcm_begin_grant_reservation_wait(", "cluster_pcm_lock_acquire_buffer(buf, pcm_mode)" }; UT_ASSERT(source != NULL); diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index aeceb6b128..2b226cfdee 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -776,11 +776,13 @@ UT_TEST(test_bufmgr_generation_bump_failure_is_classified_under_header_lock) UT_TEST(test_lockbuffer_reservation_failures_use_busy_corrupt_classifier) { static const char *const initial_reservation_contract[] - = { "cluster_pcm_own_begin_grant_reservation", "pcm_pending_result != CLUSTER_PCM_OWN_OK", - "cluster_pcm_own_report_bump_failure", "pcm_pending_set = true" }; + = { "cluster_bufmgr_pcm_begin_grant_reservation_wait", + "pcm_pending_result != CLUSTER_PCM_OWN_OK", "cluster_pcm_own_report_bump_failure", + "pcm_pending_set = true" }; static const char *const revalidate_reservation_contract[] - = { "cluster_pcm_own_begin_grant_reservation", "pcm_pending_result != CLUSTER_PCM_OWN_OK", - "cluster_pcm_own_report_bump_failure", "pcm_pending_set = true" }; + = { "cluster_bufmgr_pcm_begin_grant_reservation_wait", + "pcm_pending_result != CLUSTER_PCM_OWN_OK", "cluster_pcm_own_report_bump_failure", + "pcm_pending_set = true" }; char *source = read_bufmgr_source(); /* A live queue reservation is BUSY, not damaged metadata. Both legacy @@ -1207,7 +1209,8 @@ UT_TEST(test_retained_image_release_and_writeback_gates_are_exact) UT_ASSERT(strstr(hint, "cluster_bufmgr_pcm_x_retained_image_locked") < strstr(hint, "BM_DIRTY | BM_JUST_DIRTIED")); if (lockbuffer != NULL) { - const char *reserve = strstr(lockbuffer, "cluster_pcm_own_begin_grant_reservation("); + const char *reserve + = strstr(lockbuffer, "cluster_bufmgr_pcm_begin_grant_reservation_wait("); const char *content = strstr(lockbuffer, "LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED)"); const char *w1_revoke From 832019b6ca064d6ffe1ffd26b5972e43846cede2 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 03:14:21 -0700 Subject: [PATCH 26/56] fix(cluster): hand the holder-gate BARRIER_CLOSED refusal to the barrier-aware caller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit t/400 L3 — the next liveness layer surfaced once the self-V2 wedge fell: the pre-crit VM acquire got its writer X granted, then the holder-gate register hit a transient (busy holder lane) and its retry wait consulted the nested guard while the backend still held the frozen-tag heap lock. The guard refusal escalated to a client ERROR ('holder operation is not ready, operation=retry wait nested guard result=13'), killing one t/400 writer per run. The holder lane now speaks the same caller-owned-unwind protocol as the writer prepare: cluster_bufmgr_pcm_x_holder_retry_wait() reports a BARRIER_CLOSED guard verdict back through the new barrier_refused out-param (counting it in pcm_x_queue_barrier_unwind_count) instead of throwing; holder_prepare unwinds its untouched ledger slot; and LockBufferInternal rolls the already-granted writer claim back through the exact ERROR-path abort before handing the refusal to heapam, which releases its own outer lock, warms the conversion unlocked and re-enters. Every other guard verdict and every non-aware call site (deferred detach, unregister, release, W1 reacquire) keeps the historical fail-closed report. --- src/backend/storage/buffer/bufmgr.c | 53 +++++++++++++++++--- src/test/cluster_unit/test_cluster_pcm_own.c | 6 +-- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 77a2ad138d..2480977f33 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -1209,7 +1209,7 @@ static void cluster_bufmgr_pcm_direct_init_snapshot_locked(BufferDesc *buf, uint static void cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, - uint32 wait_index) + uint32 wait_index, bool *barrier_refused) { long delay_ms; PcmXQueueResult guard_result; @@ -1230,8 +1230,23 @@ cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, */ guard_result = cluster_pcm_x_nested_wait_guard_before_block(); if (guard_result != PCM_X_QUEUE_OK) + { + /* + * PGRAC (t/400 L3 item 3, holder lane): a barrier-aware caller owns + * the unwind for the frozen-barrier refusal — it releases its own + * outer lock and resolves the conversion unlocked. Every other + * guard verdict, and every non-aware caller, keeps the historical + * fail-closed report. + */ + if (barrier_refused != NULL && guard_result == PCM_X_QUEUE_BARRIER_CLOSED) + { + *barrier_refused = true; + cluster_pcm_x_stats_note_barrier_unwind(); + return; + } cluster_bufmgr_pcm_x_holder_report_failure( guard_result, GetBufferDescriptor(buffer_id), "retry wait nested guard"); + } delay_ms = cluster_pcm_x_holder_retry_delay_ms(wait_index); CHECK_FOR_INTERRUPTS(); (void) WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, delay_ms, @@ -1457,7 +1472,7 @@ cluster_bufmgr_pcm_x_holder_drain_deferred(ClusterPcmXHolderLedgerEntry *entry) } cluster_bufmgr_pcm_x_holder_retry_wait( entry->content_lock, entry->buffer_id, - wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); + wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS, NULL); wait_index++; if (wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS == 0) { @@ -1475,7 +1490,7 @@ cluster_bufmgr_pcm_x_holder_drain_deferred(ClusterPcmXHolderLedgerEntry *entry) } static ClusterPcmXHolderLedgerEntry * -cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf) +cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf, bool *barrier_refused) { ClusterPcmXHolderLedgerEntry *entry; ClusterPcmXWriterLedgerEntry *writer_entry; @@ -1636,7 +1651,9 @@ cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf) cluster_bufmgr_pcm_x_holder_report_failure(result, buf, "register"); cluster_bufmgr_pcm_x_holder_retry_wait( content_lock, buf->buf_id, - wait_index++ % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); + wait_index++ % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS, barrier_refused); + if (barrier_refused != NULL && *barrier_refused) + return NULL; } return entry; @@ -1716,7 +1733,7 @@ cluster_bufmgr_pcm_x_holder_unregister(ClusterPcmXHolderLedgerEntry *entry) result, GetBufferDescriptor(entry->buffer_id), "unregister"); } cluster_bufmgr_pcm_x_holder_retry_wait(entry->content_lock, entry->buffer_id, - waits_used++); + waits_used++, NULL); } } @@ -2096,7 +2113,8 @@ cluster_bufmgr_pcm_x_writer_release(ClusterPcmXWriterLedgerEntry *entry) } cluster_bufmgr_pcm_x_holder_retry_wait(entry->content_lock, entry->buffer_id, waits_used++ - % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); + % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS, + NULL); } } @@ -8190,7 +8208,20 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) CLUSTER_INJECTION_POINT("cluster-pcm-writer-cached-x-stall"); PG_TRY(); { - pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf); + pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf, pcm_barrier_refused); + if (pcm_barrier_refused != NULL && *pcm_barrier_refused) + { + /* + * PGRAC (t/400 L3 item 3, holder lane): the holder gate + * refused under the frozen barrier. Roll the granted writer + * claim back exactly like the ERROR path would and take + * nothing; the barrier-aware caller owns the unwind. + */ + if (pcm_x_writer != NULL) + cluster_bufmgr_pcm_x_writer_abort_acquiring(pcm_x_writer); + } + else + { #endif if (mode == BUFFER_LOCK_SHARE) LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED); @@ -8236,7 +8267,7 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) pcm_pending_base.flags, "LockBuffer revalidate reservation"); pcm_pending_set = true; pcm_acquired = cluster_pcm_lock_acquire_buffer(buf, pcm_mode); - pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf); + pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf, NULL); if (mode == BUFFER_LOCK_SHARE) LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED); else @@ -8245,6 +8276,7 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) } } cluster_bufmgr_pcm_x_holder_activate(pcm_x_holder); + } } PG_CATCH(); { @@ -8289,6 +8321,11 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) } PG_END_TRY(); + /* PGRAC (t/400 L3 item 3, holder lane): nothing is held and the + * writer claim was rolled back — hand the refusal to the caller. */ + if (pcm_barrier_refused != NULL && *pcm_barrier_refused) + return; + /* * PGRAC ownership-generation wave (W3) — grant-finalize window. A * real X acquire has installed the grant but pcm_state is still N diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index 2b226cfdee..e7a08e2ce8 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -1319,10 +1319,10 @@ UT_TEST(test_lockbuffer_pcm_x_holder_ledger_brackets_both_content_acquires) "LWLockRelease(BufferDescriptorGetContentLock(buf))", "cluster_bufmgr_pcm_x_holder_unregister" }; static const char *const acquire_contract[] - = { "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf)", + = { "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf,", "LWLockAcquire(BufferDescriptorGetContentLock(buf)", "LWLockRelease(BufferDescriptorGetContentLock(buf))", - "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf)", + "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf,", "LWLockAcquire(BufferDescriptorGetContentLock(buf)", "cluster_bufmgr_pcm_x_holder_activate(pcm_x_holder)" }; char *source = read_bufmgr_source(); @@ -1877,7 +1877,7 @@ UT_TEST(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_aut static const char *const acquire_contract[] = { "pcm_covered = cluster_pcm_x_cached_cover_bypasses_queue(", "pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode,", - "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf)", + "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf,", "LWLockAcquire(BufferDescriptorGetContentLock(buf)", "cluster_bufmgr_pcm_x_writer_activate(pcm_x_writer)" }; static const char *const cleanup_contract[] From f042473282805dc17892fcf25650d718847597d2 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 04:00:51 -0700 Subject: [PATCH 27/56] fix(cluster): log the refused self-source DRAIN proof before fusing the node The t/400 drain-leg wedge froze with the master DRAIN leg armed at the requester node forever: that node fused at the self-source DRAIN proof arm (cluster_gcs_block.c fail-closed site), swallowed the ack, and every later poll died silently on the runtime gate. The fail-closed record only names file:line, so the refusal sub-arm - generation mismatch versus BUSY versus an evicted buffer - is invisible, and each sub-arm needs a different repair. Capture a one-shot descriptor snapshot (generation, token, own flags, pcm_state, buffer_type, BM_VALID, tag match) inside the existing header lock in cluster_bufmgr_pcm_own_self_handoff_x_exact and log it at the caller right before the fuse. Snapshot fill only under the spinlock; the ereport runs after both locks are released. No shmem or wire ABI change; the sample struct is process-local. --- src/backend/cluster/cluster_gcs_block.c | 34 +++++++++++++++++----- src/backend/storage/buffer/bufmgr.c | 20 +++++++++++-- src/include/cluster/cluster_pcm_x_bufmgr.h | 18 ++++++++++-- 3 files changed, 59 insertions(+), 13 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 8288b96fba..66bae0d53d 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -11835,6 +11835,8 @@ gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, PcmXLocalHolderProgress progress; PcmXQueueResult progress_result; PcmXQueueResult result; + ClusterPcmOwnSelfHandoffSample handoff_sample; + ClusterPcmOwnResult handoff_result; ClusterPcmXRevokeFinishMode finish_mode; uint64 holder_image_id = 0; bool holder_image = false; @@ -11888,14 +11890,30 @@ gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, return PCM_X_QUEUE_OK; self_source_handoff = binding.identity.image.source_node == (uint32)cluster_node_id && poll->ref.identity.node_id == cluster_node_id; - if (self_source_handoff - && cluster_bufmgr_pcm_own_self_handoff_x_exact(&poll->ref.identity.tag, - binding.identity.image.source_own_generation) - != CLUSTER_PCM_OWN_OK) { - /* FINAL makes the in-place X descriptor authoritative before DRAIN. - * Preserve the immutable record if that exact proof is missing. */ - cluster_pcm_x_runtime_fail_closed(); - return PCM_X_QUEUE_CORRUPT; + if (self_source_handoff) { + handoff_result = cluster_bufmgr_pcm_own_self_handoff_x_exact( + &poll->ref.identity.tag, binding.identity.image.source_own_generation, &handoff_sample); + if (handoff_result != CLUSTER_PCM_OWN_OK) { + /* FINAL makes the in-place X descriptor authoritative before DRAIN. + * Preserve the immutable record if that exact proof is missing. + * The refusal shape is one-shot evidence: this arm fuses the node, + * so the descriptor snapshot must reach the log before it does. */ + ereport( + LOG, + (errmsg("PCM-X self-source DRAIN proof refused"), + errdetail("result=%d source_gen=%llu live_gen=%llu own_flags=%u " + "own_token=%llu pcm_state=%u buffer_type=%u bm_valid=%d found=%d", + (int)handoff_result, + (unsigned long long)binding.identity.image.source_own_generation, + (unsigned long long)handoff_sample.live_generation, + (unsigned int)handoff_sample.own_flags, + (unsigned long long)handoff_sample.live_token, + (unsigned int)handoff_sample.pcm_state, + (unsigned int)handoff_sample.buffer_type, + handoff_sample.bm_valid ? 1 : 0, handoff_sample.buffer_found ? 1 : 0))); + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } } if (cluster_gcs_block_dedup_pcm_x_release_exact(worker_id, &key, &poll->ref.identity.tag, &binding) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 2480977f33..a1a6afbd6f 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -12015,19 +12015,23 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, * handoff. This is intentionally read-only: a delayed DRAIN may release its * immutable A-record, but it must never mutate the current X descriptor. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, uint64 source_generation) +cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, uint64 source_generation, + ClusterPcmOwnSelfHandoffSample *sample_out) { ClusterPcmOwnResult live_result; ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; BufferDesc *buf; BufferTag lookup_tag; LWLock *partition_lock; + uint64 live_generation; uint64 live_token; uint32 flags; uint32 hashcode; uint32 buf_state; int buf_id; + if (sample_out != NULL) + memset(sample_out, 0, sizeof(*sample_out)); if (tag == NULL || source_generation == UINT64_MAX) return CLUSTER_PCM_OWN_INVALID; if (ClusterPcmOwnArray == NULL) @@ -12043,11 +12047,21 @@ cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, uint64 source_ } buf = GetBufferDescriptor(buf_id); buf_state = LockBufHdr(buf); + live_generation = cluster_pcm_own_gen_get(buf->buf_id); live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); - if (!BufferTagsEqual(&buf->tag, tag) - || cluster_pcm_own_gen_get(buf->buf_id) != source_generation + 1) + if (sample_out != NULL) { + /* Snapshot under the header lock only; the caller logs after unlock. */ + sample_out->live_generation = live_generation; + sample_out->live_token = live_token; + sample_out->own_flags = flags; + sample_out->pcm_state = buf->pcm_state; + sample_out->buffer_type = buf->buffer_type; + sample_out->bm_valid = (buf_state & BM_VALID) != 0; + sample_out->buffer_found = BufferTagsEqual(&buf->tag, tag); + } + if (!BufferTagsEqual(&buf->tag, tag) || live_generation != source_generation + 1) result = CLUSTER_PCM_OWN_STALE; else if (live_result == CLUSTER_PCM_OWN_CORRUPT) result = live_result; diff --git a/src/include/cluster/cluster_pcm_x_bufmgr.h b/src/include/cluster/cluster_pcm_x_bufmgr.h index fbb639caa2..6a6f868c55 100644 --- a/src/include/cluster/cluster_pcm_x_bufmgr.h +++ b/src/include/cluster/cluster_pcm_x_bufmgr.h @@ -362,10 +362,24 @@ extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_finish_revoke_retain( ClusterPcmOwnSnapshot *out_retained); extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, uint64 source_generation); +/* Process-local descriptor evidence captured by the DRAIN proof below so the + * caller can log the exact refusal shape before deciding fail-closed. The + * fields are one header-locked snapshot; never persisted or sent on wire. */ +typedef struct ClusterPcmOwnSelfHandoffSample { + uint64 live_generation; + uint64 live_token; + uint32 own_flags; + uint8 pcm_state; + uint8 buffer_type; + bool bm_valid; + bool buffer_found; +} ClusterPcmOwnSelfHandoffSample; + /* Read-only DRAIN proof for a sole-requester source that committed S->X in * place. Exact base+1 X/XCUR authority means DRAIN may release only the * immutable dedup record and must not invalidate this descriptor. */ -extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, - uint64 source_generation); +extern ClusterPcmOwnResult +cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, uint64 source_generation, + ClusterPcmOwnSelfHandoffSample *sample_out); #endif /* CLUSTER_PCM_X_BUFMGR_H */ From 440e9633682f4cf7276cac7dba9e1b91792bddd5 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 04:17:19 -0700 Subject: [PATCH 28/56] fix(cluster): count master leg re-arms and log a repeating drive stall The t/400 wedge dumps print leg_retry from the master ticket, but every re-drive path re-arms its leg through the DUPLICATE case without touching retry_count, so a frozen dump shows leg_retry=0 whether the retry pump is re-staging every tick or died entirely - the two failure families that need opposite repairs. Count each exact DUPLICATE re-arm (terminal DRAIN/RETIRE legs and the transfer REVOKE leg, saturation-guarded like the local resend counter) so leg_retry becomes that discriminator, and give the periodic master drive a log-once note that fires when the same non-progress result repeats consecutively for the same ticket, naming the refusing arm. Progress results stay silent and reset the streak. Unit: the wire 49-56 and terminal-ack tests now pin the increment (recorded red at 0 != 1 / 0 != 2 before the counting change). --- src/backend/cluster/cluster_gcs_block.c | 48 +++++++++++++++++-- src/backend/cluster/cluster_pcm_x_convert.c | 21 ++++++-- .../cluster_unit/test_cluster_pcm_x_convert.c | 7 +++ 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 66bae0d53d..8ff3a50b19 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -9850,12 +9850,44 @@ gcs_block_pcm_x_master_drive_cancel_requested(const PcmXMasterDriveSnapshot *sna } +/* Log-once evidence for the periodic master drive: a non-progress result + * that repeats consecutively with the same shape is a stalled drive, and a + * frozen wedge must name its refusing arm. Progress resets the streak. */ +static void +gcs_block_pcm_x_master_drive_note(const char *stage, PcmXQueueResult result, uint64 ticket_id) +{ + static const char *last_stage = NULL; + static uint64 last_ticket = 0; + static int last_result = 0; + static bool logged = false; + + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + last_stage = NULL; + logged = false; + return; + } + if (stage == last_stage && (int)result == last_result && ticket_id == last_ticket) { + if (!logged) { + logged = true; + ereport(LOG, (errmsg("PCM-X master drive is repeating %s (result %d, ticket %llu)", + stage, (int)result, (unsigned long long)ticket_id))); + } + return; + } + last_stage = stage; + last_result = (int)result; + last_ticket = ticket_id; + logged = false; +} + + static void gcs_block_pcm_x_master_drive_tag(const BufferTag *tag, uint64 cluster_epoch) { PcmXMasterDriveSnapshot snapshot; PcmXTicketRef active; PcmXQueueResult result; + const char *drive_stage; if (tag == NULL || cluster_epoch != cluster_epoch_get_current() || cluster_node_id < 0 || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT @@ -9871,29 +9903,37 @@ gcs_block_pcm_x_master_drive_tag(const BufferTag *tag, uint64 cluster_epoch) if (result == PCM_X_QUEUE_NOT_FOUND) cluster_gcs_pcm_x_drive_anomaly_settle(gcs_block_pcm_x_drive_anomaly_table, GCS_BLOCK_PCM_X_DRIVE_ANOMALY_SLOTS, tag); + gcs_block_pcm_x_master_drive_note("promote", result, 0); gcs_block_pcm_x_master_drive_fail_closed(result); return; } result = cluster_pcm_x_master_drive_snapshot_exact(tag, cluster_epoch, &snapshot); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK) { + gcs_block_pcm_x_master_drive_note("snapshot", result, 0); gcs_block_pcm_x_master_drive_fail_closed(result); return; } if (snapshot.ticket_state == PCM_XT_ACTIVE_PROBE && snapshot.flags == (PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED - | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED)) + | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED)) { + drive_stage = "cancel-requested"; result = gcs_block_pcm_x_master_drive_cancel_requested(&snapshot); - else if (snapshot.ticket_state == PCM_XT_ACTIVE_PROBE) { + } else if (snapshot.ticket_state == PCM_XT_ACTIVE_PROBE) { + drive_stage = "probe"; result = gcs_block_pcm_x_ensure_pending_x_claim(&snapshot); if (result == PCM_X_QUEUE_OK) result = gcs_block_pcm_x_master_drive_probe(&snapshot); - } else if (snapshot.ticket_state == PCM_XT_ACTIVE_TRANSFER) + } else if (snapshot.ticket_state == PCM_XT_ACTIVE_TRANSFER) { + drive_stage = "transfer"; result = gcs_block_pcm_x_master_drive_transfer(&snapshot); - else + } else { + drive_stage = "state"; result = PCM_X_QUEUE_CORRUPT; + } cluster_pcm_x_stats_note_queue_result(result); + gcs_block_pcm_x_master_drive_note(drive_stage, result, snapshot.ref.handle.ticket_id); /* Only definite drive progress settles the tag; indeterminate results * (NOT_READY / BUSY / STALE) must not reset a live streak, or a real * wedge interleaved with transients would never fuse. */ diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index dac13fc296..555376fab5 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -7378,9 +7378,13 @@ cluster_pcm_x_master_revoke_arm_exact(const PcmXTicketRef *ref, int32 responder_ else if (!pcm_x_transfer_leg_is_clear(&ticket->reliable)) { if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_REVOKE, responder_node, responder_session) - && ticket->image.image_id != 0 && ticket->image.source_node == (uint32)responder_node) + && ticket->image.image_id != 0 && ticket->image.source_node == (uint32)responder_node) { + /* Diagnostic: the transfer driver re-arms this exact REVOKE leg on + * every pass; count it so leg_retry exposes a live retry pump. */ + if (ticket->reliable.retry_count != PG_UINT32_MAX) + ticket->reliable.retry_count++; result = PCM_X_QUEUE_DUPLICATE; - else + } else result = PCM_X_QUEUE_BUSY; } else if (ticket->image.image_id != 0) { result = ticket->reliable.last_response_opcode == PGRAC_IC_MSG_PCM_X_IMAGE_READY @@ -7421,9 +7425,13 @@ cluster_pcm_x_master_revoke_arm_exact(const PcmXTicketRef *ref, int32 responder_ else if (!pcm_x_transfer_leg_is_clear(&ticket->reliable)) { if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_REVOKE, responder_node, responder_session) - && ticket->image.image_id != 0 && ticket->image.source_node == (uint32)responder_node) + && ticket->image.image_id != 0 && ticket->image.source_node == (uint32)responder_node) { + /* Diagnostic: the transfer driver re-arms this exact REVOKE leg on + * every pass; count it so leg_retry exposes a live retry pump. */ + if (ticket->reliable.retry_count != PG_UINT32_MAX) + ticket->reliable.retry_count++; result = PCM_X_QUEUE_DUPLICATE; - else + } else result = PCM_X_QUEUE_BUSY; } else if (ticket->image.image_id != 0) result = PCM_X_QUEUE_CORRUPT; @@ -8900,6 +8908,11 @@ cluster_pcm_x_master_terminal_leg_arm_exact(const PcmXTicketRef *ref, PcmXTermin = kind == PCM_X_TERMINAL_LEG_DRAIN ? ticket->drain_generation : 0; token_out->expected_responder_node = responder_node; token_out->kind = (uint16)kind; + /* Diagnostic: an exact re-arm is the retry pump re-staging this + * leg. The count reaches the per-slot dump as leg_retry, so a + * frozen wedge separates a dead pump from a refusing responder. */ + if (ticket->reliable.retry_count != PG_UINT32_MAX) + ticket->reliable.retry_count++; result = PCM_X_QUEUE_DUPLICATE; } else result = PCM_X_QUEUE_BUSY; diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 4dc639c781..228194316b 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -6275,6 +6275,11 @@ UT_TEST(test_master_transfer_wire_49_56_is_generation_exact) UT_ASSERT_EQ(header->next_image_id, 2); UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(&transfer, 2, source_session, &revoke), PCM_X_QUEUE_DUPLICATE); + /* Each exact re-arm is one retry-pump pass; leg_retry must count it. */ + UT_ASSERT_EQ(ticket->reliable.retry_count, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(&transfer, 2, source_session, &revoke), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(ticket->reliable.retry_count, 2); memset(&image_ready, 0, sizeof(image_ready)); image_ready.ref = transfer; @@ -8044,6 +8049,8 @@ UT_TEST(test_terminal_ack_wire_fields_are_exact_and_zero_side_effect) request.prehandle.sender_session_incarnation, &replay), PCM_X_QUEUE_DUPLICATE); UT_ASSERT(memcmp(&replay, &drain, sizeof(replay)) == 0); + /* Each exact re-arm is one retry-pump pass; leg_retry must count it. */ + UT_ASSERT_EQ(ticket->reliable.retry_count, 1); before = *ticket; UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, 1, From 6d20a13c2b0a370bb7550b2d6386dee12ceddcf9 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 04:26:08 -0700 Subject: [PATCH 29/56] fix(cluster): log the live descriptor lifecycle behind a repeating REVOKE refusal The transfer wedge shape is now proven live on the master side (the REVOKE leg re-arms every drive pass, leg_retry climbing) while the source node refuses the apply forever with own_busy ticking at exactly the resend cadence. Every source-side refusal arm - the ingress own snapshot, the materialize holder snapshot, and the materialize begin - returns silently, so the lifecycle that is holding the descriptor never names itself. Give those arms a shared log-once note that fires when the same refusal shape repeats consecutively, carrying the own snapshot (pcm_state, flags, generation, token). The flags value names the stuck lifecycle directly. A successful revoke apply resets the streak. --- src/backend/cluster/cluster_gcs_block.c | 75 +++++++++++++++++++++---- 1 file changed, 65 insertions(+), 10 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 8ff3a50b19..0b2f2a924f 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -11192,6 +11192,9 @@ gcs_block_pcm_x_stage_ready_work(int worker_id, const GcsBlockPcmXImageWork *wor } +static void gcs_block_pcm_x_revoke_refusal_note(const char *site, int own_result, + const ClusterPcmOwnSnapshot *snap); + static void gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImageWork *work) { @@ -11218,10 +11221,16 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage int buffer_id; holder_result = cluster_pcm_x_local_holder_snapshot(&work->tag, NULL, 0, &holder_snapshot); - if (holder_result == PCM_X_QUEUE_NO_CAPACITY && holder_snapshot.holder_count > 0) + if (holder_result == PCM_X_QUEUE_NO_CAPACITY && holder_snapshot.holder_count > 0) { + gcs_block_pcm_x_revoke_refusal_note("materialize-holder-capacity", (int)holder_result, + NULL); return; - if (holder_result == PCM_X_QUEUE_NOT_READY || holder_result == PCM_X_QUEUE_BUSY) + } + if (holder_result == PCM_X_QUEUE_NOT_READY || holder_result == PCM_X_QUEUE_BUSY) { + gcs_block_pcm_x_revoke_refusal_note("materialize-holder-snapshot", (int)holder_result, + NULL); return; + } if (holder_result != PCM_X_QUEUE_OK || holder_snapshot.holder_count != 0) { cluster_pcm_x_runtime_fail_closed(); return; @@ -11229,8 +11238,10 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage own_result = cluster_bufmgr_pcm_own_snapshot_by_tag(&work->tag, &buffer_id, ¤t); gcs_block_pcm_x_note_own_result(own_result); - if (own_result == CLUSTER_PCM_OWN_NOT_READY) + if (own_result == CLUSTER_PCM_OWN_NOT_READY) { + gcs_block_pcm_x_revoke_refusal_note("materialize-snapshot", (int)own_result, ¤t); return; + } source_is_n = current.pcm_state == (uint8)PCM_STATE_N; source_is_s = current.pcm_state == (uint8)PCM_STATE_S; source_is_x = current.pcm_state == (uint8)PCM_STATE_X; @@ -11252,8 +11263,10 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage else own_result = cluster_bufmgr_pcm_own_begin_x_revoke(buf, ¤t, &revoking); gcs_block_pcm_x_note_own_result(own_result); - if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY) + if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY) { + gcs_block_pcm_x_revoke_refusal_note("materialize-begin", (int)own_result, ¤t); return; + } if (own_result != CLUSTER_PCM_OWN_OK) { (void)gcs_block_pcm_x_release_image_exact(worker_id, work, &work->binding); cluster_pcm_x_runtime_fail_closed(); @@ -11419,6 +11432,45 @@ cluster_gcs_block_pcm_x_image_pump_tick(int worker_id) /* 49 only installs the holder-side revoke ledger and wakes exact registered * holders. It never copies a page, drops ownership, or synthesizes type 50 * inside the IC handler. */ +/* Log-once evidence for a repeating source-side REVOKE refusal: the live + * descriptor lifecycle blocking the transfer leg must name itself, because + * every refusal arm here is silent and the master keeps re-sending forever. + * A NULL site resets the streak (the revoke made progress). */ +static void +gcs_block_pcm_x_revoke_refusal_note(const char *site, int own_result, + const ClusterPcmOwnSnapshot *snap) +{ + static const char *last_site = NULL; + static uint32 last_flags = 0; + static int last_result = 0; + static bool logged = false; + uint32 flags = snap != NULL ? snap->flags : 0; + + if (site == NULL) { + last_site = NULL; + logged = false; + return; + } + if (site == last_site && own_result == last_result && flags == last_flags) { + if (!logged) { + logged = true; + ereport(LOG, + (errmsg("PCM-X source revoke is repeating a refusal at %s", site), + errdetail("result=%d pcm_state=%u own_flags=%u generation=%llu token=%llu", + own_result, (unsigned int)(snap != NULL ? snap->pcm_state : 0), + (unsigned int)flags, + (unsigned long long)(snap != NULL ? snap->generation : 0), + (unsigned long long)(snap != NULL ? snap->reservation_token : 0)))); + } + return; + } + last_site = site; + last_result = own_result; + last_flags = flags; + logged = false; +} + + static void cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const void *payload) { @@ -11511,8 +11563,10 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi source_is_s = own_snapshot.pcm_state == (uint8)PCM_STATE_S; source_is_x = own_snapshot.pcm_state == (uint8)PCM_STATE_X; if (own_result != CLUSTER_PCM_OWN_OK || (!source_is_n && !source_is_s && !source_is_x) - || own_snapshot.flags != 0) + || own_snapshot.flags != 0) { + gcs_block_pcm_x_revoke_refusal_note("ingress-snapshot", (int)own_result, &own_snapshot); return; + } source_generation = own_snapshot.generation; } gcs_block_pcm_x_reserved_binding(revoke, source_generation, source_session, &reserved_binding); @@ -11534,12 +11588,13 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi } result = cluster_pcm_x_local_holder_revoke_apply_exact(revoke, source_node, source_session); cluster_pcm_x_stats_note_queue_result(result); - if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + gcs_block_pcm_x_revoke_refusal_note(NULL, 0, NULL); gcs_block_pcm_x_wake_registered_holders(&revoke->ref.identity.tag); - else if (new_reservation - && cluster_gcs_block_dedup_pcm_x_release_exact( - worker_id, &image_key, &revoke->ref.identity.tag, &reserved_binding) - != GCS_BLOCK_PCM_X_IMAGE_RELEASED) + } else if (new_reservation + && cluster_gcs_block_dedup_pcm_x_release_exact( + worker_id, &image_key, &revoke->ref.identity.tag, &reserved_binding) + != GCS_BLOCK_PCM_X_IMAGE_RELEASED) cluster_pcm_x_runtime_fail_closed(); } From 089d3ce7a31ae827cf202e9e6653e3a6f8c3837e Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 04:43:57 -0700 Subject: [PATCH 30/56] fix(cluster): walk every terminal ticket per retry tick, oldest first t/400 froze with a proven cross-ticket interlock: the oldest ticket sat in RETIRE_CREDIT with its RETIRE leg re-armed 114 times while the responder refused every poll, because that node's holder lane was still bound to the NEXT ticket's transfer (holder terminal not drained) and the RETIRE preflight's cross-lane guard correctly returns NOT_READY for an undrained holder. The younger ticket's own DRAIN leg sat armed with zero retries: the terminal retry tick drove only the single oldest terminal ticket, so the very DRAIN evidence the head was waiting for could never be collected once its one-shot event chain lost a message. Give the periodic driver a floor-scoped scan (cluster_pcm_x_master_terminal_work_next_after) and walk the terminal set in ticket-id order each tick, bounded by a small budget. The oldest ticket is still kicked first, so the contiguous retirement frontier keeps its priority; younger tickets merely regain the same at-least-once re-drive the head already had. Unit: a three-ticket fixture pins oldest-first, floor iteration, and NOT_FOUND past the tail (recorded red as a missing-symbol build before the implementation). --- src/backend/cluster/cluster_gcs_block.c | 18 +++++++-- src/backend/cluster/cluster_pcm_x_convert.c | 14 ++++++- src/include/cluster/cluster_pcm_x_convert.h | 2 + .../cluster_unit/test_cluster_pcm_x_convert.c | 37 ++++++++++++++++++- 4 files changed, 65 insertions(+), 6 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 0b2f2a924f..e8cc958a53 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -12393,15 +12393,25 @@ gcs_block_pcm_x_master_drive_retry_tick(void) } +#define GCS_BLOCK_PCM_X_TERMINAL_TICK_BUDGET 8 + static void gcs_block_pcm_x_terminal_retry_tick(void) { PcmXTicketRef ref; - PcmXQueueResult result; - - result = cluster_pcm_x_master_terminal_work_next(&ref); - if (result == PCM_X_QUEUE_OK) + uint64 after = 0; + int kicks; + + /* The oldest ticket keeps frontier priority, but a stuck head must not + * starve younger terminal tickets: the head's own RETIRE preflight can be + * waiting for exactly the younger ticket's DRAIN evidence (cross-lane + * holder interlock), so each tick walks the terminal set in id order. */ + for (kicks = 0; kicks < GCS_BLOCK_PCM_X_TERMINAL_TICK_BUDGET; kicks++) { + if (cluster_pcm_x_master_terminal_work_next_after(after, &ref) != PCM_X_QUEUE_OK) + return; cluster_gcs_pcm_x_terminal_kick(&ref); + after = ref.handle.ticket_id; + } } diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index 555376fab5..cd5302183a 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -9180,6 +9180,17 @@ cluster_pcm_x_master_retire_ack_resolve_exact(const PcmXRetirePayload *ack, * starve the global contiguous retirement frontier behind a newer slot. */ PcmXQueueResult cluster_pcm_x_master_terminal_work_next(PcmXTicketRef *ref_out) +{ + return cluster_pcm_x_master_terminal_work_next_after(0, ref_out); +} + + +/* Floor-scoped scan for the retry driver's fairness loop: the oldest ticket + * keeps frontier priority, but its stuck RETIRE preflight may be waiting for + * a younger ticket's DRAIN evidence (cross-lane holder interlock), so the + * driver walks every terminal ticket in ticket-id order per pass. */ +PcmXQueueResult +cluster_pcm_x_master_terminal_work_next_after(uint64 after_ticket_id, PcmXTicketRef *ref_out) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; PcmXRuntimeSnapshot runtime; @@ -9255,7 +9266,8 @@ cluster_pcm_x_master_terminal_work_next(PcmXTicketRef *ref_out) break; continue; } - if (ticket->ref.handle.ticket_id < best_ticket_id) { + if (ticket->ref.handle.ticket_id > after_ticket_id + && ticket->ref.handle.ticket_id < best_ticket_id) { best_ticket_id = ticket->ref.handle.ticket_id; *ref_out = ticket->ref; } diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h index ff7b7a9c18..fcafe71ca0 100644 --- a/src/include/cluster/cluster_pcm_x_convert.h +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -1518,6 +1518,8 @@ extern PcmXQueueResult cluster_pcm_x_master_retire_ack_resolve_exact(const PcmXR uint64 authenticated_session, PcmXTicketRef *ref_out); extern PcmXQueueResult cluster_pcm_x_master_terminal_work_next(PcmXTicketRef *ref_out); +extern PcmXQueueResult cluster_pcm_x_master_terminal_work_next_after(uint64 after_ticket_id, + PcmXTicketRef *ref_out); extern PcmXQueueResult cluster_pcm_x_master_detach_terminal_exact(const PcmXTicketRef *ref); extern PcmXQueueResult cluster_pcm_x_local_join_begin(const PcmXWaitIdentity *identity, int32 master_node, diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 228194316b..39f26c281f 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -7743,6 +7743,40 @@ UT_TEST(test_master_cancel_is_exact_and_unlinks_middle_without_fifo_damage) assert_master_queue_baseline(header); } +UT_TEST(test_master_terminal_work_scan_covers_younger_tickets) +{ + PcmXMasterAdmission admission[3]; + PcmXTicketRef work; + int i; + + init_active_pcm_x(UINT64_C(77)); + for (i = 0; i < 3; i++) { + PcmXEnqueuePayload request + = make_enqueue(make_wait_identity(707, i, (uint32)(24 + i), UINT64_C(6101) + i), + UINT64_C(1101) + i, UINT64_C(1)); + + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(300) + i), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[2].ref), PCM_X_QUEUE_OK); + /* Frontier priority: the plain scan still returns the oldest terminal. */ + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_work_next(&work), PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&work, &admission[1].ref)); + /* A stuck head must not hide younger terminal work from the retry tick: + * the head's own RETIRE preflight can be waiting for exactly the younger + * ticket's DRAIN evidence (cross-lane holder interlock). */ + UT_ASSERT_EQ( + cluster_pcm_x_master_terminal_work_next_after(admission[1].ref.handle.ticket_id, &work), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&work, &admission[2].ref)); + UT_ASSERT_EQ( + cluster_pcm_x_master_terminal_work_next_after(admission[2].ref.handle.ticket_id, &work), + PCM_X_QUEUE_NOT_FOUND); +} + UT_TEST(test_master_prehandle_cancel_replays_exactly_and_never_hits_reused_slot) { PcmXShmemHeader *header; @@ -15123,7 +15157,7 @@ UT_TEST(test_local_retire_episode_lock_errors_fail_closed) int main(void) { - UT_PLAN(257); + UT_PLAN(258); UT_RUN(test_image_id_domain_is_canonical_and_bounded); UT_RUN(test_wire_abi_sizes_are_exact); UT_RUN(test_wire_abi_offsets_are_exact); @@ -15263,6 +15297,7 @@ main(void) UT_RUN(test_runtime_fail_closed_records_winning_site_only); UT_RUN(test_master_debug_iterators_report_live_slots); UT_RUN(test_master_cancel_is_exact_and_unlinks_middle_without_fifo_damage); + UT_RUN(test_master_terminal_work_scan_covers_younger_tickets); UT_RUN(test_master_prehandle_cancel_replays_exactly_and_never_hits_reused_slot); UT_RUN(test_master_prehandle_identity_alias_is_corruption_not_stale_cancel); UT_RUN(test_master_prehandle_cancel_first_publishes_terminal_tombstone_before_late_enqueue); From c46f51475fc5d1c504db0e330f05fcacff006e6e Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 05:18:44 -0700 Subject: [PATCH 31/56] fix(cluster): release the self-source DRAIN record on the exact completion certificate The self-source DRAIN proof demanded the live descriptor still hold the freshly-committed shape (generation == source + 1, X/XCUR), but on a hot block the descriptor legitimately moves on between FINAL_ACK and the delayed DRAIN - an X->S downgrade serving a reader was caught on t/400 with live_gen = source + 2, pcm_state = S, buffer_type = XCUR - so the proof returned STALE and fused the whole node, swallowing the DRAIN ack forever (104 futile master retries) and freezing the queue until CSSD eviction. The descriptor is the wrong authority anyway: own generations are per-buf_id, not per-BufferTag lineage, so an evicted-and-reloaded tag could alias a larger generation without proving anything. Release the immutable record against the exact protocol ledger instead. The first (writer-lane) DRAIN consumption now captures a completion certificate under its own tag lock - ticket ref, image token, committed_own_generation, master identity - through the new cluster_pcm_x_local_drain_poll_certificate_exact variant (the plain call wraps it), and the release requires the certificate to match the holder binding exactly with committed == source generation + 1, the same invariant FINAL_ACK proved against the effective grant base. An inexact certificate preserves the record and refuses without fusing; the descriptor is only probed read-only for a structurally malformed flags/token shape, which alone still fails closed. Legitimate successor lifecycles, well-formed in-flight reservations, and eviction no longer fuse a delayed DRAIN. Unit: the transfer/final-ack lifecycle test pins the captured certificate fields and that a DUPLICATE replay does not fabricate one (recorded red at the valid assertion with the capture neutralized); the source-scan contracts pin the certificate policy and refusal arm ordering ahead of the record release, and the descriptor-probe contract replaces the old exact-shape pins. --- src/backend/cluster/cluster_gcs_block.c | 78 +++++++++++++------ src/backend/cluster/cluster_pcm_x_convert.c | 28 +++++++ src/backend/storage/buffer/bufmgr.c | 34 ++++---- src/include/cluster/cluster_pcm_x_bufmgr.h | 17 ++-- src/include/cluster/cluster_pcm_x_convert.h | 18 +++++ .../cluster_unit/test_cluster_gcs_block.c | 26 +++++-- src/test/cluster_unit/test_cluster_pcm_own.c | 12 +-- .../cluster_unit/test_cluster_pcm_x_convert.c | 20 ++++- 8 files changed, 167 insertions(+), 66 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index e8cc958a53..e117bc1986 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -11928,12 +11928,14 @@ gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, GcsBlockPcmXImageBinding binding; GcsBlockDedupKey key; PcmXLocalHolderProgress progress; + PcmXLocalDrainCertificate certificate; PcmXQueueResult progress_result; PcmXQueueResult result; ClusterPcmOwnSelfHandoffSample handoff_sample; ClusterPcmOwnResult handoff_result; ClusterPcmXRevokeFinishMode finish_mode; uint64 holder_image_id = 0; + bool certificate_exact; bool holder_image = false; bool holder_ref = false; bool self_source_handoff = false; @@ -11946,8 +11948,8 @@ gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, } else if (progress_result != PCM_X_QUEUE_OK && progress_result != PCM_X_QUEUE_NOT_FOUND) return progress_result; - result = cluster_pcm_x_local_drain_poll_exact(poll, authenticated_master_node, - authenticated_master_session); + result = cluster_pcm_x_local_drain_poll_certificate_exact( + poll, authenticated_master_node, authenticated_master_session, &certificate); if (result != PCM_X_QUEUE_OK) return result; @@ -11986,29 +11988,61 @@ gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, self_source_handoff = binding.identity.image.source_node == (uint32)cluster_node_id && poll->ref.identity.node_id == cluster_node_id; if (self_source_handoff) { - handoff_result = cluster_bufmgr_pcm_own_self_handoff_x_exact( - &poll->ref.identity.tag, binding.identity.image.source_own_generation, &handoff_sample); - if (handoff_result != CLUSTER_PCM_OWN_OK) { - /* FINAL makes the in-place X descriptor authoritative before DRAIN. - * Preserve the immutable record if that exact proof is missing. - * The refusal shape is one-shot evidence: this arm fuses the node, - * so the descriptor snapshot must reach the log before it does. */ - ereport( - LOG, - (errmsg("PCM-X self-source DRAIN proof refused"), - errdetail("result=%d source_gen=%llu live_gen=%llu own_flags=%u " - "own_token=%llu pcm_state=%u buffer_type=%u bm_valid=%d found=%d", - (int)handoff_result, - (unsigned long long)binding.identity.image.source_own_generation, - (unsigned long long)handoff_sample.live_generation, - (unsigned int)handoff_sample.own_flags, - (unsigned long long)handoff_sample.live_token, - (unsigned int)handoff_sample.pcm_state, - (unsigned int)handoff_sample.buffer_type, - handoff_sample.bm_valid ? 1 : 0, handoff_sample.buffer_found ? 1 : 0))); + /* The exact completion certificate - the writer-round protocol ledger + * captured under the drain's own tag lock - is the release authority. + * The live descriptor is NOT: its per-buf_id generation carries no + * BufferTag lineage, and the tag may legitimately have moved on + * (X->S downgrade for a reader, a later lifecycle) or been evicted + * since FINAL_ACK. The descriptor probe below only reports a + * structurally malformed flags/token shape. */ + certificate_exact + = certificate.valid && gcs_block_pcm_x_ticket_ref_equal(&certificate.ref, &poll->ref) + && certificate.image.image_id == binding.identity.image.image_id + && certificate.image.source_own_generation + == binding.identity.image.source_own_generation + && certificate.image.source_node == binding.identity.image.source_node + && certificate.master_node == authenticated_master_node + && certificate.master_session_incarnation == authenticated_master_session + && certificate.committed_own_generation + == binding.identity.image.source_own_generation + 1; + handoff_result + = cluster_bufmgr_pcm_own_self_handoff_probe(&poll->ref.identity.tag, &handoff_sample); + if (handoff_result == CLUSTER_PCM_OWN_CORRUPT) { + ereport(LOG, (errmsg("PCM-X self-source DRAIN found a malformed descriptor lifecycle"), + errdetail("own_flags=%u own_token=%llu live_gen=%llu pcm_state=%u " + "buffer_type=%u bm_valid=%d found=%d", + (unsigned int)handoff_sample.own_flags, + (unsigned long long)handoff_sample.live_token, + (unsigned long long)handoff_sample.live_generation, + (unsigned int)handoff_sample.pcm_state, + (unsigned int)handoff_sample.buffer_type, + handoff_sample.bm_valid ? 1 : 0, + handoff_sample.buffer_found ? 1 : 0))); cluster_pcm_x_runtime_fail_closed(); return PCM_X_QUEUE_CORRUPT; } + if (!certificate_exact) { + /* Preserve the immutable record: without the exact certificate the + * release cannot be proven, but a mismatched ledger is not node + * corruption. The master's next poll replays as DUPLICATE. */ + ereport(LOG, + (errmsg("PCM-X self-source DRAIN certificate is not exact"), + errdetail("valid=%d cert_committed=%llu cert_image=%llu cert_source_gen=%llu " + "image=%llu source_gen=%llu source_node=%u live_gen=%llu " + "own_flags=%u pcm_state=%u found=%d", + certificate.valid ? 1 : 0, + (unsigned long long)certificate.committed_own_generation, + (unsigned long long)certificate.image.image_id, + (unsigned long long)certificate.image.source_own_generation, + (unsigned long long)binding.identity.image.image_id, + (unsigned long long)binding.identity.image.source_own_generation, + (unsigned int)binding.identity.image.source_node, + (unsigned long long)handoff_sample.live_generation, + (unsigned int)handoff_sample.own_flags, + (unsigned int)handoff_sample.pcm_state, + handoff_sample.buffer_found ? 1 : 0))); + return PCM_X_QUEUE_NOT_READY; + } } if (cluster_gcs_block_dedup_pcm_x_release_exact(worker_id, &key, &poll->ref.identity.tag, &binding) diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index cd5302183a..74de3c1b15 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -18381,6 +18381,20 @@ PcmXQueueResult cluster_pcm_x_local_drain_poll_exact(const PcmXDrainPollPayload *poll, int32 authenticated_master_node, uint64 authenticated_master_session) +{ + return cluster_pcm_x_local_drain_poll_certificate_exact(poll, authenticated_master_node, + authenticated_master_session, NULL); +} + + +/* First-consumption variant: capture the completion certificate under the + * same tag lock that publishes TERMINAL_DRAINED, so the caller can release + * the self-source immutable record against the exact protocol ledger. */ +PcmXQueueResult +cluster_pcm_x_local_drain_poll_certificate_exact(const PcmXDrainPollPayload *poll, + int32 authenticated_master_node, + uint64 authenticated_master_session, + PcmXLocalDrainCertificate *certificate_out) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; PcmXRuntimeSnapshot runtime; @@ -18400,6 +18414,8 @@ cluster_pcm_x_local_drain_poll_exact(const PcmXDrainPollPayload *poll, bool fail_closed = false; bool promote_candidate; + if (certificate_out != NULL) + memset(certificate_out, 0, sizeof(*certificate_out)); if (header == NULL || poll == NULL || !pcm_x_local_terminal_ref_valid(&poll->ref) || poll->drain_generation == 0 || poll->drain_generation == UINT64_MAX || authenticated_master_node < 0 || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT @@ -18595,6 +18611,18 @@ cluster_pcm_x_local_drain_poll_exact(const PcmXDrainPollPayload *poll, tag_slot->terminal_drain_generation = poll->drain_generation; pg_write_barrier(); (void)pcm_x_slot_flags_fetch_or(&tag_slot->slot, PCM_X_LOCAL_TAG_F_TERMINAL_DRAINED); + /* Completion certificate: the writer-round ledger is captured under this + * same lock, before RETIRE can clear it, so the caller can verify the + * self-source release against the exact protocol evidence instead of the + * live descriptor (which may legitimately have moved on or been evicted). */ + if (certificate_out != NULL) { + certificate_out->ref = tag_slot->ref; + certificate_out->image = tag_slot->image; + certificate_out->committed_own_generation = tag_slot->committed_own_generation; + certificate_out->master_session_incarnation = tag_slot->master_session_incarnation; + certificate_out->master_node = tag_slot->master_node; + certificate_out->valid = true; + } result = PCM_X_QUEUE_OK; drain_release_gate: diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index a1a6afbd6f..47729f688c 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -12011,19 +12011,21 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, return result; } -/* Prove that a sole-requester source image was adopted by the exact S->X - * handoff. This is intentionally read-only: a delayed DRAIN may release its - * immutable A-record, but it must never mutate the current X descriptor. */ +/* Corruption-only probe under a delayed self-source DRAIN. The release + * authority is the exact protocol completion certificate captured by the + * DRAIN consumption; the live descriptor is NOT that authority, because its + * per-buf_id generation is no BufferTag lineage - the tag may legitimately + * have moved on (X->S downgrade for a reader, a later lifecycle) or been + * evicted and reloaded on another descriptor. This probe is intentionally + * read-only and only reports a structurally malformed flags/token shape. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, uint64 source_generation, - ClusterPcmOwnSelfHandoffSample *sample_out) +cluster_bufmgr_pcm_own_self_handoff_probe(const BufferTag *tag, + ClusterPcmOwnSelfHandoffSample *sample_out) { ClusterPcmOwnResult live_result; - ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; BufferDesc *buf; BufferTag lookup_tag; LWLock *partition_lock; - uint64 live_generation; uint64 live_token; uint32 flags; uint32 hashcode; @@ -12032,7 +12034,7 @@ cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, uint64 source_ if (sample_out != NULL) memset(sample_out, 0, sizeof(*sample_out)); - if (tag == NULL || source_generation == UINT64_MAX) + if (tag == NULL) return CLUSTER_PCM_OWN_INVALID; if (ClusterPcmOwnArray == NULL) return CLUSTER_PCM_OWN_NOT_READY; @@ -12043,17 +12045,16 @@ cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, uint64 source_ buf_id = BufTableLookup(&lookup_tag, hashcode); if (buf_id < 0) { LWLockRelease(partition_lock); - return CLUSTER_PCM_OWN_STALE; + return CLUSTER_PCM_OWN_OK; } buf = GetBufferDescriptor(buf_id); buf_state = LockBufHdr(buf); - live_generation = cluster_pcm_own_gen_get(buf->buf_id); live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); if (sample_out != NULL) { /* Snapshot under the header lock only; the caller logs after unlock. */ - sample_out->live_generation = live_generation; + sample_out->live_generation = cluster_pcm_own_gen_get(buf->buf_id); sample_out->live_token = live_token; sample_out->own_flags = flags; sample_out->pcm_state = buf->pcm_state; @@ -12061,18 +12062,9 @@ cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, uint64 source_ sample_out->bm_valid = (buf_state & BM_VALID) != 0; sample_out->buffer_found = BufferTagsEqual(&buf->tag, tag); } - if (!BufferTagsEqual(&buf->tag, tag) || live_generation != source_generation + 1) - result = CLUSTER_PCM_OWN_STALE; - else if (live_result == CLUSTER_PCM_OWN_CORRUPT) - result = live_result; - else if (flags != 0) - result = CLUSTER_PCM_OWN_BUSY; - else if (live_token == 0 || (buf_state & BM_VALID) == 0 || buf->pcm_state != (uint8)PCM_STATE_X - || buf->buffer_type != (uint8)BUF_TYPE_XCUR) - result = CLUSTER_PCM_OWN_STALE; UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); - return result; + return live_result == CLUSTER_PCM_OWN_CORRUPT ? CLUSTER_PCM_OWN_CORRUPT : CLUSTER_PCM_OWN_OK; } /* ======================================================================== diff --git a/src/include/cluster/cluster_pcm_x_bufmgr.h b/src/include/cluster/cluster_pcm_x_bufmgr.h index 6a6f868c55..614aa3455b 100644 --- a/src/include/cluster/cluster_pcm_x_bufmgr.h +++ b/src/include/cluster/cluster_pcm_x_bufmgr.h @@ -362,9 +362,9 @@ extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_finish_revoke_retain( ClusterPcmOwnSnapshot *out_retained); extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, uint64 source_generation); -/* Process-local descriptor evidence captured by the DRAIN proof below so the - * caller can log the exact refusal shape before deciding fail-closed. The - * fields are one header-locked snapshot; never persisted or sent on wire. */ +/* Process-local descriptor evidence captured by the DRAIN-time probe below + * so the caller can log the observed shape as diagnostics. The fields are + * one header-locked snapshot; never persisted or sent on wire. */ typedef struct ClusterPcmOwnSelfHandoffSample { uint64 live_generation; uint64 live_token; @@ -375,11 +375,12 @@ typedef struct ClusterPcmOwnSelfHandoffSample { bool buffer_found; } ClusterPcmOwnSelfHandoffSample; -/* Read-only DRAIN proof for a sole-requester source that committed S->X in - * place. Exact base+1 X/XCUR authority means DRAIN may release only the - * immutable dedup record and must not invalidate this descriptor. */ +/* Read-only corruption probe for a delayed sole-requester source DRAIN. The + * release authority is the protocol completion certificate; the descriptor + * may legitimately have moved on or been evicted, so this only reports a + * structurally malformed flags/token shape (and never mutates anything). */ extern ClusterPcmOwnResult -cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, uint64 source_generation, - ClusterPcmOwnSelfHandoffSample *sample_out); +cluster_bufmgr_pcm_own_self_handoff_probe(const BufferTag *tag, + ClusterPcmOwnSelfHandoffSample *sample_out); #endif /* CLUSTER_PCM_X_BUFMGR_H */ diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h index fcafe71ca0..0525036554 100644 --- a/src/include/cluster/cluster_pcm_x_convert.h +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -1657,6 +1657,24 @@ extern PcmXQueueResult cluster_pcm_x_local_terminal_publish_exact( extern PcmXQueueResult cluster_pcm_x_local_drain_poll_exact(const PcmXDrainPollPayload *poll, int32 authenticated_master_node, uint64 authenticated_master_session); + +/* Process-local completion certificate captured under the tag lock by the + * first (writer-lane) DRAIN consumption. The exact protocol ledger - not + * the live buffer descriptor, whose per-buf_id generation may legitimately + * move on or vanish behind eviction - is the release authority for the + * self-source immutable image record. Never persisted or sent on wire. */ +typedef struct PcmXLocalDrainCertificate { + PcmXTicketRef ref; + PcmXImageToken image; + uint64 committed_own_generation; + uint64 master_session_incarnation; + int32 master_node; + bool valid; +} PcmXLocalDrainCertificate; + +extern PcmXQueueResult cluster_pcm_x_local_drain_poll_certificate_exact( + const PcmXDrainPollPayload *poll, int32 authenticated_master_node, + uint64 authenticated_master_session, PcmXLocalDrainCertificate *certificate_out); extern PcmXQueueResult cluster_pcm_x_local_retire_up_to_exact(const PcmXRetirePayload *request, int32 authenticated_master_node, uint64 authenticated_master_session); diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 033f472252..c3895fa442 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -2101,9 +2101,17 @@ UT_TEST(test_pcm_x_ready_materializes_exact_n_s_or_x_source_without_wire_change) UT_ASSERT_NOT_NULL(strstr(abort, "cluster_bufmgr_pcm_own_abort_x_revoke(")); } if (drain != NULL) { - const char *local_drain = strstr(drain, "cluster_pcm_x_local_drain_poll_exact("); + const char *local_drain + = strstr(drain, "cluster_pcm_x_local_drain_poll_certificate_exact("); const char *duplicate_guard = local_drain != NULL ? strstr(local_drain, "if (result != PCM_X_QUEUE_OK)") : NULL; + /* The self-source release authority is the exact completion + * certificate; an inexact ledger preserves the record without a + * node fuse (the descriptor probe alone may report corruption). */ + const char *certificate_policy = strstr(drain, "source_own_generation + 1"); + const char *certificate_refusal = certificate_policy != NULL + ? strstr(certificate_policy, "PCM_X_QUEUE_NOT_READY") + : NULL; const char *release_record = strstr(drain, "cluster_gcs_block_dedup_pcm_x_release_exact("); const char *finish_mode_gate = strstr(drain, "cluster_pcm_x_revoke_finish_mode("); const char *drop_arm = strstr(drain, "CLUSTER_PCM_X_REVOKE_FINISH_DROP"); @@ -2112,15 +2120,19 @@ UT_TEST(test_pcm_x_ready_materializes_exact_n_s_or_x_source_without_wire_change) UT_ASSERT_NOT_NULL(local_drain); UT_ASSERT_NOT_NULL(duplicate_guard); + UT_ASSERT_NOT_NULL(certificate_policy); + UT_ASSERT_NOT_NULL(certificate_refusal); UT_ASSERT_NOT_NULL(release_record); UT_ASSERT_NOT_NULL(finish_mode_gate); UT_ASSERT_NOT_NULL(drop_arm); UT_ASSERT_NOT_NULL(release_retained); - if (local_drain != NULL && duplicate_guard != NULL && release_record != NULL - && finish_mode_gate != NULL && drop_arm != NULL && release_retained != NULL) - UT_ASSERT(local_drain < duplicate_guard && duplicate_guard < release_record - && release_record < finish_mode_gate && finish_mode_gate < drop_arm - && drop_arm < release_retained); + if (local_drain != NULL && duplicate_guard != NULL && certificate_policy != NULL + && certificate_refusal != NULL && release_record != NULL && finish_mode_gate != NULL + && drop_arm != NULL && release_retained != NULL) + UT_ASSERT(local_drain < duplicate_guard && duplicate_guard < certificate_policy + && certificate_policy < certificate_refusal + && certificate_refusal < release_record && release_record < finish_mode_gate + && finish_mode_gate < drop_arm && drop_arm < release_retained); } if (generic_install != NULL) { const char *content = strstr(generic_install, "LWLockAcquire(content_lock, LW_EXCLUSIVE)"); @@ -2542,7 +2554,7 @@ UT_TEST(test_pcm_x_self_source_handoff_is_no_copy_and_drain_preserves_x) if (self_arm != NULL && remote_finish != NULL && materialize_end != NULL) UT_ASSERT(self_arm < remote_finish && remote_finish < materialize_end); - verify_x = strstr(drain, "cluster_bufmgr_pcm_own_self_handoff_x_exact("); + verify_x = strstr(drain, "cluster_bufmgr_pcm_own_self_handoff_probe("); release_record = strstr(drain, "cluster_gcs_block_dedup_pcm_x_release_exact("); self_return = release_record != NULL ? strstr(release_record, "if (self_source_handoff)") : NULL; diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index e7a08e2ce8..29e05438f5 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -1756,12 +1756,12 @@ UT_TEST(test_queue_self_source_handoff_is_single_lifecycle_and_readonly_drain) "cluster_pcm_own_revoke_to_grant_handoff_exact", "UnlockBufHdr" }; static const char *const drain_proof_contract[] = { "BufMappingPartitionLock", + "BufTableLookup", + "CLUSTER_PCM_OWN_OK", "LockBufHdr", - "source_generation + 1", - "flags != 0", - "PCM_STATE_X", - "BUF_TYPE_XCUR", - "UnlockBufHdr" }; + "cluster_pcm_own_classify_live_flags", + "UnlockBufHdr", + "CLUSTER_PCM_OWN_CORRUPT" }; char *source = read_bufmgr_source(); const char *handoff; const char *handoff_end; @@ -1785,7 +1785,7 @@ UT_TEST(test_queue_self_source_handoff_is_single_lifecycle_and_readonly_drain) forbidden = strstr(handoff, "cluster_pcm_own_reservation_abort_exact("); UT_ASSERT(forbidden == NULL || forbidden >= handoff_end); } - assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_self_handoff_x_exact(", + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_self_handoff_probe(", "\n/* ==========", drain_proof_contract, lengthof(drain_proof_contract)); free(source); diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 39f26c281f..ca05f60db9 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -9413,6 +9413,7 @@ UT_TEST(test_local_transfer_prepare_commit_and_final_ack_are_exact) PcmXPhasePayload final_confirm; PcmXPhasePayload duplicate_confirm; PcmXDrainPollPayload poll; + PcmXLocalDrainCertificate drain_certificate; PcmXRetirePayload retire; PcmXLocalReliableToken token; PcmXLocalReliableToken retry_token; @@ -9677,9 +9678,24 @@ UT_TEST(test_local_transfer_prepare_commit_and_final_ack_are_exact) UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&writer_claim), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + /* First consumption captures the completion certificate under the same + * tag lock; a DUPLICATE replay must not fabricate one. */ + memset(&drain_certificate, 0xff, sizeof(drain_certificate)); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_certificate_exact(&poll, 1, master_session, + &drain_certificate), + PCM_X_QUEUE_OK); + UT_ASSERT(drain_certificate.valid); + UT_ASSERT(ticket_refs_equal(&drain_certificate.ref, &poll.ref)); + UT_ASSERT_EQ(drain_certificate.master_node, 1); + UT_ASSERT_EQ(drain_certificate.master_session_incarnation, master_session); + UT_ASSERT_EQ(drain_certificate.committed_own_generation, tag_slot->committed_own_generation); + UT_ASSERT_EQ(drain_certificate.image.image_id, tag_slot->image.image_id); + UT_ASSERT_EQ(drain_certificate.image.source_own_generation, + tag_slot->image.source_own_generation); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_certificate_exact(&poll, 1, master_session, + &drain_certificate), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(!drain_certificate.valid); UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); UT_ASSERT_EQ(progress.member_state, PCM_XL_GRANTED); UT_ASSERT_EQ(progress.pending_opcode, 0); From 26bf334205e31e99da4bce21dc6da66361f15a0b Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 06:14:08 -0700 Subject: [PATCH 32/56] fix(cluster): name the retire-collect arm before its shared fail-closed exit Every corruption arm of the local retire collect funnels to one fail-closed line (the retire_failed label), so the fuse that now leads two of four wedge autopsies - followed by thousands of futile RETIRE retries against the fused node - cannot say which surprise it hit or what the uncoerced result was. Log a one-shot note at each arm (preflight, allocator view, second-pass candidate, holder/writer detach, wake match, wake count) carrying the slot index and the raw result before the CORRUPT coercion. --- src/backend/cluster/cluster_pcm_x_convert.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index 74de3c1b15..99a089c013 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -19842,6 +19842,18 @@ pcm_x_local_wake_actual_match(const PcmXLocalWakeBatch *wake_batch, } +/* One-shot evidence ahead of the retire-collect fuse: every corruption arm + * funnels to the same fail-closed line, so name the arm and its uncoerced + * detail here. For scan arms a/b are the slot index and the raw result; + * for the wake-count arm they are the projected and actual wake counts. */ +static void +pcm_x_local_retire_collect_note(const char *site, uint64 a, uint64 b) +{ + ereport(LOG, (errmsg("PCM-X local retire collect fails closed at %s (a=%llu, b=%llu)", site, + (unsigned long long)a, (unsigned long long)b))); +} + + PcmXQueueResult cluster_pcm_x_local_retire_up_to_collect_exact(const PcmXRetirePayload *request, int32 authenticated_master_node, @@ -19933,6 +19945,7 @@ cluster_pcm_x_local_retire_up_to_collect_exact(const PcmXRetirePayload *request, return result; } if (!pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_TAG, &view)) { + pcm_x_local_retire_collect_note("allocator-view", 0, 0); result = PCM_X_QUEUE_CORRUPT; fail_closed = true; goto retire_failed; @@ -19983,6 +19996,7 @@ cluster_pcm_x_local_retire_up_to_collect_exact(const PcmXRetirePayload *request, &writer_candidate_ref, &writer_candidate, &holder_candidate, &candidate, &contains_watermark, &holder_wake, &holder_wake_candidate); if (result != PCM_X_QUEUE_OK) { + pcm_x_local_retire_collect_note("second-pass-candidate", (uint64)i, (uint64)result); fail_closed = true; result = PCM_X_QUEUE_CORRUPT; goto retire_failed; @@ -20006,6 +20020,9 @@ cluster_pcm_x_local_retire_up_to_collect_exact(const PcmXRetirePayload *request, } } if (result != PCM_X_QUEUE_OK) { + pcm_x_local_retire_collect_note(holder_candidate ? "holder-detach" + : "writer-detach", + (uint64)i, (uint64)result); fail_closed = true; result = PCM_X_QUEUE_CORRUPT; goto retire_failed; @@ -20014,6 +20031,7 @@ cluster_pcm_x_local_retire_up_to_collect_exact(const PcmXRetirePayload *request, result = pcm_x_local_wake_actual_match(wake_batch, &writer_wake, projected_wake_count, &actual_wake_count); if (result != PCM_X_QUEUE_OK) { + pcm_x_local_retire_collect_note("wake-match", (uint64)i, (uint64)result); fail_closed = true; result = PCM_X_QUEUE_CORRUPT; goto retire_failed; @@ -20022,6 +20040,8 @@ cluster_pcm_x_local_retire_up_to_collect_exact(const PcmXRetirePayload *request, } while (candidate); } if (wake_batch != NULL && actual_wake_count != projected_wake_count) { + pcm_x_local_retire_collect_note("wake-count", (uint64)projected_wake_count, + (uint64)actual_wake_count); result = PCM_X_QUEUE_CORRUPT; fail_closed = true; goto retire_failed; @@ -20059,6 +20079,7 @@ cluster_pcm_x_local_retire_up_to_collect_exact(const PcmXRetirePayload *request, /* No tag was mutated, so a retryable/stale preflight can release the claim. * Structural corruption retains both exact episode witnesses. */ if (result == PCM_X_QUEUE_CORRUPT) { + pcm_x_local_retire_collect_note("preflight", 0, (uint64)result); fail_closed = true; goto retire_failed; } From 1be19983799597456b2c9b70c038b2cf964fd0d0 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 09:31:27 -0700 Subject: [PATCH 33/56] fix(cluster): classify the independent dual terminal instead of fusing it t/400 loop3/loop4b first-fuse autopsy (dump flags=0x3d, writer ref ticket 3, holder ticket 4, drain generations 7/7, one unlinked terminal membership, every FIFO index invalid): a granted writer terminal awaiting RETIRE and a newer revoked holder terminal legally share one tag under the writer round's REVOKE_BARRIER with two DISTINCT master tickets, but pcm_x_local_same_ref_dual_retire_state coerced any unequal-ref dual to CORRUPT, so RETIRE_UP_TO(3) closed the runtime and the master's RETIRE leg retried forever (leg_retry 6665/6662). Classifier: validate each lane's complete DRAIN evidence first; only ref-equal duals run the same-ref drain-generation/image rules; unequal refs under distinct ticket ids are the legal independent form (not a dual); a ticket-id collision under a different ref stays CORRUPT (master-unique ticket ids, the cross-lane alias class). Writer detach: the exact proven narrow shape (one unlinked terminal membership, all indexes invalid, clear writer leg, holder lane exact) retires ONLY the writer lane -- prehandle/ref/image/reliable/committed/ base/terminal/membership -- keeping the holder lane, the barrier and the master binding byte-exact on the LIVE tag. Any wider distinct-ticket dual stays NOT_READY with zero mutation, in both the RETIRE preflight and the detach. The holder ticket's own later watermark then consumes the holder lane and closes the still-frozen empty round (round advance + barrier drop) before the tag detaches. RED (recorded pre-fix): the new production-path unit test built the 0x3d shape and RETIRE_UP_TO(3) returned CORRUPT with the runtime RECOVERY_BLOCKED at test_cluster_pcm_x_convert.c:10636-10637. GREEN: 260/260 unit, full-suite 182 binaries, check-format clean. Alias / half-mask / poisoned-drain-leg probes keep the fail-closed verdict. --- src/backend/cluster/cluster_pcm_x_convert.c | 110 +++++- .../cluster_unit/test_cluster_pcm_x_convert.c | 335 +++++++++++++++++- 2 files changed, 434 insertions(+), 11 deletions(-) diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index 99a089c013..bf62feaa4f 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -16868,16 +16868,29 @@ pcm_x_local_same_ref_dual_retire_state(const PcmXLocalTagSlot *tag_slot, uint32 source_holder = !pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref); external_ref = source_holder ? &tag_slot->holder_ref : &tag_slot->blocker_snapshot_ref; memset(&zero_image, 0, sizeof(zero_image)); + /* Per-lane completeness comes first: each terminal lane must carry its + * own full DRAIN evidence before any dual/independent classification. */ if (writer_flags != PCM_X_LOCAL_TAG_F_TERMINAL_MASK || holder_flags != PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK || tag_slot->membership_count == 0 - || pcm_x_ticket_ref_is_zero(external_ref) - || !pcm_x_ticket_ref_equal(&tag_slot->ref, external_ref) - || tag_slot->terminal_drain_generation == 0 + || pcm_x_ticket_ref_is_zero(external_ref) || tag_slot->terminal_drain_generation == 0 || tag_slot->terminal_drain_generation == UINT64_MAX - || tag_slot->terminal_drain_generation != tag_slot->holder_terminal_drain_generation || !pcm_x_local_terminal_pending_clear(tag_slot) || pcm_x_local_holder_lane_retire_state(tag_slot, flags) != PCM_X_QUEUE_OK) return PCM_X_QUEUE_CORRUPT; + if (!pcm_x_ticket_ref_equal(&tag_slot->ref, external_ref)) { + /* Two complete terminals under DISTINCT master tickets are the legal + * independent form: a granted writer awaiting RETIRE plus a newer + * revoked holder lane on the same tag (the t/400 dump-0x3d shape). + * Not a dual -- each lane retires by its own watermark. A ticket-id + * collision under a different ref can only be corruption: per-session + * ticket ids are master-unique (see the cross-lane alias-class note + * at pcm_x_local_cross_lane_holder_terminal_retirable). */ + if (external_ref->handle.ticket_id != tag_slot->ref.handle.ticket_id) + return PCM_X_QUEUE_OK; + return PCM_X_QUEUE_CORRUPT; + } + if (tag_slot->terminal_drain_generation != tag_slot->holder_terminal_drain_generation) + return PCM_X_QUEUE_CORRUPT; if (external_ref->grant_generation == 0) { if (source_holder || tag_slot->ref.grant_generation != 0 || (flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0 @@ -18889,6 +18902,7 @@ pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_pr bool detach_after_close = false; bool detach_tag = false; bool gate_claimed = false; + bool retain_independent_holder = false; bool same_ref_dual = false; bool ready_leader_candidate = false; bool target_in_closed_round; @@ -19013,6 +19027,36 @@ pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_pr goto detach_local_domain_done; } } + /* Independent dual terminal (dump-0x3d): a complete writer terminal + * and a complete holder terminal under DISTINCT master tickets. Only + * the exact proven narrow shape may retire its writer lane while the + * holder lane, the barrier and the master binding stay byte-exact on + * the LIVE tag for the holder ticket's own later watermark: one + * unlinked terminal membership, every FIFO/holder index invalid and + * a clear writer application leg. Any other distinct-ticket dual is + * unproven and stays NOT_READY with zero mutation. */ + if (!same_ref_dual && !cancel_same_ref_dual + && (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) + == PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK + && (flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) == PCM_X_LOCAL_TAG_F_TERMINAL_MASK) { + const PcmXTicketRef *external_ref = pcm_x_local_external_terminal_ref(tag_slot); + + retain_independent_holder + = external_ref != NULL && external_ref->grant_generation != 0 + && external_ref->handle.ticket_id != tag_slot->ref.handle.ticket_id + && tag_slot->membership_count == 1 + && tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->active_holder_head_index == PCM_X_INVALID_SLOT_INDEX + && pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) + && pcm_x_local_holder_lane_retire_state(tag_slot, flags) == PCM_X_QUEUE_OK; + if (!retain_independent_holder) { + result = PCM_X_QUEUE_NOT_READY; + goto detach_local_domain_done; + } + } } target_in_closed_round = (flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && member->admitted_round == tag_slot->local_round @@ -19027,7 +19071,8 @@ pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_pr result = PCM_X_QUEUE_CORRUPT; goto detach_local_domain_done; } - if (retire_protocol && target_in_closed_round && tag_slot->closed_round_member_count == 1) { + if (retire_protocol && !retain_independent_holder && target_in_closed_round + && tag_slot->closed_round_member_count == 1) { result = pcm_x_local_final_member_round_plan_locked(tag_slot, tag_ref, &round_plan); if (result != PCM_X_QUEUE_OK) goto detach_local_domain_done; @@ -19045,10 +19090,14 @@ pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_pr /* Writer membership and holder transfer are independent lifetimes on * the same tag. Neither returning to holder-only form nor deleting the * tag may erase the master binding while an exact holder ticket, send - * leg, or terminal-drain outcome remains authoritative. */ - if (!pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref) - || !pcm_x_transfer_leg_is_clear(&tag_slot->holder_reliable) - || ((flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 && !cancel_same_ref_dual)) { + * leg, or terminal-drain outcome remains authoritative. The proven + * independent dual terminal is the one exception: its writer lane may + * retire alone, leaving the holder lane authoritative on the tag. */ + if (!retain_independent_holder + && (!pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref) + || !pcm_x_transfer_leg_is_clear(&tag_slot->holder_reliable) + || ((flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 + && !cancel_same_ref_dual))) { result = PCM_X_QUEUE_NOT_READY; goto detach_local_domain_done; } @@ -19061,7 +19110,7 @@ pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_pr goto detach_local_domain_done; } pcm_x_local_reset_holder_only_queue(tag_slot); - } else if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0) { + } else if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0 && !retain_independent_holder) { pg_write_barrier(); pcm_x_slot_state_write(&tag_slot->slot, PCM_X_TAG_DETACHING); detach_tag = true; @@ -19446,6 +19495,22 @@ pcm_x_local_retire_candidate_at(Size slot_index, const PcmXRetirePayload *reques result = PCM_X_QUEUE_NOT_READY; goto candidate_done; } + /* An independent (distinct-ticket) dual terminal may offer its writer + * lane only in the exact proven narrow shape; see the retain arm in + * pcm_x_local_detach_terminal_common. Refusing here keeps the whole + * RETIRE preflight retryable instead of tripping the second-pass + * fail-closed exit on an unproven wider shape. */ + if ((flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 && !same_ref_dual + && !cancel_same_ref_dual + && (tag_slot->membership_count != 1 || tag_slot->head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->tail_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->leader_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX + || !pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable))) { + result = PCM_X_QUEUE_NOT_READY; + goto candidate_done; + } if (tag_slot->ref.handle.ticket_id == request->retire_through_ticket_id) *contains_watermark_out = true; if (tag_slot->ref.handle.ticket_id <= request->retire_through_ticket_id) { @@ -19604,6 +19669,31 @@ pcm_x_local_holder_detach_terminal_exact(const PcmXTicketRef *ref, int32 authent tag_slot->holder_terminal_drain_generation = 0; (void)pcm_x_slot_flags_fetch_and(&tag_slot->slot, ~PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); flags = pcm_x_slot_flags_read(&tag_slot->slot); + /* An independent dual terminal retires its writer lane first while the + * holder lane keeps the writer round's REVOKE_BARRIER standing (the + * writer detach skips its final-member round close to preserve the + * holder evidence). Once the holder lane is consumed too and the tag is + * otherwise empty, close the frozen round exactly like + * pcm_x_local_empty_frozen_round_apply_locked would with no successor: + * advance the round and drop the barrier. All guards are read-only and + * the mutation is infallible, so no retryable result can strand the + * already-consumed holder lane. */ + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 + && (flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) == 0 + && tag_slot->closed_round_member_count == 0 && tag_slot->membership_count == 0 + && tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->active_holder_head_index == PCM_X_INVALID_SLOT_INDEX + && pcm_x_ticket_ref_is_zero(&tag_slot->ref) + && pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) && tag_slot->local_round != 0 + && tag_slot->local_round != UINT32_MAX) { + tag_slot->local_round++; + pg_write_barrier(); + (void)pcm_x_slot_flags_fetch_and(&tag_slot->slot, ~PCM_X_LOCAL_TAG_F_REVOKE_BARRIER); + flags = pcm_x_slot_flags_read(&tag_slot->slot); + } detach_tag = tag_slot->membership_count == 0 && tag_slot->closed_round_member_count == 0 && tag_slot->active_holder_head_index == PCM_X_INVALID_SLOT_INDEX diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index ca05f60db9..98fa438bf1 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -2059,6 +2059,13 @@ test_slot_flags(PcmXSlotHeader *slot) return pg_atomic_read_u32(&slot->state_flags) >> PCM_X_SLOT_FLAGS_SHIFT; } +/* Direct poke for protocol-impossible flag shapes (CORRUPT-arm probes). */ +static void +test_clear_slot_flags(PcmXSlotHeader *slot, uint32 flags_to_clear) +{ + pg_atomic_fetch_and_u32(&slot->state_flags, ~(flags_to_clear << PCM_X_SLOT_FLAGS_SHIFT)); +} + static PcmXSlotHeader * reserve_slot(PcmXAllocatorKind kind, PcmXSlotRef *ref) { @@ -10394,6 +10401,330 @@ UT_TEST(test_local_cross_lane_retire_exemption_requires_distinct_ticket) UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); } +typedef struct TestLocalIndependentDualTerminal { + PcmXLocalHandle writer; + PcmXTicketRef writer_ref; + PcmXTicketRef holder_ref; + PcmXLocalTagSlot *tag_slot; + uint64 master_session; +} TestLocalIndependentDualTerminal; + +/* + * Build the exact t/400 independent dual-terminal shape (dump flags 0x3d) + * through production paths only, in the production interleaving: + * + * Older local writer (writer_ticket): full grant chain (join -> claim -> + * revoke-cutoff -> ENQUEUE -> ADMIT -> its own empty PROBE -> PREPARE_GRANT + * -> INSTALL_READY -> COMMIT_X -> FINAL_ACK -> FINAL_COMMIT_ACK), with its + * writer-lane DRAIN deliberately delayed. + * + * Newer external cohort (holder_ticket > writer_ticket): REVOKEs this node + * as its image source before that delayed DRAIN lands (empty PROBE -> + * blocker ACK -> REVOKE apply publishes holder_ref -> IMAGE_READY). Both + * lanes then DRAIN, leaving TERMINAL_READY|DRAINED on the writer lane and + * HOLDER_TERMINAL_READY|DRAINED on the holder lane under one REVOKE_BARRIER + * with two DISTINCT master tickets -- the 2026-07-19 t/400 first-fuse shape. + */ +static void +prepare_local_independent_dual_terminal(BlockNumber block, uint64 master_session, + uint64 writer_ticket, uint64 holder_ticket, + TestLocalIndependentDualTerminal *fixture) +{ + PcmXShmemHeader *header; + PcmXLocalWriterClaim writer_claim; + PcmXLocalCutoff writer_cutoff; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXWaitIdentity writer_identity; + PcmXAdmitAckPayload admit_ack; + PcmXEnqueuePayload enqueue; + PcmXPhasePayload admit_confirm; + PcmXGrantPayload prepare; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + PcmXLocalReliableToken token; + PcmXTicketRef probe_ref; + PcmXRevokePayload revoke; + PcmXGrantPayload ready; + PcmXGrantPayload replay; + PcmXDrainPollPayload poll; + uint32 flags; + + UT_ASSERT_NOT_NULL(fixture); + memset(fixture, 0, sizeof(*fixture)); + fixture->master_session = master_session; + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + + /* Older local writer through the full production grant chain. */ + writer_identity = make_wait_identity(block, 0, 3, master_session + UINT64_C(4)); + UT_ASSERT_EQ( + cluster_pcm_x_local_join_begin(&writer_identity, 1, master_session, &fixture->writer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(fixture->writer.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&fixture->writer, &writer_claim), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&fixture->writer, &writer_cutoff), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&fixture->writer, &enqueue, &token), + PCM_X_QUEUE_OK); + memset(&admit_ack, 0, sizeof(admit_ack)); + admit_ack.ref.identity = writer_identity; + admit_ack.ref.handle.ticket_id = writer_ticket; + admit_ack.ref.handle.queue_generation = UINT64_C(1); + admit_ack.prehandle = enqueue.prehandle; + admit_ack.result = PCM_X_QUEUE_OK; + admit_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ( + cluster_pcm_x_local_apply_admit_ack_exact(&fixture->writer, &admit_ack, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_admit_confirm_arm_exact(&fixture->writer, &admit_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&fixture->writer, &admit_confirm, 1, + master_session), + PCM_X_QUEUE_OK); + /* The writer's grant-time PROBE targets only nodes holding S pins; this + * node has none, so no blocker tombstone is planted here (the production + * shape: the later external PROBE must not find a stale locator). */ + memset(&prepare, 0, sizeof(prepare)); + prepare.ref = admit_ack.ref; + prepare.ref.grant_generation = UINT64_C(2); + UT_ASSERT( + cluster_pcm_x_image_id_encode(1, master_session + UINT64_C(300), &prepare.image.image_id)); + prepare.image.source_own_generation = UINT64_C(9); + prepare.image.page_scn = UINT64_C(10); + prepare.image.page_lsn = UINT64_C(11); + prepare.image.source_node = 2; + prepare.image.page_checksum = UINT32_C(12); + UT_ASSERT_EQ( + cluster_pcm_x_local_prepare_grant_exact(&fixture->writer, &prepare, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_install_ready_arm_exact( + &fixture->writer, &prepare.ref, &prepare.image, &install_ready, &token), + PCM_X_QUEUE_OK); + memset(&commit, 0, sizeof(commit)); + commit.ref = prepare.ref; + commit.phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; + UT_ASSERT_EQ(cluster_pcm_x_local_commit_x_exact(&fixture->writer, &commit, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_final_ack_arm_exact(&fixture->writer, 1, &final_ack, &token), + PCM_X_QUEUE_OK); + memset(&final_commit, 0, sizeof(final_commit)); + final_commit.ref = prepare.ref; + final_commit.phase = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; + UT_ASSERT_EQ(cluster_pcm_x_local_final_commit_ack_exact(&fixture->writer, &final_commit, 1, + master_session, &final_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&writer_claim), PCM_X_QUEUE_OK); + fixture->writer_ref = prepare.ref; + + /* The newer external cohort REVOKEs this node as image source before the + * writer-lane DRAIN lands (the production interleaving). */ + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(block, 3, 23, master_session + UINT64_C(3)); + probe_ref.handle.ticket_id = holder_ticket; + probe_ref.handle.queue_generation = UINT64_C(11); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + &holder_copy, 1, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holder_snapshot.holder_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + memset(&revoke, 0, sizeof(revoke)); + revoke.ref = probe_ref; + revoke.ref.grant_generation = UINT64_C(2); + UT_ASSERT(cluster_pcm_x_image_id_encode(1, master_session + UINT64_C(400), &revoke.image_id)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + PCM_X_QUEUE_OK); + memset(&ready, 0, sizeof(ready)); + ready.ref = revoke.ref; + ready.image.image_id = revoke.image_id; + ready.image.source_own_generation = UINT64_C(61); + ready.image.page_scn = UINT64_C(62); + ready.image.page_lsn = UINT64_C(63); + ready.image.source_node = 0; + ready.image.page_checksum = UINT32_C(64); + UT_ASSERT_EQ( + cluster_pcm_x_local_holder_image_ready_arm_exact(&ready, 1, master_session, &replay), + PCM_X_QUEUE_OK); + fixture->holder_ref = revoke.ref; + + /* Both lanes now DRAIN: the delayed writer leg first, then the holder. */ + memset(&poll, 0, sizeof(poll)); + poll.ref = fixture->writer_ref; + poll.drain_generation = UINT64_C(7); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); + memset(&poll, 0, sizeof(poll)); + poll.ref = fixture->holder_ref; + poll.drain_generation = UINT64_C(7); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); + + /* Pin the t/400 dump shape: 0x3d, two distinct tickets, one unlinked + * terminal membership, every FIFO index invalid. */ + fixture->tag_slot = &local_tag_slots(header)[fixture->writer.tag_slot.slot_index]; + flags = test_slot_flags(&fixture->tag_slot->slot); + UT_ASSERT((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK, PCM_X_LOCAL_TAG_F_TERMINAL_MASK); + UT_ASSERT_EQ(flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, + PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + UT_ASSERT_EQ(fixture->tag_slot->ref.handle.ticket_id, writer_ticket); + UT_ASSERT_EQ(fixture->tag_slot->ref.grant_generation, UINT64_C(2)); + UT_ASSERT_EQ(fixture->tag_slot->holder_ref.handle.ticket_id, holder_ticket); + UT_ASSERT_EQ(fixture->tag_slot->membership_count, 1); + UT_ASSERT_EQ(fixture->tag_slot->head_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(fixture->tag_slot->tail_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(fixture->tag_slot->leader_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(fixture->tag_slot->active_writer_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT(fixture->tag_slot->terminal_drain_generation != 0); + UT_ASSERT(fixture->tag_slot->holder_terminal_drain_generation != 0); +} + +/* + * RED for the t/400 independent dual-terminal false fuse (2026-07-19): a + * fully DRAINED older writer terminal (T) and a fully DRAINED newer holder + * terminal (H, distinct master ticket) legally share one tag under the + * writer round's REVOKE_BARRIER. Before the fix the same-ref-dual + * classifier coerces the distinct-ticket shape to CORRUPT (the ref-equality + * arm of pcm_x_local_same_ref_dual_retire_state), RETIRE_UP_TO(T) blocks the + * runtime, and the master's RETIRE leg retries forever -- the 20088-family + * first fuse in loop3/loop4b. After the fix RETIRE_UP_TO(T) must consume + * exactly the writer lane (holder lane byte-exact, tag LIVE) and a later + * RETIRE_UP_TO(H) must drain the tag to the queue baseline. + */ +UT_TEST(test_local_independent_dual_terminal_retires_writer_then_holder) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalTagSlot *tag_slot; + PcmXLocalProgress progress; + PcmXTicketRef holder_ref_before; + PcmXImageToken holder_image_before; + PcmXReliableLegState holder_reliable_before; + PcmXRetirePayload retire; + PcmXSlotRef found; + uint64 holder_drain_before; + uint64 reliable_sequence_before; + const uint64 master_session = UINT64_C(1811); + + prepare_local_independent_dual_terminal(7117, master_session, UINT64_C(96001), UINT64_C(96002), + &fixture); + tag_slot = fixture.tag_slot; + holder_ref_before = tag_slot->holder_ref; + holder_image_before = tag_slot->holder_image; + holder_reliable_before = tag_slot->holder_reliable; + holder_drain_before = tag_slot->holder_terminal_drain_generation; + reliable_sequence_before = tag_slot->reliable.state_sequence; + + /* RETIRE watermark covers only the older writer ticket. */ + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.writer_ref.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = fixture.writer_ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + + /* Exactly the writer lane was consumed... */ + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_TERMINAL_MASK, 0); + UT_ASSERT(ticket_refs_equal(&tag_slot->ref, &(PcmXTicketRef){ 0 })); + UT_ASSERT_EQ(tag_slot->terminal_drain_generation, 0); + UT_ASSERT_EQ(tag_slot->committed_own_generation, 0); + UT_ASSERT_EQ(tag_slot->membership_count, 0); + UT_ASSERT_EQ(tag_slot->reliable.state_sequence, reliable_sequence_before); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&fixture.writer, &progress), + PCM_X_QUEUE_NOT_FOUND); + + /* ...while the holder lane, its master binding and the barrier survive + * byte-exact on the still-LIVE tag. */ + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, + PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + UT_ASSERT(ticket_refs_equal(&tag_slot->holder_ref, &holder_ref_before)); + UT_ASSERT(memcmp(&tag_slot->holder_image, &holder_image_before, sizeof(holder_image_before)) + == 0); + UT_ASSERT( + memcmp(&tag_slot->holder_reliable, &holder_reliable_before, sizeof(holder_reliable_before)) + == 0); + UT_ASSERT_EQ(tag_slot->holder_terminal_drain_generation, holder_drain_before); + UT_ASSERT_EQ(tag_slot->master_node, 1); + UT_ASSERT_EQ(tag_slot->master_session_incarnation, master_session); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &fixture.writer_ref.identity.tag, &found), + PCM_X_DIRECTORY_OK); + + /* The later watermark drains the holder lane and the tag to baseline. */ + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.holder_ref.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = fixture.holder_ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &fixture.writer_ref.identity.tag, &found), + PCM_X_DIRECTORY_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + assert_local_queue_baseline(ClusterPcmXConvertShmem); +} + +/* + * Defense probes for the independent dual-terminal classification: it must + * stay strictly distinct-ticket. A ticket-id collision under a different + * ref, a half-drained writer mask and a poisoned holder drain leg are all + * protocol-impossible shapes and must keep the CORRUPT fail-closed verdict + * (direct slot pokes, like the CORRUPT arms elsewhere in this file). + */ +UT_TEST(test_local_independent_dual_terminal_alias_and_malformed_lanes_fail_closed) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalTagSlot *tag_slot; + PcmXRetirePayload retire; + const uint64 master_session = UINT64_C(1813); + + /* Ticket-id collision under a distinct holder ref. */ + prepare_local_independent_dual_terminal(7118, master_session, UINT64_C(96001), UINT64_C(96002), + &fixture); + tag_slot = fixture.tag_slot; + tag_slot->holder_ref.handle.ticket_id = fixture.writer_ref.handle.ticket_id; + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.writer_ref.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = fixture.writer_ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* Half-drained writer mask: READY without DRAINED is not independent. */ + prepare_local_independent_dual_terminal(7118, master_session, UINT64_C(96001), UINT64_C(96002), + &fixture); + tag_slot = fixture.tag_slot; + test_clear_slot_flags(&tag_slot->slot, PCM_X_LOCAL_TAG_F_TERMINAL_DRAINED); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* Poisoned holder drain leg: a response tombstone is never clean. */ + prepare_local_independent_dual_terminal(7118, master_session, UINT64_C(96001), UINT64_C(96002), + &fixture); + tag_slot = fixture.tag_slot; + tag_slot->holder_reliable.response_tombstone_mask = UINT32_C(1); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + typedef struct TestLocalRebaseFixture { PcmXLocalHandle writer; PcmXLocalWriterClaim writer_claim; @@ -15173,7 +15504,7 @@ UT_TEST(test_local_retire_episode_lock_errors_fail_closed) int main(void) { - UT_PLAN(258); + UT_PLAN(260); UT_RUN(test_image_id_domain_is_canonical_and_bounded); UT_RUN(test_wire_abi_sizes_are_exact); UT_RUN(test_wire_abi_offsets_are_exact); @@ -15350,6 +15681,8 @@ main(void) UT_RUN(test_local_non_source_blocker_participant_drains_and_retires_exactly); UT_RUN(test_local_cross_lane_holder_terminal_retires_under_revoke_barrier); UT_RUN(test_local_cross_lane_retire_exemption_requires_distinct_ticket); + UT_RUN(test_local_independent_dual_terminal_retires_writer_then_holder); + UT_RUN(test_local_independent_dual_terminal_alias_and_malformed_lanes_fail_closed); UT_RUN(test_local_grant_rebase_publish_is_one_shot_and_effective); UT_RUN(test_local_grant_rebase_conflict_fails_closed); UT_RUN(test_local_follower_claim_inherits_published_rebase); From 0d26b44204c23f1a98d71ff0fce2561fd6358b50 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 10:02:57 -0700 Subject: [PATCH 34/56] fix(cluster): close the frozen round when the independent dual's last lane retires Adversarial review of the independent dual-terminal fix found the narrow retain arm proved too little and too asymmetrically: - A next-round follower joining BEFORE the writer's RETIRE pushed the shape out of the members==1 eligibility, and the follower waits for exactly this RETIRE to drop the barrier: a deterministic cycle. - A follower joining BETWEEN the two watermarks hit the opposite hole: the holder detach consumed the last terminal authority, answered OK and left the barrier standing over the follower forever. - A cancelled external lane (zero grant generation on the blocker locator, reachable while the writer cohort is still terminal) passed the preflight but failed the detach's gen!=0 check, escalating a retryable disagreement into the collect's fail-closed exit. - A single watermark covering both tickets never got past the holder lane's cross-lane refusal even though the same episode retires the writer lane first. One shared predicate now decides writer-lane eligibility for both passes (barrier standing, one closed round of exactly the target, round/cutoff invariants, FIFO holding at most next-round followers, clear writer leg, exact holder lane; cancelled externals eligible with their staged second half defined in the cross-lane guard). The holder detach rehearses the frozen-round close read-only BEFORE consuming any evidence -- reusing follower_chain_ready for the promotion plan -- and applies round-advance/barrier-drop/promotion infallibly afterwards; the RETIRE preflight runs the same rehearsal (projected past the writer retire when one watermark covers both lanes), so no fallible check ever runs after the first mutation. The writer preflight stops projecting a final-member promotion the retain arm never performs. RED (recorded on detached 1be1998379 with the pure test diff): all five new production-path legs fail exactly on their gap -- pre-joined follower NOT_READY(7) and never promoted; between-watermarks join left REVOKE_BARRIER=1 and role=FOLLOWER after the holder RETIRE; cancelled external CORRUPT(10)+RECOVERY_BLOCKED; single watermark NOT_READY(7); invariant-break answered CORRUPT instead of a byte-exact NOT_READY. GREEN: 265/265 unit, full-suite 182 binaries, check-format clean. --- src/backend/cluster/cluster_pcm_x_convert.c | 266 ++++++++++++---- .../cluster_unit/test_cluster_pcm_x_convert.c | 289 ++++++++++++++++-- 2 files changed, 467 insertions(+), 88 deletions(-) diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index bf62feaa4f..3456a4941b 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -18124,6 +18124,74 @@ pcm_x_local_empty_frozen_round_apply_locked(PcmXLocalTagSlot *tag_slot, } +/* Read-only rehearsal of the frozen-round close that must accompany the LAST + * terminal authority leaving a barrier-frozen tag. An independent dual + * terminal retires its writer lane first while the barrier stands, so by the + * holder ticket's own RETIRE the round may hold next-round followers that + * joined in between; consuming the holder lane without closing the round + * would strand them behind a barrier with no terminal authority left to drop + * it. Every fallible check runs here BEFORE the holder lane is consumed; + * the caller then applies the close infallibly afterwards. The RETIRE + * preflight runs the same rehearsal so both passes agree (the retire gate + * excludes ordinary mutators for the whole episode). + * + * after_writer_retire projects the state a single watermark covering both + * lanes will reach: the writer target is still the closed round of one and + * still owns tag_slot->ref/membership when the preflight runs, so those two + * facts are checked in their pre-retire form instead. */ +static PcmXQueueResult +pcm_x_local_independent_holder_close_plan_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + bool after_writer_retire, + PcmXLocalRoundClosePlan *plan) +{ + PcmXQueueResult result; + Size projected_membership; + Size projected_closed; + bool candidate_promote = false; + + if (tag_slot == NULL || plan == NULL) + return PCM_X_QUEUE_CORRUPT; + memset(plan, 0, sizeof(*plan)); + plan->candidate_ref.slot_index = PCM_X_INVALID_SLOT_INDEX; + if (after_writer_retire) { + if (tag_slot->closed_round_member_count != 1 || tag_slot->membership_count == 0 + || pcm_x_ticket_ref_is_zero(&tag_slot->ref)) + return PCM_X_QUEUE_CORRUPT; + projected_membership = tag_slot->membership_count - 1; + projected_closed = tag_slot->closed_round_member_count - 1; + } else { + if (tag_slot->closed_round_member_count != 0 || !pcm_x_ticket_ref_is_zero(&tag_slot->ref)) + return PCM_X_QUEUE_CORRUPT; + projected_membership = tag_slot->membership_count; + projected_closed = tag_slot->closed_round_member_count; + } + if (tag_slot->local_round == 0 || tag_slot->local_round == UINT32_MAX + || tag_slot->next_sequence == 0 || tag_slot->cutoff_sequence >= tag_slot->next_sequence + || projected_closed != 0 + || ((projected_membership == 0) != (tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX)) + || ((tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX)) + || tag_slot->leader_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->leader_slot_generation != 0 + || tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_writer_slot_generation != 0 + || tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX + || !pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable)) + return PCM_X_QUEUE_CORRUPT; + if (tag_slot->head_index != PCM_X_INVALID_SLOT_INDEX) { + result = pcm_x_local_follower_chain_ready(tag_slot, tag_ref, tag_slot->head_index, + PCM_X_INVALID_SLOT_INDEX, 0, &plan->candidate, + &plan->candidate_ref, &candidate_promote); + if (result != PCM_X_QUEUE_OK) + return result == PCM_X_QUEUE_BUSY ? PCM_X_QUEUE_NOT_READY : result; + if (plan->candidate == NULL || candidate_promote) + return PCM_X_QUEUE_CORRUPT; + } + plan->close_round = true; + return PCM_X_QUEUE_OK; +} + + /* Project the state immediately after RETIRE removes the final member of a * closed writer cohort. The terminal member is already unlinked by DRAIN, so * the remaining FIFO, if any, must consist entirely of next-round followers. @@ -18878,6 +18946,52 @@ static PcmXQueueResult pcm_x_local_ready_leader_wake_locked(PcmXLocalTagSlot *ta bool *candidate_out); +/* Shared eligibility for retiring ONLY the writer lane of an independent + * (distinct-ticket) dual terminal while the holder lane stays authoritative. + * The RETIRE preflight (candidate scan) and the detach (retain arm) MUST + * agree bit-for-bit through this one predicate: a first-pass yes with a + * second-pass no escalates to the collect's fail-closed exit. + * + * The narrow proven shape is one closed writer round of exactly the terminal + * target under a standing REVOKE_BARRIER with sane round/cutoff invariants; + * the FIFO may hold only next-round followers (they are promoted when the + * holder ticket's own later RETIRE closes the still-frozen round -- refusing + * them here would deadlock: a pre-joined follower waits for exactly this + * RETIRE to drop the barrier). A zero grant generation on the external lane + * (a cancelled blocker-participant DRAIN) is eligible too: its staged + * retirement is writer lane first, then the cancelled lane by its own + * watermark (see pcm_x_local_cross_lane_holder_terminal_retirable). + * Caller guarantees: both terminal masks complete and the same-ref-dual + * classifier returned OK with dual=false (distinct tickets). */ +static bool +pcm_x_local_independent_writer_retire_eligible(const PcmXLocalTagSlot *tag_slot, uint32 flags) +{ + const PcmXTicketRef *external_ref; + + if (tag_slot == NULL) + return false; + external_ref = pcm_x_local_external_terminal_ref(tag_slot); + return external_ref != NULL && !pcm_x_ticket_ref_is_zero(external_ref) + && external_ref->handle.ticket_id != tag_slot->ref.handle.ticket_id + && (flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 + && tag_slot->closed_round_member_count == 1 && tag_slot->membership_count >= 1 + && tag_slot->local_round != 0 && tag_slot->local_round != UINT32_MAX + && tag_slot->next_sequence != 0 && tag_slot->cutoff_sequence != 0 + && tag_slot->cutoff_sequence < tag_slot->next_sequence + && ((tag_slot->membership_count == 1) + == (tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX)) + && ((tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX) + == (tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX)) + && tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->leader_slot_generation == 0 + && tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->active_writer_slot_generation == 0 + && tag_slot->active_holder_head_index == PCM_X_INVALID_SLOT_INDEX + && pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) + && pcm_x_local_holder_lane_retire_state(tag_slot, flags) == PCM_X_QUEUE_OK; +} + + static PcmXQueueResult pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_protocol, PcmXLocalHandle *promoted_out) @@ -19029,29 +19143,20 @@ pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_pr } /* Independent dual terminal (dump-0x3d): a complete writer terminal * and a complete holder terminal under DISTINCT master tickets. Only - * the exact proven narrow shape may retire its writer lane while the - * holder lane, the barrier and the master binding stay byte-exact on - * the LIVE tag for the holder ticket's own later watermark: one - * unlinked terminal membership, every FIFO/holder index invalid and - * a clear writer application leg. Any other distinct-ticket dual is - * unproven and stays NOT_READY with zero mutation. */ + * the shared narrow eligibility (one closed writer round of exactly + * this target under the standing barrier; FIFO holds at most + * next-round followers) may retire the writer lane, keeping the + * holder lane, the barrier and the master binding byte-exact on the + * LIVE tag for the holder ticket's own later watermark. Any other + * distinct-ticket dual stays NOT_READY with zero mutation. */ if (!same_ref_dual && !cancel_same_ref_dual && (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) == PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK && (flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) == PCM_X_LOCAL_TAG_F_TERMINAL_MASK) { - const PcmXTicketRef *external_ref = pcm_x_local_external_terminal_ref(tag_slot); - retain_independent_holder - = external_ref != NULL && external_ref->grant_generation != 0 - && external_ref->handle.ticket_id != tag_slot->ref.handle.ticket_id - && tag_slot->membership_count == 1 - && tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX - && tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX - && tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX - && tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX - && tag_slot->active_holder_head_index == PCM_X_INVALID_SLOT_INDEX - && pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) - && pcm_x_local_holder_lane_retire_state(tag_slot, flags) == PCM_X_QUEUE_OK; + = member->admitted_round == tag_slot->local_round + && member->local_sequence <= tag_slot->cutoff_sequence + && pcm_x_local_independent_writer_retire_eligible(tag_slot, flags); if (!retain_independent_holder) { result = PCM_X_QUEUE_NOT_READY; goto detach_local_domain_done; @@ -19395,9 +19500,17 @@ pcm_x_local_cross_lane_holder_terminal_retirable(const PcmXLocalTagSlot *tag_slo return false; if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) != 0 || (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) - != PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK - || external_ref->grant_generation == 0) + != PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) return false; + /* A cancelled lane (zero grant generation) that shared the tag with a + * DISTINCT-ticket writer terminal is the staged second half of an + * independent dual: same-ref cancel forms route through the + * same-ref-dual path and never reach this guard, so once the writer + * lane has fully retired (no terminal bits above, ref consumed) the + * cancelled lane retires by its own watermark exactly like a granted + * one. While any writer ref remains, keep refusing it. */ + if (external_ref->grant_generation == 0) + return pcm_x_ticket_ref_is_zero(&tag_slot->ref); /* Per-session ticket ids are master-unique (monotonic allocator, exhaustion * fails closed), so id equality alone is complete for the alias class. Do * not "simplify" this to full-ref equality: that would stop refusing an id @@ -19496,20 +19609,30 @@ pcm_x_local_retire_candidate_at(Size slot_index, const PcmXRetirePayload *reques goto candidate_done; } /* An independent (distinct-ticket) dual terminal may offer its writer - * lane only in the exact proven narrow shape; see the retain arm in - * pcm_x_local_detach_terminal_common. Refusing here keeps the whole - * RETIRE preflight retryable instead of tripping the second-pass - * fail-closed exit on an unproven wider shape. */ + * lane only through the shared eligibility predicate; see the retain + * arm in pcm_x_local_detach_terminal_common. Refusing here keeps the + * whole RETIRE preflight retryable instead of tripping the + * second-pass fail-closed exit on an unproven wider shape. When the + * same watermark also covers the holder lane, rehearse the frozen + * round close it will need in its projected post-writer-retire form + * -- every fallible check must precede the second pass's first + * mutation. */ if ((flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 && !same_ref_dual - && !cancel_same_ref_dual - && (tag_slot->membership_count != 1 || tag_slot->head_index != PCM_X_INVALID_SLOT_INDEX - || tag_slot->tail_index != PCM_X_INVALID_SLOT_INDEX - || tag_slot->leader_index != PCM_X_INVALID_SLOT_INDEX - || tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX - || tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX - || !pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable))) { - result = PCM_X_QUEUE_NOT_READY; - goto candidate_done; + && !cancel_same_ref_dual) { + if (!pcm_x_local_independent_writer_retire_eligible(tag_slot, flags)) { + result = PCM_X_QUEUE_NOT_READY; + goto candidate_done; + } + external_ref = pcm_x_local_external_terminal_ref(tag_slot); + if (external_ref != NULL + && external_ref->handle.ticket_id <= request->retire_through_ticket_id) { + PcmXLocalRoundClosePlan close_plan; + + result = pcm_x_local_independent_holder_close_plan_locked(tag_slot, tag_ref, true, + &close_plan); + if (result != PCM_X_QUEUE_OK) + goto candidate_done; + } } if (tag_slot->ref.handle.ticket_id == request->retire_through_ticket_id) *contains_watermark_out = true; @@ -19534,12 +19657,33 @@ pcm_x_local_retire_candidate_at(Size slot_index, const PcmXRetirePayload *reques if (external_ref->handle.ticket_id == request->retire_through_ticket_id) *contains_watermark_out = true; if (external_ref->handle.ticket_id <= request->retire_through_ticket_id) { + /* A selected writer candidate on the same slot retires first; the + * next scan iteration re-evaluates this holder lane against the + * post-writer-retire flags (its close was already rehearsed in + * the writer arm above when the watermark covers both lanes). */ + if (!same_ref_dual && *candidate_out) + goto candidate_done; if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual && !pcm_x_local_cross_lane_holder_terminal_retirable(tag_slot, flags, external_ref)) { result = PCM_X_QUEUE_NOT_READY; goto candidate_done; } + /* When this holder RETIRE will be the last terminal authority on + * a barrier-frozen tag, rehearse the round close its detach must + * apply; the second pass repeats the same read-only rehearsal + * before consuming the lane. */ + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual + && !cancel_same_ref_dual && (flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) == 0 + && tag_slot->closed_round_member_count == 0 + && pcm_x_ticket_ref_is_zero(&tag_slot->ref)) { + PcmXLocalRoundClosePlan close_plan; + + result = pcm_x_local_independent_holder_close_plan_locked(tag_slot, tag_ref, false, + &close_plan); + if (result != PCM_X_QUEUE_OK) + goto candidate_done; + } /* A generation-zero dual terminal must validate and consume the * CANCELLED membership before its blocker evidence disappears. Keep * the writer candidate selected above; detach_terminal_common closes @@ -19573,6 +19717,7 @@ pcm_x_local_holder_detach_terminal_exact(const PcmXTicketRef *ref, int32 authent bool *ready_leader_candidate_out) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXLocalRoundClosePlan round_plan; PcmXLocalTagSlot *tag_slot; const PcmXTicketRef *external_ref; PcmXSlotRef tag_ref; @@ -19595,6 +19740,7 @@ pcm_x_local_holder_detach_terminal_exact(const PcmXTicketRef *ref, int32 authent memset(ready_leader_out, 0, sizeof(*ready_leader_out)); *ready_leader_candidate_out = false; memset(&ready_leader, 0, sizeof(ready_leader)); + memset(&round_plan, 0, sizeof(round_plan)); pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_SHARED, NULL); directory_result @@ -19656,6 +19802,26 @@ pcm_x_local_holder_detach_terminal_exact(const PcmXTicketRef *ref, int32 authent fail_closed = result == PCM_X_QUEUE_CORRUPT; goto holder_detach_domain_done; } + /* When this holder lane is the LAST terminal authority on a + * barrier-frozen tag (an independent dual whose writer lane already + * retired without closing the round), rehearse the frozen-round close + * BEFORE any evidence is consumed: next-round followers may have joined + * between the two watermarks, and consuming the holder lane without a + * validated close would strand them behind a barrier nothing can drop. + * Every fallible check lives in the rehearsal; the apply below is + * infallible. */ + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual + && (flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) == 0 + && tag_slot->closed_round_member_count == 0 && pcm_x_ticket_ref_is_zero(&tag_slot->ref)) { + result = pcm_x_local_independent_holder_close_plan_locked(tag_slot, tag_ref, false, + &round_plan); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + if (!fail_closed) + goto holder_detach_release_gate; + goto holder_detach_domain_done; + } + } if (blocker_terminal) { memset(&tag_slot->blocker_snapshot_ref, 0, sizeof(tag_slot->blocker_snapshot_ref)); tag_slot->blocker_set_generation = 0; @@ -19668,32 +19834,8 @@ pcm_x_local_holder_detach_terminal_exact(const PcmXTicketRef *ref, int32 authent } tag_slot->holder_terminal_drain_generation = 0; (void)pcm_x_slot_flags_fetch_and(&tag_slot->slot, ~PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + pcm_x_local_empty_frozen_round_apply_locked(tag_slot, &round_plan); flags = pcm_x_slot_flags_read(&tag_slot->slot); - /* An independent dual terminal retires its writer lane first while the - * holder lane keeps the writer round's REVOKE_BARRIER standing (the - * writer detach skips its final-member round close to preserve the - * holder evidence). Once the holder lane is consumed too and the tag is - * otherwise empty, close the frozen round exactly like - * pcm_x_local_empty_frozen_round_apply_locked would with no successor: - * advance the round and drop the barrier. All guards are read-only and - * the mutation is infallible, so no retryable result can strand the - * already-consumed holder lane. */ - if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 - && (flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) == 0 - && tag_slot->closed_round_member_count == 0 && tag_slot->membership_count == 0 - && tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX - && tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX - && tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX - && tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX - && tag_slot->active_holder_head_index == PCM_X_INVALID_SLOT_INDEX - && pcm_x_ticket_ref_is_zero(&tag_slot->ref) - && pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) && tag_slot->local_round != 0 - && tag_slot->local_round != UINT32_MAX) { - tag_slot->local_round++; - pg_write_barrier(); - (void)pcm_x_slot_flags_fetch_and(&tag_slot->slot, ~PCM_X_LOCAL_TAG_F_REVOKE_BARRIER); - flags = pcm_x_slot_flags_read(&tag_slot->slot); - } detach_tag = tag_slot->membership_count == 0 && tag_slot->closed_round_member_count == 0 && tag_slot->active_holder_head_index == PCM_X_INVALID_SLOT_INDEX @@ -19877,6 +20019,14 @@ pcm_x_local_final_member_round_preflight_exact(const PcmXLocalHandle *handle, result = PCM_X_QUEUE_OK; goto preflight_done; } + /* An independent (distinct-ticket) dual terminal retires its writer lane + * WITHOUT closing the round -- the holder lane keeps the fence and its + * own later RETIRE closes and promotes. Projecting the final-member + * promotion here would demand a wake the detach never produces. */ + if ((flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 && !same_ref_dual) { + result = PCM_X_QUEUE_OK; + goto preflight_done; + } result = pcm_x_local_final_member_round_plan_locked(tag_slot, tag_ref, &plan); if (result == PCM_X_QUEUE_OK && plan.candidate != NULL) { if (*projected_wake_candidate_out) { diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 98fa438bf1..68c4ec2618 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -10428,6 +10428,7 @@ typedef struct TestLocalIndependentDualTerminal { static void prepare_local_independent_dual_terminal(BlockNumber block, uint64 master_session, uint64 writer_ticket, uint64 holder_ticket, + bool cancelled_external, TestLocalIndependentDualTerminal *fixture) { PcmXShmemHeader *header; @@ -10541,24 +10542,32 @@ prepare_local_independent_dual_terminal(BlockNumber block, uint64 master_session UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, 1, master_session), PCM_X_QUEUE_OK); - memset(&revoke, 0, sizeof(revoke)); - revoke.ref = probe_ref; - revoke.ref.grant_generation = UINT64_C(2); - UT_ASSERT(cluster_pcm_x_image_id_encode(1, master_session + UINT64_C(400), &revoke.image_id)); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), - PCM_X_QUEUE_OK); - memset(&ready, 0, sizeof(ready)); - ready.ref = revoke.ref; - ready.image.image_id = revoke.image_id; - ready.image.source_own_generation = UINT64_C(61); - ready.image.page_scn = UINT64_C(62); - ready.image.page_lsn = UINT64_C(63); - ready.image.source_node = 0; - ready.image.page_checksum = UINT32_C(64); - UT_ASSERT_EQ( - cluster_pcm_x_local_holder_image_ready_arm_exact(&ready, 1, master_session, &replay), - PCM_X_QUEUE_OK); - fixture->holder_ref = revoke.ref; + if (cancelled_external) { + /* The external cohort cancels: no REVOKE/IMAGE_READY; its DRAIN + * upgrades the blocker locator in place with a zero grant + * generation (the cancelled blocker-participant form). */ + fixture->holder_ref = probe_ref; + } else { + memset(&revoke, 0, sizeof(revoke)); + revoke.ref = probe_ref; + revoke.ref.grant_generation = UINT64_C(2); + UT_ASSERT( + cluster_pcm_x_image_id_encode(1, master_session + UINT64_C(400), &revoke.image_id)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + PCM_X_QUEUE_OK); + memset(&ready, 0, sizeof(ready)); + ready.ref = revoke.ref; + ready.image.image_id = revoke.image_id; + ready.image.source_own_generation = UINT64_C(61); + ready.image.page_scn = UINT64_C(62); + ready.image.page_lsn = UINT64_C(63); + ready.image.source_node = 0; + ready.image.page_checksum = UINT32_C(64); + UT_ASSERT_EQ( + cluster_pcm_x_local_holder_image_ready_arm_exact(&ready, 1, master_session, &replay), + PCM_X_QUEUE_OK); + fixture->holder_ref = revoke.ref; + } /* Both lanes now DRAIN: the delayed writer leg first, then the holder. */ memset(&poll, 0, sizeof(poll)); @@ -10570,17 +10579,23 @@ prepare_local_independent_dual_terminal(BlockNumber block, uint64 master_session poll.drain_generation = UINT64_C(7); UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); - /* Pin the t/400 dump shape: 0x3d, two distinct tickets, one unlinked - * terminal membership, every FIFO index invalid. */ + /* Pin the t/400 dump shape byte-exactly: flags 0x3d, one closed round of + * one, two distinct tickets, one unlinked terminal membership, every + * FIFO index invalid. */ fixture->tag_slot = &local_tag_slots(header)[fixture->writer.tag_slot.slot_index]; flags = test_slot_flags(&fixture->tag_slot->slot); - UT_ASSERT((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); - UT_ASSERT_EQ(flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK, PCM_X_LOCAL_TAG_F_TERMINAL_MASK); - UT_ASSERT_EQ(flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, - PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + UT_ASSERT_EQ(flags, PCM_X_LOCAL_TAG_F_REVOKE_BARRIER | PCM_X_LOCAL_TAG_F_TERMINAL_MASK + | PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + UT_ASSERT_EQ(fixture->tag_slot->closed_round_member_count, 1); UT_ASSERT_EQ(fixture->tag_slot->ref.handle.ticket_id, writer_ticket); UT_ASSERT_EQ(fixture->tag_slot->ref.grant_generation, UINT64_C(2)); - UT_ASSERT_EQ(fixture->tag_slot->holder_ref.handle.ticket_id, holder_ticket); + if (cancelled_external) { + UT_ASSERT(ticket_refs_equal(&fixture->tag_slot->holder_ref, &(PcmXTicketRef){ 0 })); + UT_ASSERT_EQ(fixture->tag_slot->blocker_snapshot_ref.handle.ticket_id, holder_ticket); + UT_ASSERT_EQ(fixture->tag_slot->blocker_snapshot_ref.grant_generation, 0); + } else { + UT_ASSERT_EQ(fixture->tag_slot->holder_ref.handle.ticket_id, holder_ticket); + } UT_ASSERT_EQ(fixture->tag_slot->membership_count, 1); UT_ASSERT_EQ(fixture->tag_slot->head_index, PCM_X_INVALID_SLOT_INDEX); UT_ASSERT_EQ(fixture->tag_slot->tail_index, PCM_X_INVALID_SLOT_INDEX); @@ -10617,7 +10632,7 @@ UT_TEST(test_local_independent_dual_terminal_retires_writer_then_holder) const uint64 master_session = UINT64_C(1811); prepare_local_independent_dual_terminal(7117, master_session, UINT64_C(96001), UINT64_C(96002), - &fixture); + false, &fixture); tag_slot = fixture.tag_slot; holder_ref_before = tag_slot->holder_ref; holder_image_before = tag_slot->holder_image; @@ -10694,7 +10709,7 @@ UT_TEST(test_local_independent_dual_terminal_alias_and_malformed_lanes_fail_clos /* Ticket-id collision under a distinct holder ref. */ prepare_local_independent_dual_terminal(7118, master_session, UINT64_C(96001), UINT64_C(96002), - &fixture); + false, &fixture); tag_slot = fixture.tag_slot; tag_slot->holder_ref.handle.ticket_id = fixture.writer_ref.handle.ticket_id; memset(&retire, 0, sizeof(retire)); @@ -10708,7 +10723,7 @@ UT_TEST(test_local_independent_dual_terminal_alias_and_malformed_lanes_fail_clos /* Half-drained writer mask: READY without DRAINED is not independent. */ prepare_local_independent_dual_terminal(7118, master_session, UINT64_C(96001), UINT64_C(96002), - &fixture); + false, &fixture); tag_slot = fixture.tag_slot; test_clear_slot_flags(&tag_slot->slot, PCM_X_LOCAL_TAG_F_TERMINAL_DRAINED); UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), @@ -10717,7 +10732,7 @@ UT_TEST(test_local_independent_dual_terminal_alias_and_malformed_lanes_fail_clos /* Poisoned holder drain leg: a response tombstone is never clean. */ prepare_local_independent_dual_terminal(7118, master_session, UINT64_C(96001), UINT64_C(96002), - &fixture); + false, &fixture); tag_slot = fixture.tag_slot; tag_slot->holder_reliable.response_tombstone_mask = UINT32_C(1); UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), @@ -10725,6 +10740,215 @@ UT_TEST(test_local_independent_dual_terminal_alias_and_malformed_lanes_fail_clos UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); } +/* Join one next-round follower on the fixture's frozen tag; production + * admission stays legal under the barrier and links it behind the round. */ +static void +join_independent_dual_follower(const TestLocalIndependentDualTerminal *fixture, uint32 procno, + uint64 request_id, PcmXLocalHandle *follower_out) +{ + PcmXWaitIdentity identity + = make_wait_identity(fixture->writer_ref.identity.tag.blockNum, 0, procno, request_id); + + UT_ASSERT_EQ( + cluster_pcm_x_local_join_begin(&identity, 1, fixture->master_session, follower_out), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(follower_out->role, PCM_X_LOCAL_ROLE_FOLLOWER); +} + +static void +retire_watermark(uint64 cluster_epoch, uint64 master_session, uint64 through_ticket, + PcmXRetirePayload *retire_out) +{ + memset(retire_out, 0, sizeof(*retire_out)); + retire_out->cluster_epoch = cluster_epoch; + retire_out->master_session_incarnation = master_session; + retire_out->retire_through_ticket_id = through_ticket; + retire_out->sender_node = 0; +} + +/* + * RED (2026-07-19 adversarial review, leg A1): a next-round follower that + * joined BEFORE the writer's RETIRE must not push the independent dual out + * of eligibility -- the follower waits for exactly this barrier to drop, so + * refusing the writer lane here is a deterministic deadlock. The holder + * ticket's later RETIRE must close the frozen round and promote it. + */ +UT_TEST(test_local_independent_dual_terminal_supports_pre_joined_follower) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalHandle follower; + PcmXLocalHandle refreshed; + PcmXLocalTagSlot *tag_slot; + PcmXRetirePayload retire; + PcmXSlotRef found; + const uint64 master_session = UINT64_C(1815); + + prepare_local_independent_dual_terminal(7119, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + join_independent_dual_follower(&fixture, 5, master_session + UINT64_C(10), &follower); + UT_ASSERT_EQ(tag_slot->membership_count, 2); + + retire_watermark(fixture.writer_ref.identity.cluster_epoch, master_session, + fixture.writer_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_TERMINAL_MASK, 0); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(tag_slot->membership_count, 1); + UT_ASSERT_EQ(tag_slot->holder_ref.handle.ticket_id, fixture.holder_ref.handle.ticket_id); + + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER, 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&follower.identity, &refreshed), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(refreshed.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &fixture.writer_ref.identity.tag, &found), + PCM_X_DIRECTORY_OK); +} + +/* + * RED (leg A2): the same promotion must hold when the follower joins in the + * window BETWEEN the writer's RETIRE and the holder's RETIRE -- before the + * fix the holder detach consumed its lane, answered OK and left the barrier + * standing over the follower with no terminal authority left to drop it. + */ +UT_TEST(test_local_independent_dual_terminal_promotes_follower_joined_between_watermarks) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalHandle follower; + PcmXLocalHandle refreshed; + PcmXLocalTagSlot *tag_slot; + PcmXRetirePayload retire; + const uint64 master_session = UINT64_C(1817); + + prepare_local_independent_dual_terminal(7120, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + retire_watermark(fixture.writer_ref.identity.cluster_epoch, master_session, + fixture.writer_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + join_independent_dual_follower(&fixture, 5, master_session + UINT64_C(10), &follower); + UT_ASSERT_EQ(tag_slot->membership_count, 1); + + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&follower.identity, &refreshed), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(refreshed.role, PCM_X_LOCAL_ROLE_NODE_LEADER); +} + +/* + * RED (leg B): a cancelled external cohort (zero grant generation on the + * blocker locator) is the staged form of the independent dual -- the writer + * lane retires first, then the cancelled lane by its own watermark. Before + * the fix the writer RETIRE either fused (classifier) or wedged forever + * (asymmetric gen!=0 eligibility). + */ +UT_TEST(test_local_independent_dual_terminal_staged_retire_of_cancelled_external) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalTagSlot *tag_slot; + PcmXRetirePayload retire; + PcmXSlotRef found; + const uint64 master_session = UINT64_C(1819); + + prepare_local_independent_dual_terminal(7121, master_session, UINT64_C(96001), UINT64_C(96002), + true, &fixture); + tag_slot = fixture.tag_slot; + retire_watermark(fixture.writer_ref.identity.cluster_epoch, master_session, + fixture.writer_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_TERMINAL_MASK, 0); + UT_ASSERT_EQ(tag_slot->blocker_snapshot_ref.handle.ticket_id, + fixture.holder_ref.handle.ticket_id); + + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &fixture.writer_ref.identity.tag, &found), + PCM_X_DIRECTORY_NOT_FOUND); + assert_local_queue_baseline(ClusterPcmXConvertShmem); +} + +/* + * RED (leg: single watermark): one RETIRE watermark covering BOTH tickets + * must consume the writer lane and then the holder lane in the same + * episode's candidate loop and drain the tag to baseline. + */ +UT_TEST(test_local_independent_dual_terminal_single_watermark_covers_both_lanes) +{ + TestLocalIndependentDualTerminal fixture; + PcmXRetirePayload retire; + PcmXSlotRef found; + const uint64 master_session = UINT64_C(1821); + + prepare_local_independent_dual_terminal(7122, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &fixture.writer_ref.identity.tag, &found), + PCM_X_DIRECTORY_NOT_FOUND); + assert_local_queue_baseline(ClusterPcmXConvertShmem); +} + +/* + * RED (leg: non-narrow refusal is byte-exact): breaking one eligibility + * invariant must leave the whole tag slot untouched under a retryable + * NOT_READY -- never a mutation, never a fuse -- and the retire must + * succeed once the invariant is restored. + */ +UT_TEST(test_local_independent_dual_terminal_non_narrow_refusal_is_byte_exact) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalTagSlot saved; + PcmXLocalTagSlot *tag_slot; + PcmXRetirePayload retire; + uint64 cutoff_before; + const uint64 master_session = UINT64_C(1823); + + prepare_local_independent_dual_terminal(7123, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + cutoff_before = tag_slot->cutoff_sequence; + tag_slot->cutoff_sequence = tag_slot->next_sequence; + saved = *tag_slot; + retire_watermark(fixture.writer_ref.identity.cluster_epoch, master_session, + fixture.writer_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT(memcmp(&saved, tag_slot, sizeof(saved)) == 0); + + tag_slot->cutoff_sequence = cutoff_before; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + assert_local_queue_baseline(ClusterPcmXConvertShmem); +} + typedef struct TestLocalRebaseFixture { PcmXLocalHandle writer; PcmXLocalWriterClaim writer_claim; @@ -15504,7 +15728,7 @@ UT_TEST(test_local_retire_episode_lock_errors_fail_closed) int main(void) { - UT_PLAN(260); + UT_PLAN(265); UT_RUN(test_image_id_domain_is_canonical_and_bounded); UT_RUN(test_wire_abi_sizes_are_exact); UT_RUN(test_wire_abi_offsets_are_exact); @@ -15683,6 +15907,11 @@ main(void) UT_RUN(test_local_cross_lane_retire_exemption_requires_distinct_ticket); UT_RUN(test_local_independent_dual_terminal_retires_writer_then_holder); UT_RUN(test_local_independent_dual_terminal_alias_and_malformed_lanes_fail_closed); + UT_RUN(test_local_independent_dual_terminal_supports_pre_joined_follower); + UT_RUN(test_local_independent_dual_terminal_promotes_follower_joined_between_watermarks); + UT_RUN(test_local_independent_dual_terminal_staged_retire_of_cancelled_external); + UT_RUN(test_local_independent_dual_terminal_single_watermark_covers_both_lanes); + UT_RUN(test_local_independent_dual_terminal_non_narrow_refusal_is_byte_exact); UT_RUN(test_local_grant_rebase_publish_is_one_shot_and_effective); UT_RUN(test_local_grant_rebase_conflict_fails_closed); UT_RUN(test_local_follower_claim_inherits_published_rebase); From ad67e653b5f80c69f49600280b3e2bebd0c031e8 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 10:28:44 -0700 Subject: [PATCH 35/56] fix(cluster): let a cancelled next-round member detach under terminal evidence Adversarial review P0-12: a next-round follower cancelled around the independent dual's watermarks leaves an unlinked CANCELLED membership on the tag, and the generic detach refuses to touch anything while terminal evidence occupies it. Cancelled before the writer watermark, the membership wedges writer-lane eligibility forever (members>1 with an empty FIFO); cancelled between the watermarks, the holder RETIRE's frozen-round close read membership=1/empty-FIFO as corruption and fused the runtime. An exact unlinked next-round CANCELLED member now detaches by releasing only its own membership -- the dual terminals, the holder lane, the barrier and the master binding stay byte-exact -- and the frozen-round close plan treats a nonzero membership behind an empty FIFO as the legal transient it is (NOT_READY until those members' own detaches land) instead of guessing it away as corruption. RED (recorded on detached 0d26b44204 with the pure test diff): cancel-before-watermark leaves detach NOT_READY(7) and the membership stuck at 2 with every later RETIRE refusing; cancel-between-watermarks turns the holder RETIRE into CORRUPT(10) + RECOVERY_BLOCKED. GREEN: 267/267 unit, full-suite 182 binaries, check-format clean. --- src/backend/cluster/cluster_pcm_x_convert.c | 23 ++++ .../cluster_unit/test_cluster_pcm_x_convert.c | 102 +++++++++++++++++- 2 files changed, 124 insertions(+), 1 deletion(-) diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index 3456a4941b..63bbe2d36c 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -18165,6 +18165,12 @@ pcm_x_local_independent_holder_close_plan_locked(PcmXLocalTagSlot *tag_slot, Pcm projected_membership = tag_slot->membership_count; projected_closed = tag_slot->closed_round_member_count; } + /* Unlinked CANCELLED next-round members awaiting their own detach leave + * the FIFO empty while the membership count is still nonzero. That is + * a legal transient, not corruption: wait for their backends to release + * them (retire-gated episodes never race those detaches). */ + if (tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX && projected_membership > 0) + return PCM_X_QUEUE_NOT_READY; if (tag_slot->local_round == 0 || tag_slot->local_round == UINT32_MAX || tag_slot->next_sequence == 0 || tag_slot->cutoff_sequence >= tag_slot->next_sequence || projected_closed != 0 @@ -19101,6 +19107,23 @@ pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_pr goto detach_local_domain_done; } flags = pcm_x_slot_flags_read(&tag_slot->slot); + /* An exact unlinked next-round CANCELLED member detaches by releasing + * only its own membership even while terminal evidence occupies the tag: + * it was never part of the frozen round, so the dual terminals, the + * holder lane, the barrier and the master binding all stay byte-exact. + * Without this arm the cancelled membership lingers (the generic path + * refuses under terminal evidence) and wedges both the writer-lane + * eligibility and the holder RETIRE's frozen-round close. */ + if (!retire_protocol && pcm_x_slot_state_read(&member->slot) == PCM_XL_CANCELLED + && member->admitted_round == tag_slot->local_round + 1 + && (flags & (PCM_X_LOCAL_TAG_F_TERMINAL_MASK | PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK)) + != 0) { + tag_slot->membership_count--; + pg_write_barrier(); + pcm_x_slot_state_write(&member->slot, PCM_XL_DETACHING); + result = PCM_X_QUEUE_OK; + goto detach_local_domain_done; + } if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) != 0) { uint32 terminal_flags = flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK; diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 68c4ec2618..cfe16575cc 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -10911,6 +10911,104 @@ UT_TEST(test_local_independent_dual_terminal_single_watermark_covers_both_lanes) assert_local_queue_baseline(ClusterPcmXConvertShmem); } +/* + * RED (2026-07-20 adversarial review P0-12, leg 1): a next-round follower + * cancelled BEFORE the writer's watermark leaves an unlinked CANCELLED + * membership on the tag. Its own detach must release exactly that + * membership while both terminals, the barrier and the master binding stay + * byte-exact -- before the fix the detach answered NOT_READY under the + * writer terminal and the eligibility mismatch (members>1, empty FIFO) + * refused the writer lane forever. + */ +UT_TEST(test_local_independent_dual_terminal_cancel_before_writer_watermark_converges) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalHandle follower; + PcmXLocalTagSlot *tag_slot; + PcmXRetirePayload retire; + uint32 flags; + const uint64 master_session = UINT64_C(1825); + + prepare_local_independent_dual_terminal(7124, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + join_independent_dual_follower(&fixture, 5, master_session + UINT64_C(10), &follower); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(tag_slot->membership_count, 2); + UT_ASSERT_EQ(tag_slot->head_index, PCM_X_INVALID_SLOT_INDEX); + + /* Writer lane refuses retryably while the cancelled membership lingers. */ + retire_watermark(fixture.writer_ref.identity.cluster_epoch, master_session, + fixture.writer_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + + /* The cancelled member's own detach releases only its membership. */ + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower), PCM_X_QUEUE_OK); + flags = test_slot_flags(&tag_slot->slot); + UT_ASSERT_EQ(flags, PCM_X_LOCAL_TAG_F_REVOKE_BARRIER | PCM_X_LOCAL_TAG_F_TERMINAL_MASK + | PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + UT_ASSERT_EQ(tag_slot->membership_count, 1); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 1); + UT_ASSERT_EQ(tag_slot->ref.handle.ticket_id, fixture.writer_ref.handle.ticket_id); + UT_ASSERT_EQ(tag_slot->holder_ref.handle.ticket_id, fixture.holder_ref.handle.ticket_id); + + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + assert_local_queue_baseline(ClusterPcmXConvertShmem); +} + +/* + * RED (P0-12, leg 2): the follower cancelled BETWEEN the two watermarks. + * Before the fix the holder RETIRE's frozen-round close saw membership=1 + * with an empty FIFO and fused the runtime; it must instead wait retryably + * for the cancelled member's own detach, byte-exact, then close. + */ +UT_TEST(test_local_independent_dual_terminal_cancel_between_watermarks_converges) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalHandle follower; + PcmXLocalTagSlot *tag_slot; + PcmXRetirePayload retire; + uint64 holder_drain_before; + const uint64 master_session = UINT64_C(1827); + + prepare_local_independent_dual_terminal(7125, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + retire_watermark(fixture.writer_ref.identity.cluster_epoch, master_session, + fixture.writer_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + join_independent_dual_follower(&fixture, 5, master_session + UINT64_C(10), &follower); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_OK); + holder_drain_before = tag_slot->holder_terminal_drain_generation; + + /* Holder RETIRE waits retryably; the holder lane stays byte-exact. */ + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, + PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(tag_slot->holder_terminal_drain_generation, holder_drain_before); + UT_ASSERT_EQ(tag_slot->holder_ref.handle.ticket_id, fixture.holder_ref.handle.ticket_id); + + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + assert_local_queue_baseline(ClusterPcmXConvertShmem); +} + /* * RED (leg: non-narrow refusal is byte-exact): breaking one eligibility * invariant must leave the whole tag slot untouched under a retryable @@ -15728,7 +15826,7 @@ UT_TEST(test_local_retire_episode_lock_errors_fail_closed) int main(void) { - UT_PLAN(265); + UT_PLAN(267); UT_RUN(test_image_id_domain_is_canonical_and_bounded); UT_RUN(test_wire_abi_sizes_are_exact); UT_RUN(test_wire_abi_offsets_are_exact); @@ -15909,6 +16007,8 @@ main(void) UT_RUN(test_local_independent_dual_terminal_alias_and_malformed_lanes_fail_closed); UT_RUN(test_local_independent_dual_terminal_supports_pre_joined_follower); UT_RUN(test_local_independent_dual_terminal_promotes_follower_joined_between_watermarks); + UT_RUN(test_local_independent_dual_terminal_cancel_before_writer_watermark_converges); + UT_RUN(test_local_independent_dual_terminal_cancel_between_watermarks_converges); UT_RUN(test_local_independent_dual_terminal_staged_retire_of_cancelled_external); UT_RUN(test_local_independent_dual_terminal_single_watermark_covers_both_lanes); UT_RUN(test_local_independent_dual_terminal_non_narrow_refusal_is_byte_exact); From c53af5afa2370e554f21fa8c997b5457c054d5e7 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 10:38:58 -0700 Subject: [PATCH 36/56] fix(cluster): deliver the frozen-round promotion through the RETIRE wake batch Review P1 on the independent dual-terminal close: the holder RETIRE's frozen-round close promotes the next-round follower but the promotion never reached the wake batch -- the preflight rehearsal dropped its close plan's candidate, and the detach collected its wake output before the promotion applied. Production wakes only what the batch carries, so a promoted hot-block leader degraded to its 2-32ms poll interval. The preflight (both the holder arm and the writer arm's projected single-watermark rehearsal) now projects the close plan's candidate into the wake batch, and the holder detach backfills the promoted identity into its wake output after the infallible apply. A zero capacity batch therefore refuses the whole episode with NO_CAPACITY before any mutation. RED (recorded on detached ad67e653b5 with the pure test diff): both new collect legs fail -- capacity 0 answered OK and mutated the tag, capacity 1 delivered count=0 with the promoted identity missing. GREEN: 269/269 unit, full-suite 182 binaries, check-format clean. --- src/backend/cluster/cluster_pcm_x_convert.c | 21 +++- .../cluster_unit/test_cluster_pcm_x_convert.c | 110 +++++++++++++++++- 2 files changed, 129 insertions(+), 2 deletions(-) diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index 63bbe2d36c..bcd0f5782a 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -19655,6 +19655,13 @@ pcm_x_local_retire_candidate_at(Size slot_index, const PcmXRetirePayload *reques &close_plan); if (result != PCM_X_QUEUE_OK) goto candidate_done; + /* The close this episode will apply promotes that follower; + * project its wake now or the promotion degrades to the + * waiters' poll interval. */ + if (close_plan.candidate != NULL) { + *holder_wake_out = close_plan.candidate->identity; + *holder_wake_candidate_out = true; + } } } if (tag_slot->ref.handle.ticket_id == request->retire_through_ticket_id) @@ -19706,6 +19713,11 @@ pcm_x_local_retire_candidate_at(Size slot_index, const PcmXRetirePayload *reques &close_plan); if (result != PCM_X_QUEUE_OK) goto candidate_done; + /* Project the promotion this close will apply. */ + if (close_plan.candidate != NULL) { + *holder_wake_out = close_plan.candidate->identity; + *holder_wake_candidate_out = true; + } } /* A generation-zero dual terminal must validate and consume the * CANCELLED membership before its blocker evidence disappears. Keep @@ -19723,7 +19735,7 @@ pcm_x_local_retire_candidate_at(Size slot_index, const PcmXRetirePayload *reques *candidate_out = true; } } - if (result == PCM_X_QUEUE_OK && *holder_candidate_out) + if (result == PCM_X_QUEUE_OK && *holder_candidate_out && !*holder_wake_candidate_out) result = pcm_x_local_ready_leader_wake_locked(tag_slot, tag_ref, holder_wake_out, NULL, holder_wake_candidate_out); @@ -19858,6 +19870,13 @@ pcm_x_local_holder_detach_terminal_exact(const PcmXTicketRef *ref, int32 authent tag_slot->holder_terminal_drain_generation = 0; (void)pcm_x_slot_flags_fetch_and(&tag_slot->slot, ~PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); pcm_x_local_empty_frozen_round_apply_locked(tag_slot, &round_plan); + /* Report the promoted next-round leader through the same wake output the + * preflight projected; without it the promotion is only discovered by + * the waiter's poll interval. */ + if (round_plan.close_round && round_plan.candidate != NULL) { + ready_leader = round_plan.candidate->identity; + ready_leader_candidate = true; + } flags = pcm_x_slot_flags_read(&tag_slot->slot); detach_tag = tag_slot->membership_count == 0 && tag_slot->closed_round_member_count == 0 diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index cfe16575cc..152c06c282 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -11009,6 +11009,112 @@ UT_TEST(test_local_independent_dual_terminal_cancel_between_watermarks_converges assert_local_queue_baseline(ClusterPcmXConvertShmem); } +/* + * RED (2026-07-20 review P1): the frozen-round close's follower promotion + * must travel through the RETIRE wake batch -- capacity 0 refuses the whole + * episode with NO_CAPACITY and zero mutation, capacity 1 delivers exactly + * the promoted identity. Without it the promotion is only discovered by + * the waiter's poll interval (a 2-32ms hot-block regression). + */ +UT_TEST(test_local_independent_dual_terminal_collect_reports_promotion_wake) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalHandle follower; + PcmXLocalHandle refreshed; + PcmXLocalTagSlot saved; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity wake_items[2]; + PcmXLocalWakeBatch wake_batch; + PcmXRetirePayload retire; + const uint64 master_session = UINT64_C(1829); + + prepare_local_independent_dual_terminal(7126, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + retire_watermark(fixture.writer_ref.identity.cluster_epoch, master_session, + fixture.writer_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + join_independent_dual_follower(&fixture, 5, master_session + UINT64_C(10), &follower); + + /* Zero capacity refuses before any mutation. */ + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + wake_batch.items = NULL; + wake_batch.capacity = 0; + wake_batch.count = 0; + saved = *tag_slot; + UT_ASSERT_EQ( + cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, master_session, &wake_batch), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT(memcmp(&saved, tag_slot, sizeof(saved)) == 0); + + /* Capacity one delivers exactly the promoted follower. */ + memset(wake_items, 0, sizeof(wake_items)); + wake_batch.items = wake_items; + wake_batch.capacity = 1; + wake_batch.count = 0; + UT_ASSERT_EQ( + cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, master_session, &wake_batch), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(wake_batch.count, 1); + UT_ASSERT(memcmp(&wake_items[0], &follower.identity, sizeof(wake_items[0])) == 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&follower.identity, &refreshed), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(refreshed.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +/* + * RED (review P1, single-watermark form): one watermark covering both lanes + * with a pre-joined follower must deliver the same promotion wake through + * the batch it validated in the writer arm's projected rehearsal. + */ +UT_TEST(test_local_independent_dual_terminal_single_watermark_reports_promotion_wake) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalHandle follower; + PcmXLocalHandle refreshed; + PcmXLocalTagSlot saved; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity wake_items[2]; + PcmXLocalWakeBatch wake_batch; + PcmXRetirePayload retire; + const uint64 master_session = UINT64_C(1831); + + prepare_local_independent_dual_terminal(7127, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + join_independent_dual_follower(&fixture, 5, master_session + UINT64_C(10), &follower); + + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + wake_batch.items = NULL; + wake_batch.capacity = 0; + wake_batch.count = 0; + saved = *tag_slot; + UT_ASSERT_EQ( + cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, master_session, &wake_batch), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT(memcmp(&saved, tag_slot, sizeof(saved)) == 0); + + memset(wake_items, 0, sizeof(wake_items)); + wake_batch.items = wake_items; + wake_batch.capacity = 1; + wake_batch.count = 0; + UT_ASSERT_EQ( + cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, master_session, &wake_batch), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(wake_batch.count, 1); + UT_ASSERT(memcmp(&wake_items[0], &follower.identity, sizeof(wake_items[0])) == 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&follower.identity, &refreshed), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(refreshed.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + /* * RED (leg: non-narrow refusal is byte-exact): breaking one eligibility * invariant must leave the whole tag slot untouched under a retryable @@ -15826,7 +15932,7 @@ UT_TEST(test_local_retire_episode_lock_errors_fail_closed) int main(void) { - UT_PLAN(267); + UT_PLAN(269); UT_RUN(test_image_id_domain_is_canonical_and_bounded); UT_RUN(test_wire_abi_sizes_are_exact); UT_RUN(test_wire_abi_offsets_are_exact); @@ -16009,6 +16115,8 @@ main(void) UT_RUN(test_local_independent_dual_terminal_promotes_follower_joined_between_watermarks); UT_RUN(test_local_independent_dual_terminal_cancel_before_writer_watermark_converges); UT_RUN(test_local_independent_dual_terminal_cancel_between_watermarks_converges); + UT_RUN(test_local_independent_dual_terminal_collect_reports_promotion_wake); + UT_RUN(test_local_independent_dual_terminal_single_watermark_reports_promotion_wake); UT_RUN(test_local_independent_dual_terminal_staged_retire_of_cancelled_external); UT_RUN(test_local_independent_dual_terminal_single_watermark_covers_both_lanes); UT_RUN(test_local_independent_dual_terminal_non_narrow_refusal_is_byte_exact); From 9b651a96cb0c03df9ff7355d592f72af24d693a7 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 10:54:41 -0700 Subject: [PATCH 37/56] fix(cluster): report the diverged descriptor tuple when an exact finish fails The fast-fail finish family (result=1 STALE at the exact grant finish, now the leading first cause in loop8: three of four runs, all four clients at the same instant, queue draining cleanly afterwards) rolls back a durable master grant because the live ownership tuple no longer matches the reservation base -- but the error never said WHICH of the five compared terms (tag, generation, token, flags, pcm_state) moved, or to what, so the producer of the divergence cannot be named. Thread the classification-time live sample out of cluster_pcm_own_finish_grant_reservation (the post-rollback descriptor is useless: the rollback itself mutates flags and possibly the generation) and carry live-vs-base into the existing error detail. Cold path only; no behavior change. --- src/backend/storage/buffer/bufmgr.c | 31 ++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 47729f688c..c68705c1ab 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -819,10 +819,18 @@ cluster_bufmgr_pcm_own_handoff_s_revoke_to_x_reservation( out_token); } +/* + * PGRAC: live_out, when non-NULL, receives the descriptor tuple sampled + * under the header lock at classification time, so a failing finish can + * report exactly which term diverged from the reservation base. The + * post-rollback descriptor is useless for that: the rollback itself + * mutates the flags and possibly the generation. + */ static ClusterPcmOwnResult cluster_pcm_own_finish_grant_reservation(BufferDesc *buf, const ClusterPcmOwnSnapshot *base, uint64 reservation_token, uint8 new_pcm_state, - uint64 *out_committed_generation) + uint64 *out_committed_generation, + ClusterPcmOwnSnapshot *live_out) { ClusterPcmOwnSnapshot live; ClusterPcmXGrantReservationKind kind; @@ -839,6 +847,8 @@ cluster_pcm_own_finish_grant_reservation(BufferDesc *buf, const ClusterPcmOwnSna buf_state = LockBufHdr(buf); cluster_pcm_own_snapshot_locked(buf, &live); + if (live_out != NULL) + *live_out = live; kind = cluster_pcm_x_grant_reservation_kind(&live, base, reservation_token); if (kind == CLUSTER_PCM_X_GRANT_RESERVATION_INVALID) { ClusterPcmOwnResult live_shape @@ -871,7 +881,8 @@ cluster_bufmgr_pcm_own_finish_x_commit(BufferDesc *buf, const ClusterPcmOwnSnaps && expected->pcm_state != (uint8)PCM_STATE_X)) return CLUSTER_PCM_OWN_STALE; return cluster_pcm_own_finish_grant_reservation(buf, expected, reservation_token, - (uint8)PCM_STATE_X, out_committed_generation); + (uint8)PCM_STATE_X, out_committed_generation, + NULL); } static ClusterPcmOwnResult @@ -1050,11 +1061,13 @@ cluster_pcm_own_finish_grant_or_rollback(BufferDesc *buf, const ClusterPcmOwnSna PcmLockMode acquired_mode, uint64 *out_committed_generation) { + ClusterPcmOwnSnapshot live; ClusterPcmOwnResult finish_result; ClusterPcmOwnResult rollback_result; + memset(&live, 0, sizeof(live)); finish_result = cluster_pcm_own_finish_grant_reservation( - buf, base, reservation_token, new_pcm_state, out_committed_generation); + buf, base, reservation_token, new_pcm_state, out_committed_generation, &live); if (finish_result == CLUSTER_PCM_OWN_OK) return; @@ -1095,8 +1108,16 @@ cluster_pcm_own_finish_grant_or_rollback(BufferDesc *buf, const ClusterPcmOwnSna (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not finish exact cluster PCM grant reservation: result=%d", (int)finish_result), - errdetail("master grant was rolled back for buffer %d token=%llu", - buf != NULL ? buf->buf_id : -1, (unsigned long long)reservation_token))); + errdetail("master grant was rolled back for buffer %d token=%llu; " + "live state=%u gen=%llu token=%llu flags=%u vs " + "base state=%u gen=%llu token=%llu flags=%u", + buf != NULL ? buf->buf_id : -1, (unsigned long long)reservation_token, + (unsigned int)live.pcm_state, (unsigned long long)live.generation, + (unsigned long long)live.reservation_token, (unsigned int)live.flags, + base != NULL ? (unsigned int)base->pcm_state : 0, + (unsigned long long)(base != NULL ? base->generation : 0), + (unsigned long long)(base != NULL ? base->reservation_token : 0), + base != NULL ? (unsigned int)base->flags : 0))); } ClusterPcmOwnResult From f40e54a6f148f2d60fc51d8f3506dc1d6c567555 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 11:43:21 -0700 Subject: [PATCH 38/56] fix(cluster): enumerate the S-holder X upgrade as a legal grant finish shape The fast-fail finish family (S3 errors=0 hard blocker; leading first cause in loop7/loop8) is not a race at all: an S holder upgrading to X through the legacy master acquire begins a fresh-token GRANT_PENDING reservation from S/flags=0 -- the reservation begin snapshots whatever coherent state it finds, and the S-base abort path has always demoted S to N on rollback for exactly that shape -- but the grant-reservation classifier enumerated only N_NEW and the three revoke handoffs. Every such upgrade therefore failed its finish deterministically, rolled a durable master grant back and surfaced result=1 to the client; the loop9 evidence detail shows the live tuple matching the reservation identity exactly (S, same generation, token+1, GRANT_PENDING) against the S/flags=0 base. Add the S_UPGRADE arm for that shape. The finish gate already demands PCM_STATE_X plus a held content lock for every non-N_NEW kind, the grant commit is state-agnostic (one generation bump, flags cleared), and the queue-lane consumers compare against explicit expected kinds, so nothing else widens. A fresh-token X base stays deliberately unenumerated: a live X cover never re-acquires through this path. RED (recorded on detached 9b651a96cb with the pure test diff): the shape cannot even be named -- use of undeclared identifier CLUSTER_PCM_X_GRANT_RESERVATION_S_UPGRADE at the first assertion. GREEN: 47/47 pcm_own unit, full-suite 182 binaries, check-format clean. --- src/include/cluster/cluster_pcm_x_bufmgr.h | 27 ++++++---- src/test/cluster_unit/test_cluster_pcm_own.c | 55 +++++++++++++++++++- 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/src/include/cluster/cluster_pcm_x_bufmgr.h b/src/include/cluster/cluster_pcm_x_bufmgr.h index 614aa3455b..bc9c66f45d 100644 --- a/src/include/cluster/cluster_pcm_x_bufmgr.h +++ b/src/include/cluster/cluster_pcm_x_bufmgr.h @@ -191,14 +191,20 @@ typedef enum ClusterPcmXGrantReservationKind { CLUSTER_PCM_X_GRANT_RESERVATION_N_NEW, CLUSTER_PCM_X_GRANT_RESERVATION_N_REVOKE_HANDOFF, CLUSTER_PCM_X_GRANT_RESERVATION_S_REVOKE_HANDOFF, - CLUSTER_PCM_X_GRANT_RESERVATION_X_REVOKE_HANDOFF + CLUSTER_PCM_X_GRANT_RESERVATION_X_REVOKE_HANDOFF, + CLUSTER_PCM_X_GRANT_RESERVATION_S_UPGRADE } ClusterPcmXGrantReservationKind; /* Classify every legal PREPARE reservation shape. Ordinary remote-image N - * acquisition allocates the next monotonic token. A requester acting as its - * own N/S/X image source instead reuses the already-live revoke token and - * changes only its role. Every shape retains tag/generation/pcm_state until - * the single grant commit. */ + * acquisition allocates the next monotonic token, and so does an S holder + * upgrading itself to X through the legacy master acquire (the reservation + * begin snapshots whatever coherent state it finds; the S-base abort path + * has always demoted S to N on rollback for exactly that shape). A + * requester acting as its own N/S/X image source instead reuses the + * already-live revoke token and changes only its role. Every shape retains + * tag/generation/pcm_state until the single grant commit. A fresh-token X + * base is deliberately NOT enumerated: a live X cover never re-acquires + * through this path (cluster_pcm_x_cached_cover_bypasses_queue). */ static inline ClusterPcmXGrantReservationKind cluster_pcm_x_grant_reservation_kind(const ClusterPcmOwnSnapshot *live, const ClusterPcmOwnSnapshot *base, uint64 reservation_token) @@ -209,10 +215,13 @@ cluster_pcm_x_grant_reservation_kind(const ClusterPcmOwnSnapshot *live, || live->pcm_state != base->pcm_state) return CLUSTER_PCM_X_GRANT_RESERVATION_INVALID; - if (base->pcm_state == (uint8)PCM_STATE_N && base->flags == 0 - && base->reservation_token != UINT64_MAX - && reservation_token == base->reservation_token + 1) - return CLUSTER_PCM_X_GRANT_RESERVATION_N_NEW; + if (base->flags == 0 && base->reservation_token != UINT64_MAX + && reservation_token == base->reservation_token + 1) { + if (base->pcm_state == (uint8)PCM_STATE_N) + return CLUSTER_PCM_X_GRANT_RESERVATION_N_NEW; + if (base->pcm_state == (uint8)PCM_STATE_S) + return CLUSTER_PCM_X_GRANT_RESERVATION_S_UPGRADE; + } if (base->flags == PCM_OWN_FLAG_REVOKING && base->reservation_token != 0 && reservation_token == base->reservation_token) { if (base->pcm_state == (uint8)PCM_STATE_N) diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index 29e05438f5..2dda1312f0 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -321,6 +321,58 @@ UT_TEST(test_revoke_handoff_kinds_cover_n_s_x_with_one_lifecycle) } } +/* + * RED for the t/400 fast-fail finish family (2026-07-20, loop9 DETAIL): an + * S holder upgrading to X through the legacy master acquire begins a + * fresh-token GRANT_PENDING reservation from S/flags=0 -- the begin side + * and the S-base abort path have always supported that shape -- but the + * grant-reservation classifier only enumerated N_NEW and the three revoke + * handoffs, so EVERY such upgrade failed its finish deterministically, + * rolled back a durable master grant and surfaced result=1 to the client + * (live S/gen/token+1/GRANT_PENDING vs base S/gen/token/0). + */ +UT_TEST(test_s_upgrade_fresh_token_is_a_legal_grant_finish_shape) +{ + ClusterPcmOwnSnapshot base; + ClusterPcmOwnSnapshot live; + uint64 committed = UINT64_MAX; + uint64 token; + + /* The exact loop9 production tuple. */ + memset(&base, 0, sizeof(base)); + base.generation = 10; + base.reservation_token = 5; + base.pcm_state = (uint8)PCM_STATE_S; + live = base; + live.reservation_token = 6; + live.flags = PCM_OWN_FLAG_GRANT_PENDING; + UT_ASSERT_EQ(cluster_pcm_x_grant_reservation_kind(&live, &base, 6), + CLUSTER_PCM_X_GRANT_RESERVATION_S_UPGRADE); + + /* A non-adjacent token is never an upgrade. */ + live.reservation_token = 7; + UT_ASSERT_EQ(cluster_pcm_x_grant_reservation_kind(&live, &base, 7), + CLUSTER_PCM_X_GRANT_RESERVATION_INVALID); + + /* A fresh-token X base stays deliberately unenumerated: a live X cover + * never re-acquires through this path. */ + base.pcm_state = (uint8)PCM_STATE_X; + live = base; + live.reservation_token = 6; + live.flags = PCM_OWN_FLAG_GRANT_PENDING; + UT_ASSERT_EQ(cluster_pcm_x_grant_reservation_kind(&live, &base, 6), + CLUSTER_PCM_X_GRANT_RESERVATION_INVALID); + + /* Full entry lifecycle: begin from the S base, commit bumps once. */ + reset_fixture(); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].generation, 7); + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 7, PCM_OWN_FLAG_GRANT_PENDING, &token), + CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ(cluster_pcm_own_grant_commit_exact(0, 7, token, &committed), CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ(committed, 8); + assert_entry(8, token, 0); +} + UT_TEST(test_revoke_commit_is_exact_and_classifies_live_races) { uint64 committed = UINT64_MAX; @@ -2098,13 +2150,14 @@ UT_TEST(test_preflight_busy_waits_then_clean_resnapshot_begins_reservation) int main(void) { - UT_PLAN(46); + UT_PLAN(47); UT_RUN(test_shmem_initializes_complete_entry); UT_RUN(test_begin_abort_is_exact_and_monotonic); UT_RUN(test_invalid_live_flag_shapes_are_corrupt_not_busy); UT_RUN(test_grant_commit_is_exact_and_bumps_once); UT_RUN(test_s_revoke_handoff_reuses_exact_token_and_bumps_once); UT_RUN(test_revoke_handoff_kinds_cover_n_s_x_with_one_lifecycle); + UT_RUN(test_s_upgrade_fresh_token_is_a_legal_grant_finish_shape); UT_RUN(test_revoke_commit_is_exact_and_classifies_live_races); UT_RUN(test_revoke_retain_commit_keeps_exact_token_until_release); UT_RUN(test_revoke_commit_exhaustion_is_side_effect_free); From 775517302f6b55672364b61570628a4557dafee0 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 12:39:42 -0700 Subject: [PATCH 39/56] Revert "fix(cluster): enumerate the S-holder X upgrade as a legal grant finish shape" Adversarial review caught a protocol hazard in f40e54a6f1 before it could regress S3: legalizing the fresh-token S-base finish shape lets the stale-cover fallback bypass the convert queue's FIFO/WFG writer arbitration entirely -- the original unordered multi-writer defect the queue exists to prevent. The reviewed production chain is: cached-X cover sampled -> in-window X->S downgrade -> revalidate detects the stale cover -> the fallback WRONGLY enters the legacy master acquire from the S base and mints the "S_NEW" tuple. loop10b (clean rebuild) confirmed f40e was not a closure anyway: the same shape now surfaced result=6 (INVALID) from the finish gate's next check. Restore the classifier to refuse the shape, and flip the unit leg into a protocol pin: S_NEW (and the X-base analogue) stay INVALID. The real fix is queue re-entry at the stale-cover fallback -- evidence first via the cached-X stall TAP leg -- never a finish-side swallow of STALE/INVALID. --- src/include/cluster/cluster_pcm_x_bufmgr.h | 37 +++++++++-------- src/test/cluster_unit/test_cluster_pcm_own.c | 43 +++++++------------- 2 files changed, 33 insertions(+), 47 deletions(-) diff --git a/src/include/cluster/cluster_pcm_x_bufmgr.h b/src/include/cluster/cluster_pcm_x_bufmgr.h index bc9c66f45d..6d0b013199 100644 --- a/src/include/cluster/cluster_pcm_x_bufmgr.h +++ b/src/include/cluster/cluster_pcm_x_bufmgr.h @@ -191,20 +191,24 @@ typedef enum ClusterPcmXGrantReservationKind { CLUSTER_PCM_X_GRANT_RESERVATION_N_NEW, CLUSTER_PCM_X_GRANT_RESERVATION_N_REVOKE_HANDOFF, CLUSTER_PCM_X_GRANT_RESERVATION_S_REVOKE_HANDOFF, - CLUSTER_PCM_X_GRANT_RESERVATION_X_REVOKE_HANDOFF, - CLUSTER_PCM_X_GRANT_RESERVATION_S_UPGRADE + CLUSTER_PCM_X_GRANT_RESERVATION_X_REVOKE_HANDOFF } ClusterPcmXGrantReservationKind; /* Classify every legal PREPARE reservation shape. Ordinary remote-image N - * acquisition allocates the next monotonic token, and so does an S holder - * upgrading itself to X through the legacy master acquire (the reservation - * begin snapshots whatever coherent state it finds; the S-base abort path - * has always demoted S to N on rollback for exactly that shape). A - * requester acting as its own N/S/X image source instead reuses the - * already-live revoke token and changes only its role. Every shape retains - * tag/generation/pcm_state until the single grant commit. A fresh-token X - * base is deliberately NOT enumerated: a live X cover never re-acquires - * through this path (cluster_pcm_x_cached_cover_bypasses_queue). */ + * acquisition allocates the next monotonic token. A requester acting as its + * own N/S/X image source instead reuses the already-live revoke token and + * changes only its role. Every shape retains tag/generation/pcm_state until + * the single grant commit. + * + * A fresh-token S base (an S holder entering the legacy master acquire, the + * loop9/loop10b "S_NEW" tuple) is deliberately NOT a legal finish shape: + * writer conversions are ordered by the convert queue's FIFO/WFG, and + * legalizing the legacy S-base short cut here would let a stale-cover + * fallback bypass that arbitration entirely (the original S3 unordered + * multi-writer defect). The stale-cover path must re-enter the queue + * instead; this classifier keeps refusing the bypass. A fresh-token X base + * is equally unenumerated: a live X cover never re-acquires through this + * path (cluster_pcm_x_cached_cover_bypasses_queue). */ static inline ClusterPcmXGrantReservationKind cluster_pcm_x_grant_reservation_kind(const ClusterPcmOwnSnapshot *live, const ClusterPcmOwnSnapshot *base, uint64 reservation_token) @@ -215,13 +219,10 @@ cluster_pcm_x_grant_reservation_kind(const ClusterPcmOwnSnapshot *live, || live->pcm_state != base->pcm_state) return CLUSTER_PCM_X_GRANT_RESERVATION_INVALID; - if (base->flags == 0 && base->reservation_token != UINT64_MAX - && reservation_token == base->reservation_token + 1) { - if (base->pcm_state == (uint8)PCM_STATE_N) - return CLUSTER_PCM_X_GRANT_RESERVATION_N_NEW; - if (base->pcm_state == (uint8)PCM_STATE_S) - return CLUSTER_PCM_X_GRANT_RESERVATION_S_UPGRADE; - } + if (base->pcm_state == (uint8)PCM_STATE_N && base->flags == 0 + && base->reservation_token != UINT64_MAX + && reservation_token == base->reservation_token + 1) + return CLUSTER_PCM_X_GRANT_RESERVATION_N_NEW; if (base->flags == PCM_OWN_FLAG_REVOKING && base->reservation_token != 0 && reservation_token == base->reservation_token) { if (base->pcm_state == (uint8)PCM_STATE_N) diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index 2dda1312f0..ad5048729f 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -322,23 +322,22 @@ UT_TEST(test_revoke_handoff_kinds_cover_n_s_x_with_one_lifecycle) } /* - * RED for the t/400 fast-fail finish family (2026-07-20, loop9 DETAIL): an - * S holder upgrading to X through the legacy master acquire begins a - * fresh-token GRANT_PENDING reservation from S/flags=0 -- the begin side - * and the S-base abort path have always supported that shape -- but the - * grant-reservation classifier only enumerated N_NEW and the three revoke - * handoffs, so EVERY such upgrade failed its finish deterministically, - * rolled back a durable master grant and surfaced result=1 to the client - * (live S/gen/token+1/GRANT_PENDING vs base S/gen/token/0). + * Protocol pin for the t/400 fast-fail finish family (2026-07-20, loop9 and + * loop10b DETAIL): a fresh-token GRANT_PENDING reservation taken from an + * S/flags=0 base ("S_NEW" -- a stale-cover fallback wrongly entering the + * legacy master acquire after an in-window X->S downgrade) must NEVER + * become a legal finish shape. Writer conversions are ordered by the + * convert queue's FIFO/WFG; legalizing this shape at the finish would let + * that fallback bypass the arbitration entirely (the original S3 unordered + * multi-writer defect). The fallback must re-enter the queue instead; + * this classifier keeps refusing the bypass. */ -UT_TEST(test_s_upgrade_fresh_token_is_a_legal_grant_finish_shape) +UT_TEST(test_s_new_fresh_token_finish_shape_stays_invalid) { ClusterPcmOwnSnapshot base; ClusterPcmOwnSnapshot live; - uint64 committed = UINT64_MAX; - uint64 token; - /* The exact loop9 production tuple. */ + /* The exact loop9/loop10b production tuple stays refused. */ memset(&base, 0, sizeof(base)); base.generation = 10; base.reservation_token = 5; @@ -347,30 +346,16 @@ UT_TEST(test_s_upgrade_fresh_token_is_a_legal_grant_finish_shape) live.reservation_token = 6; live.flags = PCM_OWN_FLAG_GRANT_PENDING; UT_ASSERT_EQ(cluster_pcm_x_grant_reservation_kind(&live, &base, 6), - CLUSTER_PCM_X_GRANT_RESERVATION_S_UPGRADE); - - /* A non-adjacent token is never an upgrade. */ - live.reservation_token = 7; - UT_ASSERT_EQ(cluster_pcm_x_grant_reservation_kind(&live, &base, 7), CLUSTER_PCM_X_GRANT_RESERVATION_INVALID); - /* A fresh-token X base stays deliberately unenumerated: a live X cover - * never re-acquires through this path. */ + /* A fresh-token X base is refused the same way: a live X cover never + * re-acquires through this path. */ base.pcm_state = (uint8)PCM_STATE_X; live = base; live.reservation_token = 6; live.flags = PCM_OWN_FLAG_GRANT_PENDING; UT_ASSERT_EQ(cluster_pcm_x_grant_reservation_kind(&live, &base, 6), CLUSTER_PCM_X_GRANT_RESERVATION_INVALID); - - /* Full entry lifecycle: begin from the S base, commit bumps once. */ - reset_fixture(); - pg_atomic_write_u64(&ClusterPcmOwnArray[0].generation, 7); - UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 7, PCM_OWN_FLAG_GRANT_PENDING, &token), - CLUSTER_PCM_OWN_OK); - UT_ASSERT_EQ(cluster_pcm_own_grant_commit_exact(0, 7, token, &committed), CLUSTER_PCM_OWN_OK); - UT_ASSERT_EQ(committed, 8); - assert_entry(8, token, 0); } UT_TEST(test_revoke_commit_is_exact_and_classifies_live_races) @@ -2157,7 +2142,7 @@ main(void) UT_RUN(test_grant_commit_is_exact_and_bumps_once); UT_RUN(test_s_revoke_handoff_reuses_exact_token_and_bumps_once); UT_RUN(test_revoke_handoff_kinds_cover_n_s_x_with_one_lifecycle); - UT_RUN(test_s_upgrade_fresh_token_is_a_legal_grant_finish_shape); + UT_RUN(test_s_new_fresh_token_finish_shape_stays_invalid); UT_RUN(test_revoke_commit_is_exact_and_classifies_live_races); UT_RUN(test_revoke_retain_commit_keeps_exact_token_until_release); UT_RUN(test_revoke_commit_exhaustion_is_side_effect_free); From ec2c84d0b6343d3e1776d3ccbe5c6574a356f5c2 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 12:51:30 -0700 Subject: [PATCH 40/56] fix(cluster): re-enter the convert queue when a stale X cover needs a writer re-acquire The t/400 fast-fail finish family's root: the cached-X cover fast path skips the convert queue join, and when the post-content-lock re-verify found the cover stale (a BAST X->S downgrade raced the window), the fallback re-acquired through the LEGACY master begin from the S base. That both bypasses the queue's FIFO/WFG writer arbitration (the original S3 unordered multi-writer defect) and mints the fresh-token S-base reservation shape ("S_NEW") the finish classifier refuses by design -- so every such re-acquire rolled back a durable master grant and surfaced result=1 to the client. The stale-cover fallback for an X request now detaches the ACQUIRING holder it prepared, re-enters the queue through the same writer-prepare gate as a fresh conversion (honoring a frozen-barrier refusal through the existing post-PG_TRY unwind exit), binds a new holder and activates the writer under the retaken content lock. The legacy begin remains only for share-mode refreshes and unmanaged relations. No finish-side result is swallowed; the S_NEW protocol pin from the preceding revert stands. RED (recorded on the parent commit 775517302f, unmodified test): t/394 L4c failed with exactly the production signature -- writer outcome err=[could not finish exact cluster PCM grant reservation: result=1], cover_stale_detected delta=1, reverify_reacquire delta=1. GREEN: t/394 all tests successful with err=[] and the same two deltas; full-suite 182 unit binaries; check-format clean (bufmgr.c is PG-core hand-styled). Follow-up legs registered: queue enqueue/commit counter growth, old-holder detach and new-holder bind assertions on the t/394 leg, each with its own baseline RED. --- src/backend/storage/buffer/bufmgr.c | 72 +++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 13 deletions(-) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index c68705c1ab..e1a1dcf82d 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -8280,20 +8280,66 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) cluster_pcm_note_writer_cover_stale_detected(); pcm_covered = false; LWLockRelease(BufferDescriptorGetContentLock(buf)); - pcm_pending_result = cluster_bufmgr_pcm_begin_grant_reservation_wait( - buf, &pcm_pending_base, &pcm_pending_token); - if (pcm_pending_result != CLUSTER_PCM_OWN_OK) - cluster_pcm_own_report_bump_failure( - buf, pcm_pending_result, pcm_pending_base.generation, - pcm_pending_base.flags, "LockBuffer revalidate reservation"); - pcm_pending_set = true; - pcm_acquired = cluster_pcm_lock_acquire_buffer(buf, pcm_mode); - pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf, NULL); - if (mode == BUFFER_LOCK_SHARE) - LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED); + + /* + * PGRAC (t/400 fast-fail finish family): a stale X cover + * means an ownership round moved this block while we + * raced the content-lock window (a BAST X->S downgrade + * being the production case). A writer re-acquire from + * that state is a NEW writer conversion and MUST go back + * through the convert queue's FIFO/WFG arbitration -- the + * legacy master begin from an S base would both bypass + * the queue (the original S3 unordered multi-writer + * defect) and mint the S_NEW reservation shape the finish + * classifier refuses by design. Detach the ACQUIRING + * holder prepared above first; the queue grant binds a + * fresh one. + */ + if (pcm_mode == PCM_LOCK_MODE_X) + { + if (pcm_x_holder != NULL) + { + cluster_bufmgr_pcm_x_holder_abort_acquiring(pcm_x_holder); + pcm_x_holder = NULL; + } + pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode, + pcm_barrier_refused); + } + if (pcm_barrier_refused != NULL && *pcm_barrier_refused) + { + /* + * The queue gate refused under a frozen barrier; the + * barrier-aware caller owns the unwind. No lock is + * retaken and no holder is bound (the post-PG_TRY + * refusal exit returns before any mirror update). + */ + } else - LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE); - cluster_pcm_note_writer_reverify_reacquire(); + { + if (pcm_x_writer == NULL) + { + /* + * Share-mode refresh, or the convert queue is not + * managing this relation: the legacy master + * acquire remains the ordered path. + */ + pcm_pending_result = cluster_bufmgr_pcm_begin_grant_reservation_wait( + buf, &pcm_pending_base, &pcm_pending_token); + if (pcm_pending_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_own_report_bump_failure( + buf, pcm_pending_result, pcm_pending_base.generation, + pcm_pending_base.flags, "LockBuffer revalidate reservation"); + pcm_pending_set = true; + pcm_acquired = cluster_pcm_lock_acquire_buffer(buf, pcm_mode); + } + pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf, NULL); + if (mode == BUFFER_LOCK_SHARE) + LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED); + else + LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE); + cluster_bufmgr_pcm_x_writer_activate(pcm_x_writer); + cluster_pcm_note_writer_reverify_reacquire(); + } } } cluster_bufmgr_pcm_x_holder_activate(pcm_x_holder); From a1496256bd38bf933e1a713c83eed3eb08443f3b Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 12:57:01 -0700 Subject: [PATCH 41/56] fix(cluster): name the finish target state in the rolled-back grant detail loop11 shows the S_NEW tuple surviving the queue re-entry fix on a SECOND producer chain; the detail line carried the live/base tuples but not whether the failing finish targeted S or X -- the one fact that separates an S->S cover refresh from an S->X upgrade. Cold path only. --- src/backend/storage/buffer/bufmgr.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index e1a1dcf82d..123d8e2b09 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -1108,10 +1108,11 @@ cluster_pcm_own_finish_grant_or_rollback(BufferDesc *buf, const ClusterPcmOwnSna (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not finish exact cluster PCM grant reservation: result=%d", (int)finish_result), - errdetail("master grant was rolled back for buffer %d token=%llu; " + errdetail("master grant was rolled back for buffer %d token=%llu target=%u; " "live state=%u gen=%llu token=%llu flags=%u vs " "base state=%u gen=%llu token=%llu flags=%u", buf != NULL ? buf->buf_id : -1, (unsigned long long)reservation_token, + (unsigned int)new_pcm_state, (unsigned int)live.pcm_state, (unsigned long long)live.generation, (unsigned long long)live.reservation_token, (unsigned int)live.flags, base != NULL ? (unsigned int)base->pcm_state : 0, From 0804d575151b88e56529453f9179239f0f0f4475 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 13:45:49 -0700 Subject: [PATCH 42/56] fix(cluster): log the begin-time ownership context of a non-N legacy reservation A legacy grant reservation that opens from a non-N base is the protocol-violating shape behind the deterministic S_NEW finish refusals: an X convert must ride the convert queue, and an S refresh over a held S must be served by the local cover gate. The finish-side errdetail already names the rolled-back five-tuple and the target mode, but cannot say which acquire entry path minted the reservation. Log the complete begin-time context (site, acquire mode, buffer id, tag, BM_VALID, and the exact ownership tuple the reservation snapshotted) at both legacy begin call sites, gated on the non-N base shape so the hot path stays silent. Zero behavior change. --- src/backend/storage/buffer/bufmgr.c | 36 +++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 123d8e2b09..a6d9bffd76 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -1316,6 +1316,36 @@ cluster_bufmgr_pcm_begin_grant_reservation_wait(BufferDesc *buf, } } +/* + * PGRAC (t/400 S_NEW forensics): a legacy grant reservation that opens from + * a non-N base is the protocol-violating shape behind the deterministic + * S_NEW finish refusals — an X convert must ride the convert queue, and an + * S refresh over a held S must be served by the local cover gate. Log the + * complete begin-time context (buffer identity, validity bit, the exact + * ownership tuple the reservation snapshotted, and the acquire mode) so the + * producing entry path can be identified from a live run. + */ +static void +cluster_bufmgr_pcm_legacy_begin_probe(BufferDesc *buf, PcmLockMode pcm_mode, + const ClusterPcmOwnSnapshot *base, + const char *site) +{ + uint32 buf_state; + + if (base->pcm_state == (uint8) PCM_STATE_N) + return; + buf_state = pg_atomic_read_u32(&buf->state); + elog(LOG, + "cluster PCM legacy reservation opened from a non-N base: site=%s mode=%d buffer=%d rel=%u fork=%d blk=%u valid=%d base_state=%u base_gen=%llu base_token=%llu base_flags=0x%x pcm_state_now=%u", + site, (int) pcm_mode, buf->buf_id, + buf->tag.relNumber, (int) buf->tag.forkNum, buf->tag.blockNum, + (buf_state & BM_VALID) != 0 ? 1 : 0, + base->pcm_state, + (unsigned long long) base->generation, + (unsigned long long) base->reservation_token, + base->flags, buf->pcm_state); +} + static ClusterPcmXHolderLedgerEntry * cluster_bufmgr_pcm_x_holder_find(BufferDesc *buf) { @@ -8160,6 +8190,9 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) cluster_pcm_own_report_bump_failure( buf, pcm_pending_result, pcm_pending_base.generation, pcm_pending_base.flags, "LockBuffer master reservation"); + cluster_bufmgr_pcm_legacy_begin_probe(buf, pcm_mode, + &pcm_pending_base, + "master-reservation"); pcm_pending_set = true; /* @@ -8330,6 +8363,9 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) cluster_pcm_own_report_bump_failure( buf, pcm_pending_result, pcm_pending_base.generation, pcm_pending_base.flags, "LockBuffer revalidate reservation"); + cluster_bufmgr_pcm_legacy_begin_probe( + buf, pcm_mode, &pcm_pending_base, + "revalidate-reservation"); pcm_pending_set = true; pcm_acquired = cluster_pcm_lock_acquire_buffer(buf, pcm_mode); } From c3b19981d9274a537deb7836464d8258fb83a076 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 19:01:05 -0700 Subject: [PATCH 43/56] fix(cluster): keep the pinned retained PI mirror instead of clearing BM_VALID under a pin t/400 loop14 root cause (P0-15): cluster_bufmgr_pcm_own_release_retained_image unconditionally cleared BM_VALID and retagged CURRENT when the exact retained write-fence token was released, even while pre-existing PG pins were live. A pin holder that then went straight to LockBuffer (never back through ReadBuffer) ran the PCM acquire ladder over invalid bytes: the legacy master reservation opened from the invalid base, the finish stamped an S mirror onto the invalid descriptor, and the next X convert was bounced from the convert queue (writer-prepare BM_VALID gate) into a legacy S-base begin whose S_NEW finish shape is refused by design -- the deterministic "could not finish exact cluster PCM grant reservation" client errors. Fix, upstream of the master reservation and converged with the proven passive-pin invalidate release shape (cluster_bufmgr_pcm_own_release_pinned_s_for_gcs): - new shared pin-aware retag helper cluster_pcm_x_retained_release_retag: refcount==0 drops the image (!BM_VALID + CURRENT, next ordinary read reloads); refcount>0 keeps the frozen PI+BM_VALID never-write/never-serve N mirror byte-exact, so the next S acquire installs over it via an exact GRANT_PENDING (content_write_permitted already admits exactly that), an X convert rides the convert queue, and eviction retags after the last pin drains. No image swap and no vanishing bytes under a pin holder. - grant finish fails closed (CORRUPT) instead of committing a durable S/X mirror over !BM_VALID or a PI mirror (Rule 8.A: never silently cover stale or absent bytes); S_NEW/X-base stay refused, no queue bypass. - release evidence LOG names dropped vs kept-pinned-pi; the direct own-transition mirror keeps its S/X-over-!BM_VALID evidence LOG. RED (recorded on parent + this tree pre-fix): missing-symbol build failure of test_retained_release_retag_respects_pin_contract (real BufferDesc, both refcount arms, byte-exact pinned assertion) and absent scan needles in test_retained_release_and_finish_never_cover_invalid_bytes (retag routed under the same header-lock hold; finish valid-image gate; reload-in-place helper pinned as forbidden). Stale drain-retag scan expectation updated to the pin-aware contract. GREEN: 49/49 + 269/269, 182 unit binaries, full build 0 errors, check-format 0. --- src/backend/storage/buffer/bufmgr.c | 65 +++++++++++++-- src/include/cluster/cluster_pcm_x_bufmgr.h | 22 +++++ src/test/cluster_unit/test_cluster_pcm_own.c | 85 ++++++++++++++++++-- 3 files changed, 157 insertions(+), 15 deletions(-) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index a6d9bffd76..a9172df99b 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -225,6 +225,14 @@ cluster_pcm_own_transition(BufferDesc *buf, uint8 new_pcm_state, uint32 set_flag uint32 buf_state; uint32 observed_flags = 0; uint64 new_generation; + /* Evidence for an S/X mirror transition over invalid page bytes: sampled + * under the header lock, logged after. The reservation finish itself + * fails closed on this shape; this covers the direct-transition mirror. */ + bool fx_log = false; + BufferTag fx_tag = {0}; + uint8 fx_old_state = 0; + uint64 fx_token = 0; + uint32 fx_refcount = 0; buf_state = LockBufHdr(buf); bump_result = cluster_pcm_own_bump_locked(buf, set_flags, clear_flags, @@ -234,8 +242,23 @@ cluster_pcm_own_transition(BufferDesc *buf, uint8 new_pcm_state, uint32 set_flag cluster_pcm_own_report_bump_failure(buf, bump_result, new_generation, observed_flags, "ownership transition"); } + if ((new_pcm_state == (uint8) PCM_STATE_S || new_pcm_state == (uint8) PCM_STATE_X) + && (buf_state & BM_VALID) == 0) + { + fx_log = true; + fx_tag = buf->tag; + fx_old_state = buf->pcm_state; + fx_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + fx_refcount = BUF_STATE_GET_REFCOUNT(buf_state); + } buf->pcm_state = new_pcm_state; UnlockBufHdr(buf, buf_state); + if (fx_log) + elog(LOG, + "cluster PCM own S/X over !BM_VALID: site=own-transition buffer=%d rel=%u fork=%d blk=%u target=%u refcount=%u old_state=%u gen=%llu token=%llu", + buf->buf_id, fx_tag.relNumber, (int) fx_tag.forkNum, fx_tag.blockNum, + (unsigned) new_pcm_state, fx_refcount, (unsigned) fx_old_state, + (unsigned long long) new_generation, (unsigned long long) fx_token); } /* @@ -859,6 +882,14 @@ cluster_pcm_own_finish_grant_reservation(BufferDesc *buf, const ClusterPcmOwnSna && (new_pcm_state != (uint8)PCM_STATE_X || !LWLockHeldByMe(BufferDescriptorGetContentLock(buf)))) result = CLUSTER_PCM_OWN_INVALID; + else if ((buf_state & BM_VALID) == 0 || buf->buffer_type == (uint8) BUF_TYPE_PI) + /* Never commit a durable S/X mirror over a page image that is not + * current: a dropped image (!BM_VALID) or a frozen PI mirror carries + * no bytes this grant may serve, so a commit here would silently + * cover stale or absent data (Rule 8.A). A grant that installs the + * shipped image sets BM_VALID + CURRENT before this finish; the + * !BM_VALID revoke-handoff precedent likewise fails closed. */ + result = CLUSTER_PCM_OWN_CORRUPT; else { result = cluster_pcm_own_grant_commit_exact(buf->buf_id, base->generation, reservation_token, out_committed_generation); @@ -12015,6 +12046,10 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, uint32 flags; uint32 buf_state; int buf_id; + /* Release evidence: sampled under the header lock, logged after. */ + bool fx_log = false; + bool fx_dropped = false; + uint32 fx_refcount = 0; if (tag == NULL || source_generation == UINT64_MAX) return CLUSTER_PCM_OWN_INVALID; @@ -12097,21 +12132,35 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, buf->buf_id, committed_generation, retained_token); if (result == CLUSTER_PCM_OWN_OK) { /* - * The exact token is the last stale-write fence. Release it - * before making the retained PI reloadable, while content - * EXCLUSIVE and the header lock keep page bytes invalid. - * Keep the mapping so any pre-existing pin can leave - * normally; the next ordinary read reloads the current page - * image. + * The exact token was the last stale-write fence; release it + * first, then retag under the same header-lock hold. With no + * pins the image is dropped so the next ordinary read reloads + * the current page bytes. With a pre-existing pin the frozen + * PI+BM_VALID N mirror is kept byte-exact instead: clearing + * BM_VALID under a pin holder would let a later LockBuffer + * open a legacy master reservation over an invalid base and + * stamp S/X ownership onto it (the deterministic S_NEW finish + * refusals), while reloading in place would swap the page + * image under the pin (both violate the pin contract). See + * cluster_pcm_x_retained_release_retag. */ - buf_state &= ~BM_VALID; - buf->buffer_type = (uint8)BUF_TYPE_CURRENT; + fx_dropped = cluster_pcm_x_retained_release_retag(&buf->buffer_type, + &buf_state); + fx_log = true; + fx_refcount = BUF_STATE_GET_REFCOUNT(buf_state); } } } UnlockBufHdr(buf, buf_state); LWLockRelease(content_lock); + if (fx_log) + elog(LOG, + "cluster PCM retained image released: image=%s buffer=%d rel=%u fork=%d blk=%u refcount=%u gen=%llu token=%llu", + fx_dropped ? "dropped" : "kept-pinned-pi", buf->buf_id, + tag->relNumber, (int) tag->forkNum, tag->blockNum, + fx_refcount, (unsigned long long) committed_generation, + (unsigned long long) retained_token); return result; } diff --git a/src/include/cluster/cluster_pcm_x_bufmgr.h b/src/include/cluster/cluster_pcm_x_bufmgr.h index 6d0b013199..d54f447706 100644 --- a/src/include/cluster/cluster_pcm_x_bufmgr.h +++ b/src/include/cluster/cluster_pcm_x_bufmgr.h @@ -271,6 +271,28 @@ cluster_pcm_x_current_image_shape(uint8 pcm_state, uint8 buffer_type, bool valid || (pcm_state == (uint8)PCM_STATE_X && buffer_type == (uint8)BUF_TYPE_XCUR)); } +/* Post-release retag of a retained PI whose exact write-fence token was just + * released, applied under the same header-lock hold. With no pins the image + * is dropped -- !BM_VALID plus a BUF_TYPE_CURRENT retag makes the next + * ordinary read reload the current page bytes. With a pre-existing pin the + * frozen bytes must neither vanish nor change under the pin holder + * (PostgreSQL pin contract): keep the established PI+BM_VALID + * never-write/never-serve N mirror (the passive-pin invalidate release + * shape). The next S acquire begins an exact GRANT_PENDING install that may + * overwrite it and republish CURRENT, an X convert rides the convert queue, + * and eviction retags once the last pin drains. Returns true when the image + * was dropped. */ +static inline bool +cluster_pcm_x_retained_release_retag(uint8 *buffer_type, uint32 *buf_state) +{ + if (BUF_STATE_GET_REFCOUNT(*buf_state) == 0) { + *buf_state &= ~BM_VALID; + *buffer_type = (uint8)BUF_TYPE_CURRENT; + return true; + } + return false; +} + /* Token zero is valid before the first reservation; a completed reservation * leaves a nonzero monotonic token idle. In both cases flags alone say * whether a lifecycle is currently active. */ diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index ad5048729f..1810b204ef 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -874,6 +874,78 @@ UT_TEST(test_bufmgr_finish_and_abort_gate_on_exact_base_state) free(source); } +UT_TEST(test_retained_release_retag_respects_pin_contract) +{ + BufferDesc buf; + uint32 buf_state; + uint8 type_before; + uint32 state_before; + + /* Unpinned: the released retained image is dropped -- !BM_VALID plus a + * BUF_TYPE_CURRENT retag makes the next ordinary read reload the current + * page bytes through the buffer-IO protocol. */ + memset(&buf, 0, sizeof(buf)); + buf.buffer_type = (uint8)BUF_TYPE_PI; + buf_state = BM_VALID | BM_TAG_VALID; + UT_ASSERT(cluster_pcm_x_retained_release_retag(&buf.buffer_type, &buf_state)); + UT_ASSERT_EQ(buf_state & BM_VALID, 0); + UT_ASSERT_EQ(buf.buffer_type, (uint8)BUF_TYPE_CURRENT); + + /* Pinned: a pre-existing PG pin freezes the page image -- the bytes may + * neither vanish (!BM_VALID under a pin breaks the pin contract and + * re-arms the legacy begin-over-invalid-base S_NEW mint) nor be reloaded + * in place (an image swap under the pin holder). The release must keep + * the established PI+BM_VALID never-write/never-serve N mirror byte-exact + * (the passive-pin invalidate release shape): the next S acquire installs + * over it via an exact GRANT_PENDING and republishes CURRENT, an X + * convert rides the convert queue, and eviction retags after the last + * pin drains. */ + memset(&buf, 0, sizeof(buf)); + buf.buffer_type = (uint8)BUF_TYPE_PI; + buf_state = (BM_VALID | BM_TAG_VALID) + BUF_REFCOUNT_ONE * 2; + type_before = buf.buffer_type; + state_before = buf_state; + UT_ASSERT(!cluster_pcm_x_retained_release_retag(&buf.buffer_type, &buf_state)); + UT_ASSERT_EQ(buf.buffer_type, type_before); + UT_ASSERT_EQ(buf_state, state_before); + + /* One pin behaves like many. */ + buf.buffer_type = (uint8)BUF_TYPE_PI; + buf_state = (BM_VALID | BM_TAG_VALID) + BUF_REFCOUNT_ONE; + UT_ASSERT(!cluster_pcm_x_retained_release_retag(&buf.buffer_type, &buf_state)); + UT_ASSERT(buf_state & BM_VALID); + UT_ASSERT_EQ(buf.buffer_type, (uint8)BUF_TYPE_PI); +} + +UT_TEST(test_retained_release_and_finish_never_cover_invalid_bytes) +{ + static const char *const release_retag_contract[] + = { "cluster_pcm_own_revoke_retain_release_exact", "cluster_pcm_x_retained_release_retag", + "UnlockBufHdr" }; + static const char *const finish_valid_gate[] + = { "cluster_pcm_x_grant_reservation_kind", "(buf_state & BM_VALID) == 0", "BUF_TYPE_PI", + "CLUSTER_PCM_OWN_CORRUPT", "cluster_pcm_own_grant_commit_exact" }; + char *source = read_bufmgr_source(); + + /* The retained release must route its descriptor retag through the shared + * pin-aware decision helper under the same header-lock hold that released + * the exact write-fence token. And a grant finish must never commit a + * durable S/X mirror over a page image that is not current (!BM_VALID or + * a PI mirror): that silent cover of stale bytes is how the pinned + * descriptor was previously stamped S over an invalid base, re-arming the + * refused legacy S_NEW convert (deterministic client ERROR) -- and, on + * the PI arm, a Rule 8.A stale read. Reloading the bytes in place under + * a foreign pin is equally forbidden. */ + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_release_retained_image(", + "\ncluster_bufmgr_pcm_own_self_handoff_probe(", + release_retag_contract, lengthof(release_retag_contract)); + assert_ordered_in_function(source, "\ncluster_pcm_own_finish_grant_reservation(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_finish_x_commit(", + finish_valid_gate, lengthof(finish_valid_gate)); + UT_ASSERT_NULL(strstr(source, "cluster_bufmgr_pcm_reload_invalid_pinned")); + free(source); +} + UT_TEST(test_d5a_release_error_keeps_descriptor_out_of_freelist) { static const char *const fail_closed_contract[] @@ -1298,12 +1370,9 @@ UT_TEST(test_retained_image_release_and_writeback_gates_are_exact) UT_TEST(test_retained_drain_retags_invalid_only_after_exact_token_release) { - static const char *const drain_contract[] = { "cluster_pcm_own_revoke_retain_release_exact", - "result == CLUSTER_PCM_OWN_OK", - "buf_state &= ~BM_VALID", - "buf->buffer_type", - "BUF_TYPE_CURRENT", - "UnlockBufHdr" }; + static const char *const drain_contract[] + = { "cluster_pcm_own_revoke_retain_release_exact", "result == CLUSTER_PCM_OWN_OK", + "cluster_pcm_x_retained_release_retag", "&buf->buffer_type", "UnlockBufHdr" }; char *source = read_bufmgr_source(); const char *release; const char *release_end; @@ -2135,7 +2204,7 @@ UT_TEST(test_preflight_busy_waits_then_clean_resnapshot_begins_reservation) int main(void) { - UT_PLAN(47); + UT_PLAN(49); UT_RUN(test_shmem_initializes_complete_entry); UT_RUN(test_begin_abort_is_exact_and_monotonic); UT_RUN(test_invalid_live_flag_shapes_are_corrupt_not_busy); @@ -2143,6 +2212,8 @@ main(void) UT_RUN(test_s_revoke_handoff_reuses_exact_token_and_bumps_once); UT_RUN(test_revoke_handoff_kinds_cover_n_s_x_with_one_lifecycle); UT_RUN(test_s_new_fresh_token_finish_shape_stays_invalid); + UT_RUN(test_retained_release_retag_respects_pin_contract); + UT_RUN(test_retained_release_and_finish_never_cover_invalid_bytes); UT_RUN(test_revoke_commit_is_exact_and_classifies_live_races); UT_RUN(test_revoke_retain_commit_keeps_exact_token_until_release); UT_RUN(test_revoke_commit_exhaustion_is_side_effect_free); From 0d70fa76ceefdf1a53235776e6faae2aed1ed1c4 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 19:14:39 -0700 Subject: [PATCH 44/56] fix(cluster): admit the direct-init IO window in the finish valid-image gate t/400 loop15 caught a regression in the new finish gate: the EXTEND / READ_MISS direct-init path legitimately commits its X grant between StartBufferIO and TerminateBufferIO(BM_VALID) -- the descriptor still owns BM_IO_IN_PROGRESS and the zeroed page is published only after the caller initializes it under content EXCLUSIVE (first INSERT into a fresh table failed with finish result=CORRUPT at warmup). Narrow the refusal to a PI mirror or a dropped image with no IO in progress; the loop14 poison shape (free-floating !BM_VALID descriptor after a retained release) stays refused. GREEN: 49/49 pcm_own, full build 0 errors, check-format 0. --- src/backend/storage/buffer/bufmgr.c | 12 +++++++++--- src/test/cluster_unit/test_cluster_pcm_own.c | 12 ++++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index a9172df99b..ce2b6da67d 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -882,13 +882,19 @@ cluster_pcm_own_finish_grant_reservation(BufferDesc *buf, const ClusterPcmOwnSna && (new_pcm_state != (uint8)PCM_STATE_X || !LWLockHeldByMe(BufferDescriptorGetContentLock(buf)))) result = CLUSTER_PCM_OWN_INVALID; - else if ((buf_state & BM_VALID) == 0 || buf->buffer_type == (uint8) BUF_TYPE_PI) + else if (buf->buffer_type == (uint8) BUF_TYPE_PI + || (buf_state & (BM_VALID | BM_IO_IN_PROGRESS)) == 0) /* Never commit a durable S/X mirror over a page image that is not - * current: a dropped image (!BM_VALID) or a frozen PI mirror carries + * current: a frozen PI mirror or a dropped image (!BM_VALID) carries * no bytes this grant may serve, so a commit here would silently * cover stale or absent data (Rule 8.A). A grant that installs the * shipped image sets BM_VALID + CURRENT before this finish; the - * !BM_VALID revoke-handoff precedent likewise fails closed. */ + * !BM_VALID revoke-handoff precedent likewise fails closed. The one + * legal !BM_VALID commit shape is the direct-init window (EXTEND / + * READ_MISS): its gate runs between StartBufferIO and + * TerminateBufferIO(BM_VALID), so the descriptor still owns + * BM_IO_IN_PROGRESS and the zeroed page is published only after the + * caller initializes it under content EXCLUSIVE. */ result = CLUSTER_PCM_OWN_CORRUPT; else { result = cluster_pcm_own_grant_commit_exact(buf->buf_id, base->generation, diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index 1810b204ef..7136b4f2ef 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -923,8 +923,9 @@ UT_TEST(test_retained_release_and_finish_never_cover_invalid_bytes) = { "cluster_pcm_own_revoke_retain_release_exact", "cluster_pcm_x_retained_release_retag", "UnlockBufHdr" }; static const char *const finish_valid_gate[] - = { "cluster_pcm_x_grant_reservation_kind", "(buf_state & BM_VALID) == 0", "BUF_TYPE_PI", - "CLUSTER_PCM_OWN_CORRUPT", "cluster_pcm_own_grant_commit_exact" }; + = { "cluster_pcm_x_grant_reservation_kind", "BUF_TYPE_PI", + "(buf_state & (BM_VALID | BM_IO_IN_PROGRESS)) == 0", "CLUSTER_PCM_OWN_CORRUPT", + "cluster_pcm_own_grant_commit_exact" }; char *source = read_bufmgr_source(); /* The retained release must route its descriptor retag through the shared @@ -934,8 +935,11 @@ UT_TEST(test_retained_release_and_finish_never_cover_invalid_bytes) * a PI mirror): that silent cover of stale bytes is how the pinned * descriptor was previously stamped S over an invalid base, re-arming the * refused legacy S_NEW convert (deterministic client ERROR) -- and, on - * the PI arm, a Rule 8.A stale read. Reloading the bytes in place under - * a foreign pin is equally forbidden. */ + * the PI arm, a Rule 8.A stale read. The one legal !BM_VALID commit + * shape is the direct-init window (EXTEND/READ_MISS), whose gate still + * owns BM_IO_IN_PROGRESS between StartBufferIO and + * TerminateBufferIO(BM_VALID). Reloading the bytes in place under a + * foreign pin is equally forbidden. */ assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_release_retained_image(", "\ncluster_bufmgr_pcm_own_self_handoff_probe(", release_retag_contract, lengthof(release_retag_contract)); From e4e2b980a8b5189d5c715097896ec8c755452376 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 19:36:40 -0700 Subject: [PATCH 45/56] fix(cluster): republish a kept-pinned PI mirror at each legacy byte-currency proof The retained release now keeps a frozen PI+BM_VALID N mirror for pre-existing pins, and the grant finish refuses to commit S/X over a PI mirror. The legacy grant lane, however, historically republished CURRENT only implicitly through the finish's SCUR/XCUR stamp -- so a legitimate re-acquire over the kept mirror (t/400 loop16: hot-page S finishes refused with CORRUPT at gen=4) was failing closed even after its bytes were proven current. Give the legacy lane an explicit republish at each byte-currency proof point, mirroring the queue lane's publish_installed_x_image: - new header predicate cluster_pcm_x_grant_pending_republish_shape: only a pcm N + live GRANT_PENDING + nonzero-token + BM_VALID + PI descriptor may flip, and only to CURRENT; - new bufmgr helper cluster_bufmgr_pcm_own_republish_grant_pending_image applies it under header authority; - call sites: the shipped-image install memcpy (same content-EXCLUSIVE hold), the storage-fallback direct SCN PASS proof, and the refresh-then-PASS proof. The SKIP arms leave PI frozen, so the finish valid-image gate keeps failing closed on unproven bytes. RED (recorded pre-fix): missing-symbol on the predicate truth-table legs in test_retained_release_retag_respects_pin_contract, and scan-needle failure of test_legacy_byte_proof_sites_republish_kept_pi_mirror (81st, gcs_block). GREEN: 50/50 + 81/81 + 269/269, full build 0 errors, check-format 0. --- src/backend/cluster/cluster_gcs_block.c | 14 ++++- src/backend/storage/buffer/bufmgr.c | 24 ++++++++ src/include/cluster/cluster_pcm_x_bufmgr.h | 15 +++++ .../cluster_unit/test_cluster_gcs_block.c | 56 ++++++++++++++++++- src/test/cluster_unit/test_cluster_pcm_own.c | 40 ++++++++++++- 5 files changed, 146 insertions(+), 3 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index e117bc1986..3317612fe2 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -1335,6 +1335,10 @@ gcs_block_install_block(BufferDesc *buf, const char *block_data, XLogRecPtr page memcpy(page, block_data, GCS_BLOCK_DATA_SIZE); gcs_block_note_install_copy(); PageSetLSN(page, page_lsn); + /* The shipped image just proved these bytes current: a kept-pinned + * retained PI mirror regains CURRENT inside the same content-EXCLUSIVE + * hold, or the grant finish would refuse the frozen PI shape. */ + cluster_bufmgr_pcm_own_republish_grant_pending_image(buf); LWLockRelease(content_lock); } @@ -1529,6 +1533,10 @@ cluster_gcs_block_fallback_verify_refresh(BufferDesc *buf, BufferTag tag, SCN ex verdict = gcs_block_lost_write_verdict(expected_scn, page_scn); if (verdict == GCS_LOST_WRITE_PASS) { pg_atomic_fetch_add_u64(&ClusterGcsBlock->fallback_scn_verify_pass_count, 1); + /* SCN proof: the local bytes (possibly a kept-pinned retained PI + * mirror) are at least the master watermark — republish CURRENT so + * the grant finish can commit over them. */ + cluster_bufmgr_pcm_own_republish_grant_pending_image(buf); return; } @@ -1545,8 +1553,12 @@ cluster_gcs_block_fallback_verify_refresh(BufferDesc *buf, BufferTag tag, SCN ex page_scn = InvalidScn; verdict = gcs_block_lost_write_verdict(expected_scn, page_scn); - if (verdict == GCS_LOST_WRITE_PASS) + if (verdict == GCS_LOST_WRITE_PASS) { + /* Refreshed from shared storage and proven: same republish as + * the direct PASS proof above. */ + cluster_bufmgr_pcm_own_republish_grant_pending_image(buf); return; + } } /* Refresh refused (dirty local copy) or the storage page is itself diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index ce2b6da67d..783bccbea5 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -711,6 +711,30 @@ cluster_bufmgr_pcm_own_publish_installed_x_image( return result; } +/* Republish a kept-pinned retained PI mirror as the CURRENT image. The + * caller stands at a byte-currency proof point of the still-open legacy + * grant (a shipped-image install under content EXCLUSIVE, a storage refresh, + * or an SCN PASS proof); only the exact republish shape is flipped, under + * header authority, and every other descriptor state is left untouched so + * the grant-finish valid-image gate keeps failing closed on unproven + * bytes. */ +void +cluster_bufmgr_pcm_own_republish_grant_pending_image(BufferDesc *buf) +{ + uint32 buf_state; + + if (buf == NULL || ClusterPcmOwnArray == NULL) + return; + buf_state = LockBufHdr(buf); + if (cluster_pcm_x_grant_pending_republish_shape(buf->pcm_state, + cluster_pcm_own_flags_get(buf->buf_id), + cluster_pcm_own_reservation_token_get(buf->buf_id), + (buf_state & BM_VALID) != 0, + buf->buffer_type)) + buf->buffer_type = (uint8) BUF_TYPE_CURRENT; + UnlockBufHdr(buf, buf_state); +} + static ClusterPcmOwnResult cluster_pcm_own_begin_grant_reservation(BufferDesc *buf, ClusterPcmOwnSnapshot *out_base, uint64 *out_token) diff --git a/src/include/cluster/cluster_pcm_x_bufmgr.h b/src/include/cluster/cluster_pcm_x_bufmgr.h index d54f447706..3f8cdbdc44 100644 --- a/src/include/cluster/cluster_pcm_x_bufmgr.h +++ b/src/include/cluster/cluster_pcm_x_bufmgr.h @@ -293,6 +293,20 @@ cluster_pcm_x_retained_release_retag(uint8 *buffer_type, uint32 *buf_state) return false; } +/* A frozen PI mirror kept for pre-existing pins regains CURRENT only while + * the exact legacy grant lifecycle that just proved its bytes (a shipped + * install, a storage refresh, or an SCN PASS proof) is still open: pcm N + + * live GRANT_PENDING + nonzero token + BM_VALID + PI. Every other shape + * stays frozen so the grant-finish valid-image gate keeps failing closed on + * an unproven stale cover. */ +static inline bool +cluster_pcm_x_grant_pending_republish_shape(uint8 pcm_state, uint32 flags, uint64 token, bool valid, + uint8 buffer_type) +{ + return pcm_state == (uint8)PCM_STATE_N && flags == PCM_OWN_FLAG_GRANT_PENDING && token != 0 + && valid && buffer_type == (uint8)BUF_TYPE_PI; +} + /* Token zero is valid before the first reservation; a completed reservation * leaves a nonzero monotonic token idle. In both cases flags alone say * whether a lifecycle is currently active. */ @@ -315,6 +329,7 @@ cluster_bufmgr_pcm_own_snapshot_by_tag(const BufferTag *tag, int *out_buffer_id, * LockBuffer/W1. GRANT_PENDING image installation is permitted; a live * source REVOKING lifecycle or retained PI+VALID descriptor is not. */ extern bool cluster_bufmgr_pcm_x_content_write_permitted(BufferDesc *buf); +extern void cluster_bufmgr_pcm_own_republish_grant_pending_image(BufferDesc *buf); /* Called only after the requester has proved the exact remote master's S->N * RELEASE application ACK. Atomically normalizes the matching descriptor * tuple and returns the fresh N snapshot used by the later PREPARE leg. */ diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index c3895fa442..ecb26b2be4 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -130,6 +130,34 @@ read_gcs_block_source(void) } +static void +assert_ordered_in_function(const char *source, const char *function_start, const char *function_end, + const char *const *needles, int needle_count) +{ + const char *cursor = strstr(source, function_start); + const char *end; + int i; + + UT_ASSERT_NOT_NULL(cursor); + if (cursor == NULL) + return; + end = strstr(cursor + strlen(function_start), function_end); + UT_ASSERT_NOT_NULL(end); + if (end == NULL) + return; + + for (i = 0; i < needle_count; i++) { + cursor = strstr(cursor, needles[i]); + UT_ASSERT_NOT_NULL(cursor); + if (cursor == NULL) + return; + UT_ASSERT(cursor < end); + if (cursor >= end) + return; + cursor += strlen(needles[i]); + } +} + static int count_occurrences(const char *source, const char *needle) { @@ -3584,11 +3612,36 @@ UT_TEST(test_pcm_x_role_refresh_accepts_only_same_member_promotion) UT_ASSERT(!cluster_gcs_pcm_x_role_refresh_exact(&follower, &promoted)); } +UT_TEST(test_legacy_byte_proof_sites_republish_kept_pi_mirror) +{ + static const char *const install_contract[] + = { "cluster_bufmgr_pcm_x_content_write_permitted", "memcpy", "PageSetLSN", + "cluster_bufmgr_pcm_own_republish_grant_pending_image", "LWLockRelease" }; + static const char *const fallback_contract[] + = { "GCS_LOST_WRITE_PASS", "cluster_bufmgr_pcm_own_republish_grant_pending_image", + "cluster_bufmgr_refresh_block_from_storage_for_gcs", "GCS_LOST_WRITE_PASS", + "cluster_bufmgr_pcm_own_republish_grant_pending_image" }; + char *source = read_gcs_block_source(); + + /* A retained-image release now keeps a frozen PI+BM_VALID N mirror for + * pre-existing pins. Its bytes regain CURRENT only where a legacy grant + * just proved them: the shipped-image install memcpy (still under the + * same content EXCLUSIVE hold) and both storage-fallback PASS proofs + * (direct SCN PASS, and refresh-then-PASS). The SKIP arms must leave PI + * frozen so the finish valid-image gate keeps failing closed on unproven + * bytes. */ + assert_ordered_in_function(source, "\ngcs_block_install_block(", "\nstatic ", install_contract, + lengthof(install_contract)); + assert_ordered_in_function(source, "\ncluster_gcs_block_fallback_verify_refresh(", "\nstatic ", + fallback_contract, lengthof(fallback_contract)); + free(source); +} + int main(void) { - UT_PLAN(80); + UT_PLAN(81); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -3669,6 +3722,7 @@ main(void) UT_RUN(test_pcm_x_retire_commit_wakes_exact_waiters_before_ack_or_resolve); UT_RUN(test_pcm_x_tagless_retire_uses_explicit_data_plane_handoff); UT_RUN(test_pcm_x_role_refresh_accepts_only_same_member_promotion); + UT_RUN(test_legacy_byte_proof_sites_republish_kept_pi_mirror); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index 7136b4f2ef..04c5b05dd8 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -915,6 +915,26 @@ UT_TEST(test_retained_release_retag_respects_pin_contract) UT_ASSERT(!cluster_pcm_x_retained_release_retag(&buf.buffer_type, &buf_state)); UT_ASSERT(buf_state & BM_VALID); UT_ASSERT_EQ(buf.buffer_type, (uint8)BUF_TYPE_PI); + + /* The kept mirror is republished CURRENT only by a byte-currency proof + * inside the exact open legacy grant lifecycle (a shipped-image install, + * a storage refresh, or an SCN PASS proof): pcm N + live GRANT_PENDING + + * nonzero token + BM_VALID + PI. Anything else must stay frozen so the + * finish valid-image gate keeps refusing an unproven stale cover. */ + UT_ASSERT(cluster_pcm_x_grant_pending_republish_shape( + (uint8)PCM_STATE_N, PCM_OWN_FLAG_GRANT_PENDING, 7, true, (uint8)BUF_TYPE_PI)); + UT_ASSERT(!cluster_pcm_x_grant_pending_republish_shape( + (uint8)PCM_STATE_S, PCM_OWN_FLAG_GRANT_PENDING, 7, true, (uint8)BUF_TYPE_PI)); + UT_ASSERT(!cluster_pcm_x_grant_pending_republish_shape((uint8)PCM_STATE_N, 0, 7, true, + (uint8)BUF_TYPE_PI)); + UT_ASSERT(!cluster_pcm_x_grant_pending_republish_shape( + (uint8)PCM_STATE_N, PCM_OWN_FLAG_REVOKING, 7, true, (uint8)BUF_TYPE_PI)); + UT_ASSERT(!cluster_pcm_x_grant_pending_republish_shape( + (uint8)PCM_STATE_N, PCM_OWN_FLAG_GRANT_PENDING, 0, true, (uint8)BUF_TYPE_PI)); + UT_ASSERT(!cluster_pcm_x_grant_pending_republish_shape( + (uint8)PCM_STATE_N, PCM_OWN_FLAG_GRANT_PENDING, 7, false, (uint8)BUF_TYPE_PI)); + UT_ASSERT(!cluster_pcm_x_grant_pending_republish_shape( + (uint8)PCM_STATE_N, PCM_OWN_FLAG_GRANT_PENDING, 7, true, (uint8)BUF_TYPE_CURRENT)); } UT_TEST(test_retained_release_and_finish_never_cover_invalid_bytes) @@ -950,6 +970,23 @@ UT_TEST(test_retained_release_and_finish_never_cover_invalid_bytes) free(source); } +UT_TEST(test_legacy_byte_proof_republishes_kept_pi_mirror) +{ + static const char *const republish_contract[] + = { "cluster_pcm_x_grant_pending_republish_shape", "BUF_TYPE_CURRENT", "UnlockBufHdr" }; + char *bufmgr_source = read_bufmgr_source(); + + /* A kept-pinned PI mirror regains CURRENT only where its bytes were just + * proven current inside the still-open legacy grant lifecycle (the + * gcs_block install / storage-fallback call sites are pinned in + * test_cluster_gcs_block). The helper flips only the exact republish + * shape under header authority. */ + assert_ordered_in_function( + bufmgr_source, "\ncluster_bufmgr_pcm_own_republish_grant_pending_image(", + "\nstatic ClusterPcmOwnResult", republish_contract, lengthof(republish_contract)); + free(bufmgr_source); +} + UT_TEST(test_d5a_release_error_keeps_descriptor_out_of_freelist) { static const char *const fail_closed_contract[] @@ -2208,7 +2245,7 @@ UT_TEST(test_preflight_busy_waits_then_clean_resnapshot_begins_reservation) int main(void) { - UT_PLAN(49); + UT_PLAN(50); UT_RUN(test_shmem_initializes_complete_entry); UT_RUN(test_begin_abort_is_exact_and_monotonic); UT_RUN(test_invalid_live_flag_shapes_are_corrupt_not_busy); @@ -2218,6 +2255,7 @@ main(void) UT_RUN(test_s_new_fresh_token_finish_shape_stays_invalid); UT_RUN(test_retained_release_retag_respects_pin_contract); UT_RUN(test_retained_release_and_finish_never_cover_invalid_bytes); + UT_RUN(test_legacy_byte_proof_republishes_kept_pi_mirror); UT_RUN(test_revoke_commit_is_exact_and_classifies_live_races); UT_RUN(test_revoke_retain_commit_keeps_exact_token_until_release); UT_RUN(test_revoke_commit_exhaustion_is_side_effect_free); From 3d058d612380c0422849142bb61b03eeab4ae3ce Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 20:27:03 -0700 Subject: [PATCH 46/56] fix(cluster): name every silent refusal arm of the source-side REVOKE handler t/400 loop17 (post P0-15): the client-error families are gone, but a form-B wedge remains -- the master re-sent REVOKE 29 times to the source node with zero refusal evidence in its log, because the streak note covered only the ingress-snapshot arm while the handler refused at one of five other silent returns. Route every refusal arm through the log-once streak note (ingress auth, holder-ledger stale, holder-progress error, image reserve, and the apply result), and keep the dedup-slot release on a failed apply inside the same refusal branch. Observability only; no protocol change. GREEN: 82/82 gcs_block (new arm-coverage scan 82nd), full build 0 errors, check-format 0. --- src/backend/cluster/cluster_gcs_block.c | 24 +++++++++++++------ .../cluster_unit/test_cluster_gcs_block.c | 20 +++++++++++++++- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 3317612fe2..5e0ac14a9a 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -11514,8 +11514,10 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi if (!cluster_gcs_pcm_x_revoke_ingress_valid(revoke, env->payload_length, source_node, current_epoch, tag_master, cluster_node_id) || !gcs_block_pcm_x_transfer_ingress_authorized(&revoke->ref.identity.tag, source_node, - current_epoch, &source_session)) + current_epoch, &source_session)) { + gcs_block_pcm_x_revoke_refusal_note("ingress-auth", 0, NULL); return; + } if (!gcs_block_pcm_x_handler_tag_shard_exact(&revoke->ref.identity.tag)) return; worker_id = cluster_ic_tier1_my_data_channel(); @@ -11534,6 +11536,8 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi if (!gcs_block_pcm_x_ticket_ref_equal(&holder_progress.ref, &revoke->ref) || holder_progress.image.image_id != revoke->image_id) { cluster_pcm_x_stats_note_queue_result(PCM_X_QUEUE_STALE); + gcs_block_pcm_x_revoke_refusal_note("holder-ledger-stale", (int)PCM_X_QUEUE_STALE, + NULL); return; } if ((holder_progress.flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0) { @@ -11559,6 +11563,7 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi } } else if (progress_result != PCM_X_QUEUE_NOT_FOUND) { cluster_pcm_x_stats_note_queue_result(progress_result); + gcs_block_pcm_x_revoke_refusal_note("holder-progress", (int)progress_result, NULL); return; } @@ -11585,8 +11590,10 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi image_result = cluster_gcs_block_dedup_pcm_x_reserve( worker_id, &image_key, &revoke->ref.identity.tag, &reserved_binding); if (image_result != GCS_BLOCK_PCM_X_IMAGE_RESERVED - && image_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) + && image_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) { + gcs_block_pcm_x_revoke_refusal_note("image-reserve", (int)image_result, NULL); return; + } new_reservation = image_result == GCS_BLOCK_PCM_X_IMAGE_RESERVED; if (!new_reservation) { image_result = cluster_gcs_block_dedup_pcm_x_rearm_exact( @@ -11603,11 +11610,14 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { gcs_block_pcm_x_revoke_refusal_note(NULL, 0, NULL); gcs_block_pcm_x_wake_registered_holders(&revoke->ref.identity.tag); - } else if (new_reservation - && cluster_gcs_block_dedup_pcm_x_release_exact( - worker_id, &image_key, &revoke->ref.identity.tag, &reserved_binding) - != GCS_BLOCK_PCM_X_IMAGE_RELEASED) - cluster_pcm_x_runtime_fail_closed(); + } else { + gcs_block_pcm_x_revoke_refusal_note("apply", (int)result, NULL); + if (new_reservation + && cluster_gcs_block_dedup_pcm_x_release_exact( + worker_id, &image_key, &revoke->ref.identity.tag, &reserved_binding) + != GCS_BLOCK_PCM_X_IMAGE_RELEASED) + cluster_pcm_x_runtime_fail_closed(); + } } diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index ecb26b2be4..8b0b14199b 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -3638,10 +3638,27 @@ UT_TEST(test_legacy_byte_proof_sites_republish_kept_pi_mirror) } +UT_TEST(test_revoke_handler_silent_refusal_arms_all_note) +{ + char *source = read_gcs_block_source(); + const char *handler = strstr(source, "\ncluster_gcs_handle_pcm_x_revoke_envelope("); + + /* The master re-sends REVOKE forever, so every refusal arm in the source + * handler must name itself through the log-once streak note (t/400 form-B + * wedges previously refused silently at un-instrumented arms): ingress + * auth, holder-ledger stale, holder-progress error, image reserve, apply, + * ingress snapshot, plus the progress reset. */ + UT_ASSERT_NOT_NULL(handler); + if (handler != NULL) + UT_ASSERT(count_occurrences(handler, "gcs_block_pcm_x_revoke_refusal_note(") >= 7); + free(source); +} + + int main(void) { - UT_PLAN(81); + UT_PLAN(82); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -3723,6 +3740,7 @@ main(void) UT_RUN(test_pcm_x_tagless_retire_uses_explicit_data_plane_handoff); UT_RUN(test_pcm_x_role_refresh_accepts_only_same_member_promotion); UT_RUN(test_legacy_byte_proof_sites_republish_kept_pi_mirror); + UT_RUN(test_revoke_handler_silent_refusal_arms_all_note); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From 0994a2a936fe9d6b2231dbc0bc3be3f9ea5e7d26 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Mon, 20 Jul 2026 14:34:21 +0800 Subject: [PATCH 47/56] fix(cluster): deny in-flight legacy readers for PCM-X --- src/backend/cluster/cluster_gcs_block.c | 240 ++++++++++++++---- src/backend/cluster/cluster_gcs_block_dedup.c | 216 ++++++++++++++++ src/backend/cluster/cluster_guc.c | 15 +- src/backend/cluster/cluster_inject.c | 3 +- src/backend/cluster/cluster_pcm_lock.c | 12 +- src/backend/storage/buffer/bufmgr.c | 226 ++++++++++++++--- src/include/cluster/cluster_gcs_block.h | 13 +- src/include/cluster/cluster_gcs_block_dedup.h | 30 ++- src/include/cluster/cluster_pcm_lock.h | 7 +- .../cluster_tap/t/114_gcs_block_3way_2node.pl | 13 +- .../t/390_gcs_block_race_convergence_3node.pl | 7 +- .../t/400_pcm_x_queue_4node_liveness.pl | 4 +- .../test_cluster_bufmgr_pcm_hook.c | 3 +- .../test_cluster_gcs_block_3way.c | 4 +- .../test_cluster_gcs_block_dedup.c | 189 +++++++++++++- .../test_cluster_pcm_direct_init.c | 2 +- src/test/cluster_unit/test_cluster_pcm_lock.c | 10 +- 17 files changed, 859 insertions(+), 135 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 5e0ac14a9a..0544a82c27 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -1893,7 +1893,8 @@ cluster_gcs_block_redo_lsn_covered(int dead_origin, XLogRecPtr required_lsn) bool cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition transition_id, - int master_node, bool clean_eligible) + int master_node, bool clean_eligible, + bool *out_retry_denied) { ClusterGcsBlockOutstandingSlot *slot; uint64 request_id = 0; @@ -1903,6 +1904,7 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans bool granted_storage_fallback = false; bool read_image = false; /* spec-5.2 D2: one-shot read image, non-durable */ bool terminal_denied = false; + bool retry_denied = false; bool retransmit_warning_emitted = false; bool suppress_direct_land = false; uint8 final_status = GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE; @@ -1924,6 +1926,12 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans /* PGRAC: spec-7.2 D6 — ship-latency histogram start stamp. */ TimestampTz ship_started_at; + Assert(out_retry_denied != NULL); + if (out_retry_denied == NULL) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cluster_gcs_send_block_request_and_wait: NULL retry result"))); + *out_retry_denied = false; if (buf == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cluster_gcs_send_block_request_and_wait: NULL BufferDesc"))); @@ -2485,34 +2493,13 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans continue; } - /* PGRAC: spec-2.36 D6 (HC117) — reader starvation guard transient - * denial. N→S request was rejected because an X writer's broadcast - * is in flight at the master; reader exponential-backoffs per - * cluster.gcs_block_starvation_max_retries and - * cluster.gcs_block_starvation_backoff_ms. Budget exhaustion - * surfaces as 53R92 ERRCODE_CLUSTER_GCS_BLOCK_STARVATION_EXHAUSTED. */ + /* Queue arbitration owns this retry boundary. Consume the exact + * denial, release this request slot, and return to bufmgr so it can + * abort the matching GRANT_PENDING token before any backoff. The + * next acquire mints both a fresh token and a fresh request_id; a + * fixed starvation budget must never surface a client ERROR. */ if (final_status == GCS_BLOCK_REPLY_DENIED_PENDING_X) { - int starvation_attempt = retry_attempt; - int starvation_max = cluster_gcs_block_starvation_max_retries; - long backoff_ms; - - if (starvation_attempt >= starvation_max) { - terminal_denied = true; - ereport(ERROR, (errcode(ERRCODE_CLUSTER_GCS_BLOCK_STARVATION_EXHAUSTED), - errmsg("cluster_gcs_block: reader starvation retry budget " - "exhausted (HC117)"))); - break; - } - backoff_ms = (long)cluster_gcs_block_starvation_backoff_ms - * (1L << (starvation_attempt < 16 ? starvation_attempt : 16)); - if (backoff_ms > 25000) - backoff_ms = 25000; - (void)WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, backoff_ms, - WAIT_EVENT_GCS_BLOCK_STARVATION_RETRY); - ResetLatch(MyLatch); - current_master = cluster_gcs_lookup_master(tag); - if (retry_attempt < max_retries) - continue; + retry_denied = true; break; } @@ -2677,9 +2664,10 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans * histogram (GRANTED / STORAGE_FALLBACK / READ_IMAGE all delivered a * usable page; the terminal-denied tail below ereports and loses the * sample, mirroring the xp scopes). */ - if (granted || granted_storage_fallback || read_image) { + if (granted || granted_storage_fallback || read_image) gcs_block_ship_hist_record(ship_started_at); + if (granted || granted_storage_fallback || read_image || retry_denied) { /* * PGRAC: GCS-race round-2 RC-F — completion proof. The terminal * reply was verified and consumed, so no retransmit of this @@ -2727,6 +2715,10 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans return true; if (read_image) return false; + if (retry_denied) { + *out_retry_denied = true; + return false; + } if (terminal_denied) { switch ((GcsBlockReplyStatus)final_status) { @@ -4874,6 +4866,26 @@ gcs_block_forward_send_admitted(int32 holder_node, const GcsBlockForwardPayload return rc == CLUSTER_IC_SEND_DONE || rc == CLUSTER_IC_SEND_WOULD_BLOCK; } +/* The generic IC sender intentionally treats self-destination as a no-op. + * GCS block replies are completion signals, so a same-node denial must enter + * the normal registered handler just like a wire reply. */ +static ClusterICSendResult +gcs_block_send_envelope_or_loopback(uint8 msg_type, int32 dest_node, const void *payload, + uint32 payload_len) +{ + ClusterICEnvelope envelope; + + if (dest_node != cluster_node_id) + return cluster_ic_send_envelope(msg_type, dest_node, payload, payload_len); + if (payload == NULL + || !cluster_ic_envelope_build(&envelope, msg_type, (uint32)cluster_node_id, + (uint32)cluster_node_id, payload, payload_len)) + return CLUSTER_IC_SEND_HARD_ERROR; + return cluster_ic_dispatch_envelope(&envelope, payload, cluster_node_id) + ? CLUSTER_IC_SEND_DONE + : CLUSTER_IC_SEND_HARD_ERROR; +} + static void gcs_block_send_reply(int32 dest_node, const GcsBlockRequestPayload *req, GcsBlockReplyStatus status, XLogRecPtr page_lsn, const char *block_data) @@ -4894,7 +4906,7 @@ gcs_block_send_reply(int32 dest_node, const GcsBlockRequestPayload *req, GcsBloc if (status == GCS_BLOCK_REPLY_GRANTED && block_data != NULL) hdr.checksum = gcs_block_compute_checksum(block_data); - if (GcsBlockRequestPayloadIsDirectLandArmed(req)) { + if (GcsBlockRequestPayloadIsDirectLandArmed(req) && dest_node != cluster_node_id) { if (status != GCS_BLOCK_REPLY_GRANTED || block_data == NULL) { char zero_page[GCS_BLOCK_DATA_SIZE]; @@ -4929,7 +4941,8 @@ gcs_block_send_reply(int32 dest_node, const GcsBlockRequestPayload *req, GcsBloc wire_hdr = (GcsBlockReplyHeader *)buf; *wire_hdr = hdr; wire_hdr->checksum = gcs_block_compute_checksum(buf + sizeof(GcsBlockReplyHeader)); - rc = cluster_ic_send_envelope(PGRAC_IC_MSG_GCS_BLOCK_REPLY, dest_node, buf, total); + rc = gcs_block_send_envelope_or_loopback(PGRAC_IC_MSG_GCS_BLOCK_REPLY, dest_node, buf, + total); pfree(buf); } @@ -4971,7 +4984,7 @@ gcs_block_deny_direct_armed_forward_request(const GcsBlockRequestPayload *req) * then issues a new request with a fresh 4-tuple key (different cluster_ * epoch field) which will MISS_REGISTERED and produce a fresh reply. */ -static void +static bool gcs_block_resend_cached_reply(int32 dest_node, const GcsBlockDedupEntry *entry) { uint32 header_len; @@ -4980,6 +4993,10 @@ gcs_block_resend_cached_reply(int32 dest_node, const GcsBlockDedupEntry *entry) GcsBlockReplyHeader *hdr; char *block_data; bool has_block_payload; + ClusterICSendResult rc; + + if (entry == NULL) + return false; header_len = entry->has_sf_dep && entry->sf_dep_count > 0 ? (uint32)sizeof(GcsBlockReplyHeaderV2) @@ -5016,7 +5033,7 @@ gcs_block_resend_cached_reply(int32 dest_node, const GcsBlockDedupEntry *entry) != GCS_BLOCK_REPLY_NO_FORWARDING_MASTER) { Assert(false); /* classification bug — lookup must route FORWARDED */ pfree(buf); - return; + return false; } block_data = buf + header_len; has_block_payload = entry->status == GCS_BLOCK_REPLY_GRANTED @@ -5024,11 +5041,19 @@ gcs_block_resend_cached_reply(int32 dest_node, const GcsBlockDedupEntry *entry) if (has_block_payload) memcpy(block_data, entry->block_data, GCS_BLOCK_DATA_SIZE); /* else: block_data already zeroed by palloc0 */ + hdr->checksum = gcs_block_compute_checksum(block_data); - cluster_gcs_block_note_send_outcome( - GCS_BLOCK_SEND_FAMILY_REPLY, - cluster_ic_send_envelope(PGRAC_IC_MSG_GCS_BLOCK_REPLY, dest_node, buf, total)); + if ((entry->request_flags & GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND) != 0 + && dest_node != cluster_node_id) { + (void)gcs_block_try_send_direct_reply(dest_node, true, hdr, + has_block_payload ? block_data : NULL, 0, NULL, NULL); + pfree(buf); + return true; + } + rc = gcs_block_send_envelope_or_loopback(PGRAC_IC_MSG_GCS_BLOCK_REPLY, dest_node, buf, total); + cluster_gcs_block_note_send_outcome(GCS_BLOCK_SEND_FAMILY_REPLY, rc); pfree(buf); + return rc == CLUSTER_IC_SEND_DONE || rc == CLUSTER_IC_SEND_WOULD_BLOCK; } @@ -5299,6 +5324,17 @@ gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, return true; } +static bool +gcs_block_queue_pending_x_authoritative(BufferTag tag) +{ + PcmAuthoritySnapshot authority; + + if (!cluster_pcm_lock_authority_snapshot(tag, &authority)) + return false; + return authority.pending_x_requester_node >= 0 + && (authority.pending_x_since_lsn & PCM_PENDING_X_QUEUE_KIND) != 0; +} + /* * cluster_gcs_handle_block_request_envelope — master-side dispatcher. * @@ -5334,6 +5370,8 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo void *block_payload_release_arg = NULL; ClusterSfDepVec sf_dep_vec; bool sf_dep_valid = false; + bool queue_pending_x_before = false; + uint8 request_flags = 0; (void)env; cluster_sf_dep_vec_reset(&sf_dep_vec); @@ -5434,11 +5472,60 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo key.request_id = req->request_id; key.cluster_epoch = req->epoch; memset(&cached_entry, 0, sizeof(cached_entry)); + if (req->transition_id == PCM_TRANS_N_TO_S) + queue_pending_x_before = gcs_block_queue_pending_x_authoritative(req->tag); dr = cluster_gcs_block_dedup_lookup_or_register( dedup_worker_id, &key, req->tag, req->transition_id, GcsBlockRequestPayloadGetLifetimeHintMs(req), cluster_sf_peer_supports_gcs_done(req->sender_node), &cached_entry); + if (dr != GCS_BLOCK_DEDUP_VALIDATION_FAIL && dr != GCS_BLOCK_DEDUP_FULL) { + if (GcsBlockRequestPayloadIsDirectLandArmed(req)) + request_flags |= GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND; + if (!cluster_gcs_block_dedup_set_request_flags_exact( + dedup_worker_id, &key, &req->tag, req->transition_id, request_flags)) { + /* The tuple was removed/replaced or its immutable request properties + * changed between lookup and pinning. Neither case may inherit the + * earlier entry's grant rights. */ + gcs_block_send_reply(req->sender_node, req, + GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, InvalidXLogRecPtr, + NULL); + return; + } + } + + /* Check both sides of registration. A queue claim that linearized before + * lookup routes this request directly to a cached exact denial; a claim + * that races after lookup is caught either here or by the claim-side scan. + * Queue ownership has no same-node exemption. */ + if (req->transition_id == PCM_TRANS_N_TO_S + && (queue_pending_x_before || gcs_block_queue_pending_x_authoritative(req->tag))) { + GcsBlockPendingXDenyResult deny_result; + + if (dr == GCS_BLOCK_DEDUP_VALIDATION_FAIL) { + gcs_block_send_reply(req->sender_node, req, + GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, InvalidXLogRecPtr, + NULL); + return; + } + if (dr == GCS_BLOCK_DEDUP_FULL) { + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_DEDUP_FULL, + InvalidXLogRecPtr, NULL); + return; + } + deny_result = cluster_gcs_block_dedup_pending_x_deny_exact( + dedup_worker_id, &key, &req->tag, req->transition_id, &cached_entry); + if (deny_result != GCS_BLOCK_PENDING_X_DENY_NEW + && deny_result != GCS_BLOCK_PENDING_X_DENY_REPLAY) { + gcs_block_send_reply(req->sender_node, req, + GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, InvalidXLogRecPtr, + NULL); + return; + } + pg_atomic_fetch_add_u64(&ClusterGcsBlock->starvation_denied_pending_x_count, 1); + (void)gcs_block_resend_cached_reply(req->sender_node, &cached_entry); + return; + } switch (dr) { case GCS_BLOCK_DEDUP_CACHED_REPLY: gcs_block_resend_cached_reply(req->sender_node, &cached_entry); @@ -5545,30 +5632,38 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo int32 pending_x; /* spec-2.36 D16 inject — force DENIED_PENDING_X for TAP coverage of - * reader starvation backoff + 53R92 budget exhaustion. */ + * exact abort + fresh-identity reader backoff. */ CLUSTER_INJECTION_POINT("cluster-gcs-block-starvation-force-denied"); if (cluster_injection_should_skip("cluster-gcs-block-starvation-force-denied")) { + GcsBlockPendingXDenyResult deny_result; + pg_atomic_fetch_add_u64(&ClusterGcsBlock->starvation_denied_pending_x_count, 1); - /* A retry-driven denial must not leave the IN_FLIGHT dedup entry - * behind: the requester's backoff retry reuses the same - * (request_id, epoch) key, so a leftover entry swallows the retry - * as IN_FLIGHT_DUPLICATE until the TTL sweep — each swallowed - * round burns a full cluster.gcs_reply_timeout_ms and the - * convergence the deny promises never re-evaluates. */ - cluster_gcs_block_dedup_remove(dedup_worker_id, &key); - gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_PENDING_X, - InvalidXLogRecPtr, NULL); + deny_result = cluster_gcs_block_dedup_pending_x_deny_exact( + dedup_worker_id, &key, &req->tag, req->transition_id, &cached_entry); + if (deny_result == GCS_BLOCK_PENDING_X_DENY_NEW + || deny_result == GCS_BLOCK_PENDING_X_DENY_REPLAY) + (void)gcs_block_resend_cached_reply(req->sender_node, &cached_entry); + else + gcs_block_send_reply(req->sender_node, req, + GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, + InvalidXLogRecPtr, NULL); return; } pending_x = cluster_pcm_lock_query_pending_x_requester(req->tag); if (pending_x >= 0 && pending_x != req->sender_node) { + GcsBlockPendingXDenyResult deny_result; + pg_atomic_fetch_add_u64(&ClusterGcsBlock->starvation_denied_pending_x_count, 1); - /* Same dedup-entry release as the inject branch above: the deny - * contract is "back off and retry, the retry re-evaluates". */ - cluster_gcs_block_dedup_remove(dedup_worker_id, &key); - gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_PENDING_X, - InvalidXLogRecPtr, NULL); + deny_result = cluster_gcs_block_dedup_pending_x_deny_exact( + dedup_worker_id, &key, &req->tag, req->transition_id, &cached_entry); + if (deny_result == GCS_BLOCK_PENDING_X_DENY_NEW + || deny_result == GCS_BLOCK_PENDING_X_DENY_REPLAY) + (void)gcs_block_resend_cached_reply(req->sender_node, &cached_entry); + else + gcs_block_send_reply(req->sender_node, req, + GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, + InvalidXLogRecPtr, NULL); return; } } @@ -9605,6 +9700,43 @@ gcs_block_pcm_x_ensure_pending_x_claim(const PcmXMasterDriveSnapshot *snapshot) return PCM_X_QUEUE_OK; } +/* Once the queue-kind cookie is visible, every older same-tag legacy reader + * must lose its grant/forward right before PROBE/REVOKE can advance. Drain + * NEW victims one by one (no bounded receiver array), then replay one exact + * unacknowledged denial per drive tick until requester DONE removes it. */ +static PcmXQueueResult +gcs_block_pcm_x_deny_legacy_readers(const PcmXMasterDriveSnapshot *snapshot) +{ + GcsBlockDedupEntry denied; + GcsBlockPendingXDenyResult deny_result; + int worker_id; + + if (snapshot == NULL + || !cluster_pcm_lock_queue_pending_x_exact(snapshot->ref.identity.tag, + snapshot->ref.identity.node_id, + snapshot->ref.handle.ticket_id)) + return PCM_X_QUEUE_BAD_STATE; + worker_id = cluster_lms_shard_for_tag(&snapshot->ref.identity.tag, cluster_lms_workers); + if (worker_id < 0 || worker_id >= cluster_lms_workers) + return PCM_X_QUEUE_INVALID; + + for (;;) { + memset(&denied, 0, sizeof(denied)); + deny_result = cluster_gcs_block_dedup_pending_x_deny_next( + worker_id, &snapshot->ref.identity.tag, &denied); + if (deny_result == GCS_BLOCK_PENDING_X_DENY_NOT_FOUND) + return PCM_X_QUEUE_OK; + if (deny_result != GCS_BLOCK_PENDING_X_DENY_NEW + && deny_result != GCS_BLOCK_PENDING_X_DENY_REPLAY) + return PCM_X_QUEUE_CORRUPT; + if (denied.key.origin_node_id >= PCM_X_PROTOCOL_NODE_LIMIT + || !gcs_block_resend_cached_reply((int32)denied.key.origin_node_id, &denied)) + return PCM_X_QUEUE_BUSY; + if (deny_result == GCS_BLOCK_PENDING_X_DENY_REPLAY) + return PCM_X_QUEUE_OK; + } +} + static PcmXQueueResult gcs_block_pcm_x_stage_queue_invalidations(const PcmXMasterDriveSnapshot *snapshot, @@ -9935,11 +10067,15 @@ gcs_block_pcm_x_master_drive_tag(const BufferTag *tag, uint64 cluster_epoch) } else if (snapshot.ticket_state == PCM_XT_ACTIVE_PROBE) { drive_stage = "probe"; result = gcs_block_pcm_x_ensure_pending_x_claim(&snapshot); + if (result == PCM_X_QUEUE_OK) + result = gcs_block_pcm_x_deny_legacy_readers(&snapshot); if (result == PCM_X_QUEUE_OK) result = gcs_block_pcm_x_master_drive_probe(&snapshot); } else if (snapshot.ticket_state == PCM_XT_ACTIVE_TRANSFER) { drive_stage = "transfer"; - result = gcs_block_pcm_x_master_drive_transfer(&snapshot); + result = gcs_block_pcm_x_deny_legacy_readers(&snapshot); + if (result == PCM_X_QUEUE_OK) + result = gcs_block_pcm_x_master_drive_transfer(&snapshot); } else { drive_stage = "state"; result = PCM_X_QUEUE_CORRUPT; diff --git a/src/backend/cluster/cluster_gcs_block_dedup.c b/src/backend/cluster/cluster_gcs_block_dedup.c index eb687b0b8b..8bcfa89b10 100644 --- a/src/backend/cluster/cluster_gcs_block_dedup.c +++ b/src/backend/cluster/cluster_gcs_block_dedup.c @@ -689,6 +689,214 @@ cluster_gcs_block_dedup_lookup_or_register(int worker_id, const GcsBlockDedupKey return result; } +static bool +dedup_pending_x_denial_is_exact(const GcsBlockDedupEntry *entry) +{ + const GcsBlockReplyHeader *header = &entry->reply_header; + + return entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_GENERIC + && entry->transition_id == (uint8)PCM_TRANS_N_TO_S + && entry->status == (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X + && entry->completed_at_ts != 0 + && header->request_id == entry->key.request_id + && header->epoch == entry->key.cluster_epoch + && header->sender_node == cluster_node_id + && header->requester_backend_id == entry->key.requester_backend_id + && header->transition_id == (uint8)PCM_TRANS_N_TO_S + && header->status == (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X + && GcsBlockReplyHeaderGetForwardingMasterNode(header) + == GCS_BLOCK_REPLY_NO_FORWARDING_MASTER; +} + +static bool +dedup_pending_x_entry_has_legacy_s_right(const GcsBlockDedupEntry *entry) +{ + GcsBlockReplyStatus status; + + if (entry->completed_at_ts == 0) + return true; + status = (GcsBlockReplyStatus)entry->status; + return status == GCS_BLOCK_REPLY_GRANTED + || status == GCS_BLOCK_REPLY_GRANTED_STORAGE_FALLBACK + || status == GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER + || status == GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER + || status == GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE; +} + +static void +dedup_pending_x_install_denial(GcsBlockDedupEntry *entry) +{ + GcsBlockReplyHeader denial; + + memset(&denial, 0, sizeof(denial)); + denial.request_id = entry->key.request_id; + denial.epoch = entry->key.cluster_epoch; + denial.sender_node = cluster_node_id; + denial.requester_backend_id = entry->key.requester_backend_id; + denial.transition_id = (uint8)PCM_TRANS_N_TO_S; + denial.status = (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X; + GcsBlockReplyHeaderSetForwardingMasterNode(&denial, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + + entry->status = (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X; + entry->reply_header = denial; + entry->has_sf_dep = false; + entry->sf_flags = 0; + entry->sf_dep_count = 0; + cluster_sf_dep_vec_reset(&entry->payload_meta.sf_dep_vec); + memset(entry->block_data, 0, sizeof(entry->block_data)); + entry->completed_at_ts = GetCurrentTimestamp(); + entry->done_at_ts = 0; +} + +/* PCM-X queue arbitration must revoke the right of an older legacy reader + * before type-49 can wait on that reader's GRANT_PENDING reservation. The + * scan returns at most one newly terminated identity per call so the caller + * can send it without a bounded/sentinel array; after all live rights are + * gone it returns one unacknowledged cached denial for loss recovery. */ +GcsBlockPendingXDenyResult +cluster_gcs_block_dedup_pending_x_deny_next(int worker_id, const BufferTag *tag, + GcsBlockDedupEntry *denied_out) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + HASH_SEQ_STATUS scan; + GcsBlockDedupEntry *entry; + GcsBlockDedupEntry replay; + bool have_replay = false; + + Assert(tag != NULL); + Assert(denied_out != NULL); + if (tag == NULL || denied_out == NULL) + return GCS_BLOCK_PENDING_X_DENY_INVALID; + memset(denied_out, 0, sizeof(*denied_out)); + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PENDING_X_DENY_INVALID; + + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + hash_seq_init(&scan, htab); + while ((entry = (GcsBlockDedupEntry *)hash_seq_search(&scan)) != NULL) { + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC + || entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 || entry->done_at_ts != 0) + continue; + + if (dedup_pending_x_denial_is_exact(entry)) { + if (!have_replay) { + replay = *entry; + have_replay = true; + } + continue; + } + if (entry->status == (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X + && entry->completed_at_ts != 0) { + dedup_pcm_x_note_failclosed(shard); + hash_seq_term(&scan); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PENDING_X_DENY_INVALID; + } + if (!dedup_pending_x_entry_has_legacy_s_right(entry)) + continue; + + dedup_pending_x_install_denial(entry); + *denied_out = *entry; + hash_seq_term(&scan); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PENDING_X_DENY_NEW; + } + + if (have_replay) + *denied_out = replay; + LWLockRelease(&shard->lock.lock); + return have_replay ? GCS_BLOCK_PENDING_X_DENY_REPLAY + : GCS_BLOCK_PENDING_X_DENY_NOT_FOUND; +} + +GcsBlockPendingXDenyResult +cluster_gcs_block_dedup_pending_x_deny_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, uint8 transition_id, + GcsBlockDedupEntry *denied_out) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + bool found = false; + + Assert(key != NULL); + Assert(tag != NULL); + Assert(denied_out != NULL); + if (key == NULL || tag == NULL || denied_out == NULL + || transition_id != (uint8)PCM_TRANS_N_TO_S) + return GCS_BLOCK_PENDING_X_DENY_INVALID; + memset(denied_out, 0, sizeof(*denied_out)); + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PENDING_X_DENY_INVALID; + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (!found || entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 + || entry->transition_id != transition_id) { + if (found && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC) + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PENDING_X_DENY_INVALID; + } + if (dedup_pending_x_denial_is_exact(entry)) { + *denied_out = *entry; + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PENDING_X_DENY_REPLAY; + } + if (entry->status == (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X + && entry->completed_at_ts != 0) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PENDING_X_DENY_INVALID; + } + + dedup_pending_x_install_denial(entry); + *denied_out = *entry; + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PENDING_X_DENY_NEW; +} + +bool +cluster_gcs_block_dedup_set_request_flags_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, uint8 transition_id, + uint8 request_flags) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + bool found = false; + bool updated = false; + + Assert(key != NULL); + Assert(tag != NULL); + if (key == NULL || tag == NULL + || (request_flags & ~GCS_BLOCK_DEDUP_REQUEST_F_VALID_MASK) != 0) + return false; + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return false; + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (found && entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_GENERIC + && memcmp(&entry->tag, tag, sizeof(*tag)) == 0 + && entry->transition_id == transition_id) { + uint8 pinned_flags = GCS_BLOCK_DEDUP_REQUEST_F_PINNED | request_flags; + + if (entry->request_flags == 0) + entry->request_flags = pinned_flags; + updated = entry->request_flags == pinned_flags; + } else if (found && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC) + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return updated; +} + GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_reserve(int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, @@ -1462,6 +1670,14 @@ cluster_gcs_block_dedup_install_reply_ex(int worker_id, const GcsBlockDedupKey * LWLockRelease(&shard->lock.lock); return; } + /* A queue-kind pending-X claim has already terminated this exact legacy + * reader. Any asynchronous old producer has permanently lost the right + * to restore GRANTED/FORWARD/page bytes; preserve the cached denial for + * retransmit until exact DONE. */ + if (dedup_pending_x_denial_is_exact(entry)) { + LWLockRelease(&shard->lock.lock); + return; + } entry->status = (uint8)status; entry->reply_header = *header; diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 5232cf7633..9999046168 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -800,8 +800,9 @@ int cluster_gcs_block_recovery_wait_ms = 200; * DENIED_INVALIDATE_TIMEOUT reply (status 11) → 53R91 SQLSTATE. * cluster_gcs_block_starvation_backoff_ms (HC117): reader backoff * base for DENIED_PENDING_X retry loop; backoff = base × 2^attempt. - * cluster_gcs_block_starvation_max_retries (HC117): reader retry - * budget; exhaustion → 53R92 SQLSTATE. + * cluster_gcs_block_starvation_max_retries (HC117): retained as a + * configuration-compatibility surface. Queue-arbitrated reader retry is + * now unbounded/cancelable and does not consume this legacy budget. */ int cluster_gcs_block_invalidate_ack_timeout_ms = 1500; int cluster_gcs_block_starvation_backoff_ms = 100; @@ -4204,10 +4205,12 @@ cluster_init_guc(void) "2^attempt. HC117. PGC_SUSET."), &cluster_gcs_block_starvation_backoff_ms, 100, 1, 60000, PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomIntVariable( - "cluster.gcs_block_starvation_max_retries", gettext_noop("S barrier reader retry budget."), - gettext_noop("Reader DENIED_PENDING_X retry budget. Budget exhausted → " - "ereport(53R92); upper-layer transaction may retry the " - "whole statement. HC117. PGC_SUSET."), + "cluster.gcs_block_starvation_max_retries", + gettext_noop("Legacy S barrier reader retry budget compatibility setting."), + gettext_noop("Retained for configuration compatibility. DENIED_PENDING_X now " + "exact-aborts its old reservation and retries with fresh identities " + "until success or query cancellation; this value no longer causes " + "a client error. HC117. PGC_SUSET."), &cluster_gcs_block_starvation_max_retries, 8, 0, 64, PGC_SUSET, 0, NULL, NULL, NULL); /* diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index c040c0fe98..8e50dd4b79 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -538,8 +538,7 @@ static ClusterInjectPoint cluster_injection_points[] = { * * cluster-gcs-block-starvation-force-denied: * Fires inside master N→S handler. SKIP forces DENIED_PENDING_X - * reply unconditionally so TAP can drive reader backoff retry + - * 53R92 ERRCODE_CLUSTER_GCS_BLOCK_STARVATION_EXHAUSTED. + * reply so TAP can drive exact abort + fresh-identity backoff retry. * * cluster-catalog-services-ready-force-closed: * Fires inside cluster_catalog_services_ready (spec-6.14 D8). SKIP diff --git a/src/backend/cluster/cluster_pcm_lock.c b/src/backend/cluster/cluster_pcm_lock.c index dd29642b4a..3c00a15af2 100644 --- a/src/backend/cluster/cluster_pcm_lock.c +++ b/src/backend/cluster/cluster_pcm_lock.c @@ -2747,12 +2747,18 @@ cluster_pcm_clean_page_xfer_consume(void) * (HC84 PageSetLSN + memcpy under content_lock EXCLUSIVE). */ bool -cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) +cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode, bool *out_retry_denied) { BufferTag tag; int master_node; bool clean_eligible; + Assert(out_retry_denied != NULL); + if (out_retry_denied == NULL) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cluster_pcm_lock_acquire_buffer: NULL retry result"))); + *out_retry_denied = false; if (buf == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cluster_pcm_lock_acquire_buffer: NULL BufferDesc"))); @@ -2857,8 +2863,8 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) * spec-5.2a D2/D3: carry clean-page eligibility (X only) so the remote * master takes the dedicated clean-page X-transfer path rather than the * conservative HG7 fail-closed DENY. */ - return cluster_gcs_send_block_request_and_wait(buf, trans, master_node, - clean_eligible && mode == PCM_LOCK_MODE_X); + return cluster_gcs_send_block_request_and_wait( + buf, trans, master_node, clean_eligible && mode == PCM_LOCK_MODE_X, out_retry_denied); } /* diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 783bccbea5..8a218685d9 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -1345,12 +1345,10 @@ cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, * fresh begin against the re-sampled complete ownership tuple; another * lifecycle's token/flags are never touched. No content lock is held at * either call site, and a backend holding OTHER frozen-tag content locks - * must not sleep (nested-guard discipline) — both the guard refusal and the - * bounded-wait exhaustion fall back to the historical fail-closed report in - * the caller. + * must not sleep (nested-guard discipline). There is deliberately no retry + * budget: ordinary ownership competition is cancelable waiting, never a + * client-visible exhaustion error. */ -#define CLUSTER_BUFMGR_PCM_RESERVATION_MAX_WAITS 64 - static ClusterPcmOwnResult cluster_bufmgr_pcm_begin_grant_reservation_wait(BufferDesc *buf, ClusterPcmOwnSnapshot *base_out, @@ -1362,8 +1360,7 @@ cluster_bufmgr_pcm_begin_grant_reservation_wait(BufferDesc *buf, for (;;) { result = cluster_pcm_own_begin_grant_reservation(buf, base_out, token_out); - if (result != CLUSTER_PCM_OWN_BUSY - || waits >= CLUSTER_BUFMGR_PCM_RESERVATION_MAX_WAITS) + if (result != CLUSTER_PCM_OWN_BUSY) return result; if (cluster_pcm_x_nested_wait_guard_before_block() != PCM_X_QUEUE_OK) return result; @@ -1373,7 +1370,8 @@ cluster_bufmgr_pcm_begin_grant_reservation_wait(BufferDesc *buf, cluster_pcm_x_holder_retry_delay_ms(waits), WAIT_EVENT_PCM_BLOCK_CONVERT_WAIT); CHECK_FOR_INTERRUPTS(); - waits++; + if (waits < PG_UINT32_MAX) + waits++; } } @@ -1407,6 +1405,84 @@ cluster_bufmgr_pcm_legacy_begin_probe(BufferDesc *buf, PcmLockMode pcm_mode, base->flags, buf->pcm_state); } +typedef enum ClusterBufmgrPcmRetryRearmResult +{ + CLUSTER_BUFMGR_PCM_RETRY_REARMED = 0, + CLUSTER_BUFMGR_PCM_RETRY_COVERED, + CLUSTER_BUFMGR_PCM_RETRY_BARRIER_REFUSED +} ClusterBufmgrPcmRetryRearmResult; + +/* Consume one authoritative DENIED_PENDING_X. The exact old reservation is + * aborted before sleeping; STALE is an ABA-safe zero-mutation result and is + * handled by re-sampling the complete ownership tuple. A successful rearm + * always publishes a fresh token, while the subsequent GCS call allocates a + * fresh request_id. */ +static ClusterBufmgrPcmRetryRearmResult +cluster_bufmgr_pcm_retry_denied_rearm(BufferDesc *buf, PcmLockMode pcm_mode, + ClusterPcmOwnSnapshot *base, + uint64 *reservation_token, uint32 wait_index, + bool *barrier_refused, uint64 *covered_generation) +{ + ClusterPcmOwnResult own_result; + PcmXQueueResult guard_result; + uint8 current_state; + uint32 current_flags; + long backoff_ms; + + if (buf == NULL || base == NULL || reservation_token == NULL + || covered_generation == NULL) + { + cluster_pcm_own_report_bump_failure(buf, CLUSTER_PCM_OWN_INVALID, 0, 0, + "pending-X retry arguments"); + return CLUSTER_BUFMGR_PCM_RETRY_BARRIER_REFUSED; + } + own_result = cluster_pcm_own_abort_grant_reservation(buf, base, *reservation_token); + if (own_result != CLUSTER_PCM_OWN_OK && own_result != CLUSTER_PCM_OWN_STALE) + cluster_pcm_own_report_bump_failure(buf, own_result, base->generation, base->flags, + "pending-X exact abort"); + + /* A stale exact abort never touches the successor. If that successor is + * already a stable covering grant, rejoin the normal post-content-lock + * generation check instead of opening another reservation over it. */ + cluster_pcm_own_read(buf, ¤t_state, covered_generation, ¤t_flags); + if (current_flags == 0 && cluster_gcs_block_local_cache + && cluster_pcm_mode_covers((PcmLockMode) current_state, pcm_mode)) + return CLUSTER_BUFMGR_PCM_RETRY_COVERED; + + guard_result = cluster_pcm_x_nested_wait_guard_before_block(); + if (guard_result != PCM_X_QUEUE_OK) + { + if (barrier_refused != NULL && guard_result == PCM_X_QUEUE_BARRIER_CLOSED) + { + *barrier_refused = true; + cluster_pcm_x_stats_note_barrier_unwind(); + return CLUSTER_BUFMGR_PCM_RETRY_BARRIER_REFUSED; + } + cluster_bufmgr_pcm_x_holder_report_failure(guard_result, buf, + "pending-X retry nested guard"); + } + + backoff_ms = (long) cluster_gcs_block_starvation_backoff_ms + * (1L << (wait_index < 16 ? wait_index : 16)); + if (backoff_ms <= 0) + backoff_ms = 1; + if (backoff_ms > 25000) + backoff_ms = 25000; + CHECK_FOR_INTERRUPTS(); + (void) WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, backoff_ms, + WAIT_EVENT_GCS_BLOCK_STARVATION_RETRY); + ResetLatch(MyLatch); + CHECK_FOR_INTERRUPTS(); + + own_result = cluster_bufmgr_pcm_begin_grant_reservation_wait(buf, base, + reservation_token); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_own_report_bump_failure(buf, own_result, base->generation, base->flags, + "pending-X retry reservation"); + cluster_bufmgr_pcm_legacy_begin_probe(buf, pcm_mode, base, "pending-X retry reservation"); + return CLUSTER_BUFMGR_PCM_RETRY_REARMED; +} + static ClusterPcmXHolderLedgerEntry * cluster_bufmgr_pcm_x_holder_find(BufferDesc *buf) { @@ -2529,7 +2605,9 @@ cluster_bufmgr_pcm_gate_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind ki ClusterPcmOwnSnapshot pending_base; ClusterPcmOwnResult pending_result; bool grant_acquired = false; + bool retry_denied = false; uint32 buf_state; + uint32 retry_wait_index = 0; uint64 pending_token = 0; uint64 committed_generation; @@ -2561,17 +2639,34 @@ cluster_bufmgr_pcm_gate_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind ki cluster_bufmgr_pcm_direct_init_report_failure(buf, pending_result, &observed, "direct-init consume"); - PG_TRY(); - { - grant_acquired = cluster_pcm_lock_acquire_buffer(buf, PCM_LOCK_MODE_X); - } - PG_CATCH(); + for (;;) { - cluster_pcm_own_abort_grant_after_error(buf, &pending_base, pending_token, + ClusterBufmgrPcmRetryRearmResult rearm_result; + uint64 covered_generation = 0; + + retry_denied = false; + PG_TRY(); + { + grant_acquired = cluster_pcm_lock_acquire_buffer(buf, PCM_LOCK_MODE_X, &retry_denied); + } + PG_CATCH(); + { + cluster_pcm_own_abort_grant_after_error(buf, &pending_base, pending_token, "direct-init acquire"); - PG_RE_THROW(); + PG_RE_THROW(); + } + PG_END_TRY(); + if (!retry_denied) + break; + rearm_result = cluster_bufmgr_pcm_retry_denied_rearm( + buf, PCM_LOCK_MODE_X, &pending_base, &pending_token, retry_wait_index, NULL, + &covered_generation); + if (retry_wait_index < PG_UINT32_MAX) + retry_wait_index++; + if (rearm_result != CLUSTER_BUFMGR_PCM_RETRY_REARMED) + cluster_bufmgr_pcm_direct_init_report_failure( + buf, CLUSTER_PCM_OWN_STALE, &observed, "direct-init pending-X rearm"); } - PG_END_TRY(); if (grant_acquired) cluster_pcm_own_finish_grant_or_rollback(buf, &pending_base, pending_token, @@ -8139,6 +8234,8 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) ClusterPcmXHolderLedgerEntry *pcm_x_holder = NULL; ClusterPcmXWriterLedgerEntry *pcm_x_writer = NULL; bool pcm_x_writer_managed = false; + bool pcm_retry_denied = false; + uint32 pcm_retry_wait_index = 0; #endif Assert(BufferIsPinned(buffer)); @@ -8263,25 +8360,44 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) * left at N (the next access re-fetches — a cached copy * with no invalidation path would go stale, Rule 8.A). */ - PG_TRY(); - { - pcm_acquired = cluster_pcm_lock_acquire_buffer(buf, pcm_mode); - } - PG_CATCH(); + for (;;) { - /* - * W3 — an acquire that THREW (upgrade-invalidate - * timeout, transfer deny, ...) must not leak - * GRANT_PENDING: a stale marker parks every later - * same-tag INVALIDATE on this node (never ACKed -> remote - * upgrades wedge, liveness). The later PG_CATCH below - * only covers the content-lock window, not this acquire. - */ - cluster_pcm_own_abort_grant_after_error( - buf, &pcm_pending_base, pcm_pending_token, "LockBuffer master acquire"); - PG_RE_THROW(); + ClusterBufmgrPcmRetryRearmResult rearm_result; + + pcm_retry_denied = false; + PG_TRY(); + { + pcm_acquired = cluster_pcm_lock_acquire_buffer(buf, pcm_mode, + &pcm_retry_denied); + } + PG_CATCH(); + { + /* An acquire that throws must not leak the exact + * GRANT_PENDING marker into later INVALIDATEs. */ + cluster_pcm_own_abort_grant_after_error( + buf, &pcm_pending_base, pcm_pending_token, + "LockBuffer master acquire"); + PG_RE_THROW(); + } + PG_END_TRY(); + if (!pcm_retry_denied) + break; + + pcm_pending_set = false; + rearm_result = cluster_bufmgr_pcm_retry_denied_rearm( + buf, pcm_mode, &pcm_pending_base, &pcm_pending_token, + pcm_retry_wait_index, pcm_barrier_refused, &pcm_covered_gen); + if (pcm_retry_wait_index < PG_UINT32_MAX) + pcm_retry_wait_index++; + if (rearm_result == CLUSTER_BUFMGR_PCM_RETRY_BARRIER_REFUSED) + return; + if (rearm_result == CLUSTER_BUFMGR_PCM_RETRY_COVERED) + { + pcm_covered = true; + break; + } + pcm_pending_set = true; } - PG_END_TRY(); } } } @@ -8413,6 +8529,8 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) { if (pcm_x_writer == NULL) { + ClusterBufmgrPcmRetryRearmResult rearm_result; + /* * Share-mode refresh, or the convert queue is not * managing this relation: the legacy master @@ -8428,15 +8546,41 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) buf, pcm_mode, &pcm_pending_base, "revalidate-reservation"); pcm_pending_set = true; - pcm_acquired = cluster_pcm_lock_acquire_buffer(buf, pcm_mode); + for (;;) + { + pcm_retry_denied = false; + pcm_acquired = cluster_pcm_lock_acquire_buffer( + buf, pcm_mode, &pcm_retry_denied); + if (!pcm_retry_denied) + break; + pcm_pending_set = false; + rearm_result = cluster_bufmgr_pcm_retry_denied_rearm( + buf, pcm_mode, &pcm_pending_base, &pcm_pending_token, + pcm_retry_wait_index, pcm_barrier_refused, + &pcm_covered_gen); + if (pcm_retry_wait_index < PG_UINT32_MAX) + pcm_retry_wait_index++; + if (rearm_result + == CLUSTER_BUFMGR_PCM_RETRY_BARRIER_REFUSED) + break; + if (rearm_result == CLUSTER_BUFMGR_PCM_RETRY_COVERED) + { + pcm_covered = true; + break; + } + pcm_pending_set = true; + } + } + if (pcm_barrier_refused == NULL || !*pcm_barrier_refused) + { + pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf, NULL); + if (mode == BUFFER_LOCK_SHARE) + LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED); + else + LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE); + cluster_bufmgr_pcm_x_writer_activate(pcm_x_writer); + cluster_pcm_note_writer_reverify_reacquire(); } - pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf, NULL); - if (mode == BUFFER_LOCK_SHARE) - LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED); - else - LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE); - cluster_bufmgr_pcm_x_writer_activate(pcm_x_writer); - cluster_pcm_note_writer_reverify_reacquire(); } } } diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index 11ec756f62..5385f493f8 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -3281,14 +3281,15 @@ extern int cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, /* * Returns true if a DURABLE PCM grant was acquired (GRANTED / STORAGE_ * FALLBACK — the caller mirrors PCM ownership into buf->pcm_state). Returns - * false for a spec-5.2 D2 one-shot READ_IMAGE: the bytes were installed for - * this read only, no S holder was registered, and the caller MUST leave - * buf->pcm_state == N so the next access re-fetches. Terminal denials - * ereport(ERROR) and do not return. + * false for a spec-5.2 D2 one-shot READ_IMAGE, or for an authoritative + * DENIED_PENDING_X with *out_retry_denied set. In the latter case the caller + * must exact-abort its GRANT_PENDING reservation before waiting/re-entering. + * Terminal denials ereport(ERROR) and do not return. */ extern bool cluster_gcs_send_block_request_and_wait(BufferDesc *buf, - PcmLockTransition transition_id, - int master_node, bool clean_eligible); + PcmLockTransition transition_id, + int master_node, bool clean_eligible, + bool *out_retry_denied); /* * spec-5.2 D2 (sub-case B) — local-master read-image forward. Used by diff --git a/src/include/cluster/cluster_gcs_block_dedup.h b/src/include/cluster/cluster_gcs_block_dedup.h index 3d8e220fb6..93d6f4ea0d 100644 --- a/src/include/cluster/cluster_gcs_block_dedup.h +++ b/src/include/cluster/cluster_gcs_block_dedup.h @@ -169,7 +169,7 @@ typedef struct GcsBlockDedupEntry { uint8 transition_id; /* 1B — HC91 collision check */ uint8 status; /* 1B — GcsBlockReplyStatus */ uint8 entry_kind; /* 1B — GcsBlockDedupEntryKind */ - uint8 _pad0; /* 1B — session @ 48 */ + uint8 request_flags; /* 1B — original request properties */ uint64 pcm_x_master_session; /* 8B — PCM-X kind only */ GcsBlockReplyHeader reply_header; /* 48B — full reply header (HC99) */ bool has_sf_dep; /* 1B — spec-6.2 v2 dep vector present */ @@ -185,6 +185,10 @@ typedef struct GcsBlockDedupEntry { int64 pinned_done_linger_us; /* 8B — round-2: quarantine pinned */ } GcsBlockDedupEntry; +#define GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND UINT8_C(0x01) +#define GCS_BLOCK_DEDUP_REQUEST_F_VALID_MASK GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND +#define GCS_BLOCK_DEDUP_REQUEST_F_PINNED UINT8_C(0x80) + StaticAssertDecl(offsetof(GcsBlockDedupEntry, entry_kind) == 46, "dedup entry kind occupies established padding at offset 46"); StaticAssertDecl(offsetof(GcsBlockDedupEntry, pcm_x_master_session) == 48, @@ -272,6 +276,17 @@ typedef enum GcsBlockDedupResult { * path rather than re-broadcast. */ } GcsBlockDedupResult; +/* Result of advancing the exact legacy-reader termination set after a + * queue-kind pending-X claim has become authoritative. NEW means one live + * N->S identity was atomically replaced with a cached retry denial; REPLAY + * returns an already-denied identity that still lacks exact DONE proof. */ +typedef enum GcsBlockPendingXDenyResult { + GCS_BLOCK_PENDING_X_DENY_INVALID = -1, + GCS_BLOCK_PENDING_X_DENY_NOT_FOUND = 0, + GCS_BLOCK_PENDING_X_DENY_NEW = 1, + GCS_BLOCK_PENDING_X_DENY_REPLAY = 2 +} GcsBlockPendingXDenyResult; + /* ============================================================ * Eager reclaim safety decision primitives (spec-7.2a; review r1). @@ -453,6 +468,19 @@ extern GcsBlockDedupResult cluster_gcs_block_dedup_lookup_or_register( uint32 requester_lifetime_hint_ms, bool requester_done_capable, GcsBlockDedupEntry *cached_reply_out); +/* Under the routed shard lock, terminate one same-tag, still-live legacy + * N->S grant/forward identity after PCM-X publishes its queue-kind claim. + * A by-value cached denial is returned for initial send or periodic replay; + * exact DONE removes it from the replay set. */ +extern GcsBlockPendingXDenyResult cluster_gcs_block_dedup_pending_x_deny_next( + int worker_id, const BufferTag *tag, GcsBlockDedupEntry *denied_out); +extern GcsBlockPendingXDenyResult cluster_gcs_block_dedup_pending_x_deny_exact( + int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, uint8 transition_id, + GcsBlockDedupEntry *denied_out); +extern bool cluster_gcs_block_dedup_set_request_flags_exact( + int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, uint8 transition_id, + uint8 request_flags); + /* Dedicated PCM-X image storage over the existing dedup entry pool. Reserve * claims capacity before the revoke lifecycle starts. Materialize publishes * immutable bytes only after the caller has established its exact revoke diff --git a/src/include/cluster/cluster_pcm_lock.h b/src/include/cluster/cluster_pcm_lock.h index 70235256f2..ee2d8935ae 100644 --- a/src/include/cluster/cluster_pcm_lock.h +++ b/src/include/cluster/cluster_pcm_lock.h @@ -315,9 +315,12 @@ extern void cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode); * Returns true if a DURABLE PCM grant was recorded (caller mirrors ownership * into buf->pcm_state). Returns false for a spec-5.2 D2 one-shot READ_IMAGE: * bytes were installed for this read only and the caller MUST leave - * buf->pcm_state == N so the next access re-fetches. + * buf->pcm_state == N so the next access re-fetches. A false return with + * *out_retry_denied set is instead a queue-arbitration retry boundary; the + * caller must exact-abort and replace its GRANT_PENDING reservation. */ -extern bool cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode); +extern bool cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode, + bool *out_retry_denied); /* * PGRAC: spec-5.2a D2 — clean-page X-transfer arm (backend-local, one-shot). diff --git a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl index 88cca599f6..cc781e1242 100644 --- a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl +++ b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl @@ -17,9 +17,8 @@ # L5 S barrier injection — DENIED_PENDING_X surfaces under # cluster-gcs-block-starvation-force-denied inject; reader # sees starvation_denied_pending_x_count tick -# L6 53R92 ERRCODE_CLUSTER_GCS_BLOCK_STARVATION_EXHAUSTED triggered -# when reader exhausts cluster.gcs_block_starvation_max_retries -# (set to 0 via runtime GUC so first inject denial → ereport) +# L6 a zero legacy starvation budget cannot surface a client error; +# one forced denial exact-aborts and re-enters with fresh identities # L7 53R91 ERRCODE_CLUSTER_GCS_BLOCK_INVALIDATE_TIMEOUT triggered # via cluster-gcs-block-invalidate-stall-ack inject (holder # never acks); budget exhaustion → DENIED_INVALIDATE_TIMEOUT @@ -161,7 +160,7 @@ sub gcs_int "L5 starvation_denied_pending_x_count tick under inject " . "(before=$r1, after=$r2)"); - # L6: 53R92 budget exhaustion — set retries=0 so first deny errors. + # L6: the legacy zero budget cannot turn queue competition into ERROR. $pair->node0->safe_psql('postgres', 'SET cluster.gcs_block_starvation_max_retries = 0'); $pair->inject_skip_set($pair->node0, @@ -173,10 +172,8 @@ sub gcs_int 'cluster-gcs-block-starvation-force-denied', 0); $pair->node0->safe_psql('postgres', 'RESET cluster.gcs_block_starvation_max_retries'); - # Result may be PASS if the local backend's transition doesn't - # route through GCS (single-node ish) — keep cmp_ok permissive. - cmp_ok($rc, '>=', 0, - "L6 starvation retry exhaustion exit_code observed (rc=$rc)"); + is($rc, 0, "L6 pending-X retry stays internal with legacy budget zero") + or diag("L6 stdout=[$stdout] stderr=[$stderr]"); # L7: 53R91 — invalidate ack timeout via stall inject (no X transfer # attempted in 2-node-S baseline, so we just observe the inject is diff --git a/src/test/cluster_tap/t/390_gcs_block_race_convergence_3node.pl b/src/test/cluster_tap/t/390_gcs_block_race_convergence_3node.pl index 76923c2ce0..1d82f13e4e 100644 --- a/src/test/cluster_tap/t/390_gcs_block_race_convergence_3node.pl +++ b/src/test/cluster_tap/t/390_gcs_block_race_convergence_3node.pl @@ -189,10 +189,9 @@ sub read_retry # ============================================================ # L2b (RC-B, injection-deterministic): force ONE DENIED_PENDING_X on a -# cross-node N->S read. The reader's backoff retry reuses the same -# (request_id, epoch) key; the master must re-evaluate it (deny released -# the dedup entry), not swallow it as IN_FLIGHT_DUPLICATE for a full -# cluster.gcs_reply_timeout_ms round. +# cross-node N->S read. The reader exact-aborts the denied reservation and +# re-enters with a fresh request_id/token; the old denial remains cached until +# exact DONE and cannot swallow or authorize the successor identity. # ============================================================ my $base_timeout2 = gcs_sum($triple, 'block_timeout_count'); my $base_starve = gcs_sum($triple, 'starvation_denied_pending_x_count'); diff --git a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl index 1d409e7c5c..cf8b4d67a6 100644 --- a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl +++ b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl @@ -631,8 +631,8 @@ sub write_file cmp_ok($queue_after - $queue_before, '>=', 4, 'L3 all four node writers entered the PCM-X queue protocol'); -is($denied_after - $denied_before, 0, - 'L3 no writer fell back to legacy reader pending-X denial'); +cmp_ok($denied_after - $denied_before, '>', 0, + 'L3 Shape-B arbitration denied an in-flight legacy reader without a client error'); cmp_ok($passive_s_after - $passive_s_before, '>', 0, 'L3 exact queue INVALIDATE exercised passive-pinned S release'); for my $key (@positive_pcm_lifecycle_keys) diff --git a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c index 24ca2b2573..d7b60618e1 100644 --- a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c +++ b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c @@ -370,7 +370,8 @@ bool cluster_gcs_send_block_request_and_wait(struct BufferDesc *buf pg_attribute_unused(), PcmLockTransition trans pg_attribute_unused(), int master_node pg_attribute_unused(), - bool clean_eligible pg_attribute_unused()) + bool clean_eligible pg_attribute_unused(), + bool *out_retry_denied pg_attribute_unused()) { abort(); } diff --git a/src/test/cluster_unit/test_cluster_gcs_block_3way.c b/src/test/cluster_unit/test_cluster_gcs_block_3way.c index b929b77267..47b4e2f7e2 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_3way.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_3way.c @@ -292,8 +292,8 @@ UT_TEST(test_placeholder_tap_114_2node_x_transfer) * S→X upgrade with bitmap → broadcast invalidate / X→X holder * direct ship via HC115 forward / invalidate ack timeout 53R91 / * dirty buffer XLogFlush-before-ack (HC123) / epoch validation / - * S barrier DENIED_PENDING_X reply / reader backoff retry → - * eventual grant / starvation 53R92 budget exhaustion. */ + * S barrier DENIED_PENDING_X reply / exact reservation abort / + * fresh-identity reader backoff retry → eventual grant. */ UT_ASSERT(true); } diff --git a/src/test/cluster_unit/test_cluster_gcs_block_dedup.c b/src/test/cluster_unit/test_cluster_gcs_block_dedup.c index ee48f05f3e..2289fe4f38 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_dedup.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_dedup.c @@ -1752,10 +1752,195 @@ UT_TEST(u32_pcm_x_staged_marker_rolls_back_exactly) (int)GCS_BLOCK_PCM_X_IMAGE_DUPLICATE); } + +/* Shape-B: a legacy S request may already be registered or forwarded before + * the PCM-X queue publishes its pending-X claim. The queue arm must revoke + * every still-live grant/forward right under the dedup lock, cache an exact + * retry denial for loss/replay, and fence a late producer from restoring a + * GRANTED reply. DONE stops periodic replay; a new request identity remains + * a normal MISS after the X round. */ +UT_TEST(u33_pending_x_arm_terminates_inflight_legacy_s_exactly) +{ + BufferTag tag = make_tag(130); + BufferTag other_tag = make_tag(131); + GcsBlockDedupKey inflight = make_key(1, 3, 3001, 17); + GcsBlockDedupKey forwarded = make_key(2, 4, 3002, 17); + GcsBlockDedupKey unrelated = make_key(3, 5, 3003, 17); + GcsBlockDedupKey writer = make_key(1, 6, 3004, 17); + GcsBlockDedupKey fresh = make_key(1, 3, 4001, 17); + GcsBlockReplyHeader forward_hdr; + GcsBlockReplyHeader late_granted; + GcsBlockDedupEntry cached; + GcsBlockDedupEntry denied; + GcsBlockDedupKey denied_keys[2]; + char page[GCS_BLOCK_DATA_SIZE]; + int result; + int i; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + memset(&forward_hdr, 0, sizeof(forward_hdr)); + memset(&late_granted, 0, sizeof(late_granted)); + memset(page, 0x6d, sizeof(page)); + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &inflight, tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &forwarded, tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); + forward_hdr.request_id = forwarded.request_id; + forward_hdr.epoch = forwarded.cluster_epoch; + forward_hdr.sender_node = 3; + forward_hdr.requester_backend_id = forwarded.requester_backend_id; + forward_hdr.transition_id = (uint8)PCM_TRANS_N_TO_S; + forward_hdr.status = (uint8)GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER; + GcsBlockReplyHeaderSetForwardingMasterNode(&forward_hdr, cluster_node_id); + cluster_gcs_block_dedup_install_reply(0, &forwarded, + GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER, &forward_hdr, NULL); + + /* Different tag and same-tag writer identities are not legacy-S victims. */ + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &unrelated, other_tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &writer, tag, PCM_TRANS_N_TO_X, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); + + for (i = 0; i < 2; i++) { + memset(&denied, 0, sizeof(denied)); + result = cluster_gcs_block_dedup_pending_x_deny_next(0, &tag, &denied); + UT_ASSERT_EQ(result, GCS_BLOCK_PENDING_X_DENY_NEW); + UT_ASSERT_EQ((int)denied.entry_kind, (int)GCS_BLOCK_DEDUP_ENTRY_GENERIC); + UT_ASSERT_EQ((int)denied.transition_id, (int)PCM_TRANS_N_TO_S); + UT_ASSERT_EQ((int)denied.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ((int)denied.reply_header.status, + (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ(denied.reply_header.request_id, denied.key.request_id); + UT_ASSERT_EQ(denied.reply_header.epoch, denied.key.cluster_epoch); + UT_ASSERT_EQ(denied.reply_header.requester_backend_id, + denied.key.requester_backend_id); + UT_ASSERT_EQ((int)denied.reply_header.transition_id, (int)PCM_TRANS_N_TO_S); + UT_ASSERT_EQ(denied.reply_header.sender_node, cluster_node_id); + UT_ASSERT_EQ(GcsBlockReplyHeaderGetForwardingMasterNode(&denied.reply_header), + GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + denied_keys[i] = denied.key; + } + UT_ASSERT(memcmp(&denied_keys[0], &denied_keys[1], sizeof(denied_keys[0])) != 0); + + /* Initial denial loss is recovered by periodic replay of the same exact + * identity, without minting another request or reviving FORWARD. */ + memset(&denied, 0, sizeof(denied)); + UT_ASSERT_EQ(cluster_gcs_block_dedup_pending_x_deny_next(0, &tag, &denied), + GCS_BLOCK_PENDING_X_DENY_REPLAY); + UT_ASSERT(memcmp(&denied.key, &denied_keys[0], sizeof(denied.key)) == 0 + || memcmp(&denied.key, &denied_keys[1], sizeof(denied.key)) == 0); + + /* An asynchronous old GRANTED producer has lost its right once the deny + * is cached; installing it must be a zero-mutation no-op. */ + late_granted.request_id = denied.key.request_id; + late_granted.epoch = denied.key.cluster_epoch; + late_granted.sender_node = cluster_node_id; + late_granted.requester_backend_id = denied.key.requester_backend_id; + late_granted.transition_id = (uint8)PCM_TRANS_N_TO_S; + late_granted.status = (uint8)GCS_BLOCK_REPLY_GRANTED; + cluster_gcs_block_dedup_install_reply(0, &denied.key, GCS_BLOCK_REPLY_GRANTED, + &late_granted, page); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &denied.key, tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_CACHED_REPLY); + UT_ASSERT_EQ((int)cached.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ((int)cached.reply_header.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + + /* Duplicate DONE is idempotent and removes both old identities from the + * denial replay set; unrelated entries remain in their original states. */ + for (i = 0; i < 2; i++) { + UT_ASSERT(cluster_gcs_block_dedup_mark_done(0, &denied_keys[i], &tag, + PCM_TRANS_N_TO_S)); + UT_ASSERT(cluster_gcs_block_dedup_mark_done(0, &denied_keys[i], &tag, + PCM_TRANS_N_TO_S)); + } + UT_ASSERT_EQ(cluster_gcs_block_dedup_pending_x_deny_next(0, &tag, &denied), + GCS_BLOCK_PENDING_X_DENY_NOT_FOUND); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &unrelated, other_tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_IN_FLIGHT_DUPLICATE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &writer, tag, PCM_TRANS_N_TO_X, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_IN_FLIGHT_DUPLICATE); + + /* Once the queue round is over, a reader with a fresh identity is admitted + * normally; the two denied identities cannot poison the new request. */ + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &fresh, tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); +} + +/* A reader arriving after the queue claim is registered only to make its + * denial reliable: exact arbitration replaces even a pre-existing cached + * grant before the handler can take a normal dedup shortcut. The original + * direct-land property remains attached to the cached denial for replay. */ +UT_TEST(u34_pending_x_new_reader_exact_deny_precedes_cached_shortcut) +{ + BufferTag tag = make_tag(132); + GcsBlockDedupKey key = make_key(2, 7, 5001, 19); + GcsBlockDedupKey absent = make_key(2, 7, 5002, 19); + GcsBlockReplyHeader granted; + GcsBlockDedupEntry cached; + GcsBlockDedupEntry denied; + char page[GCS_BLOCK_DATA_SIZE]; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + memset(&granted, 0, sizeof(granted)); + memset(page, 0x71, sizeof(page)); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &key, tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); + UT_ASSERT(cluster_gcs_block_dedup_set_request_flags_exact( + 0, &key, &tag, PCM_TRANS_N_TO_S, GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND)); + + granted.request_id = key.request_id; + granted.epoch = key.cluster_epoch; + granted.sender_node = cluster_node_id; + granted.requester_backend_id = key.requester_backend_id; + granted.transition_id = (uint8)PCM_TRANS_N_TO_S; + granted.status = (uint8)GCS_BLOCK_REPLY_GRANTED; + GcsBlockReplyHeaderSetForwardingMasterNode(&granted, + GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + cluster_gcs_block_dedup_install_reply(0, &key, GCS_BLOCK_REPLY_GRANTED, &granted, page); + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pending_x_deny_exact( + 0, &key, &tag, PCM_TRANS_N_TO_S, &denied), + (int)GCS_BLOCK_PENDING_X_DENY_NEW); + UT_ASSERT_EQ((int)denied.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ((int)denied.request_flags, + (int)(GCS_BLOCK_DEDUP_REQUEST_F_PINNED + | GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND)); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pending_x_deny_exact( + 0, &key, &tag, PCM_TRANS_N_TO_S, &denied), + (int)GCS_BLOCK_PENDING_X_DENY_REPLAY); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &key, tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_CACHED_REPLY); + UT_ASSERT_EQ((int)cached.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + + /* Missing and mismatched identities are zero-mutation invalid results. */ + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pending_x_deny_exact( + 0, &absent, &tag, PCM_TRANS_N_TO_S, &denied), + (int)GCS_BLOCK_PENDING_X_DENY_INVALID); + UT_ASSERT(!cluster_gcs_block_dedup_set_request_flags_exact( + 0, &key, &tag, PCM_TRANS_N_TO_X, GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND)); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &key, tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_CACHED_REPLY); + UT_ASSERT_EQ((int)cached.request_flags, + (int)(GCS_BLOCK_DEDUP_REQUEST_F_PINNED + | GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND)); +} + int main(void) { - UT_PLAN(32); + UT_PLAN(34); UT_RUN(u1_per_worker_isolation); UT_RUN(u2_dedup_lifecycle_per_shard); UT_RUN(u3_counters_sum_across_shards); @@ -1788,6 +1973,8 @@ main(void) UT_RUN(u30_pcm_x_owner_restart_audit_detects_and_retains_evidence); UT_RUN(u31_pcm_x_work_classes_alternate_when_both_remain_runnable); UT_RUN(u32_pcm_x_staged_marker_rolls_back_exactly); + UT_RUN(u33_pending_x_arm_terminates_inflight_legacy_s_exactly); + UT_RUN(u34_pending_x_new_reader_exact_deny_precedes_cached_shortcut); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_pcm_direct_init.c b/src/test/cluster_unit/test_cluster_pcm_direct_init.c index cfb8052722..d52a0ff80b 100644 --- a/src/test/cluster_unit/test_cluster_pcm_direct_init.c +++ b/src/test/cluster_unit/test_cluster_pcm_direct_init.c @@ -374,7 +374,7 @@ UT_TEST(test_valid_n_s_x_without_proof_enters_queue_before_legacy_wire) static const char *const order[] = { "pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode,", "cluster_bufmgr_pcm_begin_grant_reservation_wait(", - "cluster_pcm_lock_acquire_buffer(buf, pcm_mode)" }; + "cluster_pcm_lock_acquire_buffer(buf, pcm_mode," }; UT_ASSERT(source != NULL); if (source != NULL) { diff --git a/src/test/cluster_unit/test_cluster_pcm_lock.c b/src/test/cluster_unit/test_cluster_pcm_lock.c index 1e50902305..367c1aaebf 100644 --- a/src/test/cluster_unit/test_cluster_pcm_lock.c +++ b/src/test/cluster_unit/test_cluster_pcm_lock.c @@ -528,7 +528,8 @@ bool cluster_gcs_send_block_request_and_wait(struct BufferDesc *buf pg_attribute_unused(), PcmLockTransition trans pg_attribute_unused(), int master_node pg_attribute_unused(), - bool clean_eligible pg_attribute_unused()) + bool clean_eligible pg_attribute_unused(), + bool *out_retry_denied pg_attribute_unused()) { abort(); } @@ -1250,13 +1251,14 @@ UT_TEST(test_pcm_b_local_master_remote_x_holder_fail_closed) UT_TEST(test_pcm_d1_recovering_gate_fail_closed) { BufferDesc buf; + bool retry_denied = false; reset_fake_pcm_runtime(2); buf.tag = make_tag(77); fake_block_phase = GCS_BLOCK_RECOVERING; cluster_gcs_block_recovery_wait_ms = 0; /* immediate fail-closed, no sleep */ - UT_EXPECT_EREPORT(cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_S)); + UT_EXPECT_EREPORT(cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_S, &retry_denied)); fake_block_phase = GCS_BLOCK_NORMAL; cluster_gcs_block_recovery_wait_ms = 200; } @@ -1433,6 +1435,7 @@ UT_TEST(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades) BufferTag tag = make_tag(96); BufferDesc buf; bool save = cluster_gcs_block_local_cache; + bool retry_denied = false; reset_fake_pcm_runtime(4); fake_cssd_dead_node = -1; @@ -1447,7 +1450,8 @@ UT_TEST(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades) buf.tag = tag; buf.pcm_state = (uint8)PCM_STATE_N; cluster_node_id = 0; - UT_ASSERT(cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_X)); + UT_ASSERT(cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_X, &retry_denied)); + UT_ASSERT(!retry_denied); UT_ASSERT_EQ((int)cluster_pcm_lock_query(tag), (int)PCM_LOCK_MODE_X); UT_ASSERT(!cluster_pcm_master_other_live_holder_exists(tag, 0)); UT_ASSERT(cluster_pcm_master_requester_is_holder(tag, 0, PCM_TRANS_N_TO_X)); From f78f64344c5bd44df3858aa7cef5b12fa10b8ada Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 19 Jul 2026 04:45:37 -0700 Subject: [PATCH 48/56] fix(cluster): count PCM own lifecycle at the exact grant begin and X commit t/400 L3 asserts pcm_x_own_begin_count and pcm_x_own_commit_count advance during the four-writer workload, but production never called the stats API: both counters sat at 0 deterministically. Wire the begin counter at every successful GRANT_PENDING linearization in bufmgr.c -- the legacy grant reservation, the queue X reservation, the REVOKING->GRANT_PENDING handoff, and the direct-init gate -- always after UnlockBufHdr and only on CLUSTER_PCM_OWN_OK. The idempotent duplicate PREPARE that merely observes GRANT_PENDING stays uncounted, as do all BUSY/STALE/CORRUPT refusals, so replay and failure semantics stay honest. Wire the commit counter once, at the single exact-commit funnel shared by the queue finish, the LockBuffer finalize, and the direct-init grant, gated on the target state being X so an S-grant finish stays silent. Pin the wiring with a source-contract test in test_cluster_pcm_own (four begin sites, one commit funnel, counters outside the header lock). t/400 L3: own_begin/own_commit "advanced" flip red->green (begin=20 delta=9, commit=10 delta=3); own_abort/own_corrupt deltas stay 0. --- src/backend/storage/buffer/bufmgr.c | 25 ++++++++- src/test/cluster_unit/test_cluster_pcm_own.c | 57 +++++++++++++++++++- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 8a218685d9..d4299b3145 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -754,6 +754,10 @@ cluster_pcm_own_begin_grant_reservation(BufferDesc *buf, ClusterPcmOwnSnapshot * result = cluster_pcm_own_reservation_begin_exact(buf->buf_id, out_base->generation, PCM_OWN_FLAG_GRANT_PENDING, out_token); UnlockBufHdr(buf, buf_state); + /* Exact begin allocated a fresh token under header authority; a replayed + * or refused begin reports BUSY/STALE and never reaches here. */ + if (result == CLUSTER_PCM_OWN_OK) + cluster_pcm_x_stats_note_own_begin(); return result; } @@ -779,6 +783,8 @@ cluster_bufmgr_pcm_own_begin_x_reservation(BufferDesc *buf, const ClusterPcmOwnS result = cluster_pcm_own_reservation_begin_exact(buf->buf_id, expected->generation, PCM_OWN_FLAG_GRANT_PENDING, out_token); UnlockBufHdr(buf, buf_state); + if (result == CLUSTER_PCM_OWN_OK) + cluster_pcm_x_stats_note_own_begin(); return result; } @@ -801,6 +807,7 @@ cluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation( bool source_is_n; bool source_is_s; bool source_is_x; + bool handoff_transitioned = false; if (out_token != NULL) *out_token = 0; @@ -848,11 +855,21 @@ cluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation( else if (flags != PCM_OWN_FLAG_REVOKING) result = CLUSTER_PCM_OWN_BUSY; else + { result = cluster_pcm_own_revoke_to_grant_handoff_exact( buf->buf_id, expected_revoking->generation, expected_revoking->reservation_token); + handoff_transitioned = result == CLUSTER_PCM_OWN_OK; + } UnlockBufHdr(buf, buf_state); if (result == CLUSTER_PCM_OWN_OK) + { + /* Count only the REVOKING->GRANT_PENDING linearization; the exact + * duplicate PREPARE that already observes GRANT_PENDING is an + * idempotent replay and must not advance the begin counter. */ + if (handoff_transitioned) + cluster_pcm_x_stats_note_own_begin(); *out_token = expected_revoking->reservation_token; + } return result; } @@ -930,6 +947,10 @@ cluster_pcm_own_finish_grant_reservation(BufferDesc *buf, const ClusterPcmOwnSna } } UnlockBufHdr(buf, buf_state); + /* Every X ownership commit funnels through this exact commit; the S-grant + * finish and every STALE/INVALID/CORRUPT refusal stay uncounted. */ + if (result == CLUSTER_PCM_OWN_OK && new_pcm_state == (uint8)PCM_STATE_X) + cluster_pcm_x_stats_note_own_commit(); return result; } @@ -2635,7 +2656,9 @@ cluster_bufmgr_pcm_gate_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind ki } UnlockBufHdr(buf, buf_state); - if (pending_result != CLUSTER_PCM_OWN_OK) + if (pending_result == CLUSTER_PCM_OWN_OK) + cluster_pcm_x_stats_note_own_begin(); + else cluster_bufmgr_pcm_direct_init_report_failure(buf, pending_result, &observed, "direct-init consume"); diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index 04c5b05dd8..443ecd31fc 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -2242,10 +2242,64 @@ UT_TEST(test_preflight_busy_waits_then_clean_resnapshot_begins_reservation) UT_ASSERT(!cluster_gcs_pcm_x_requester_runtime_exact(&start, ¤t)); } +UT_TEST(test_own_lifecycle_counters_land_on_exact_begin_and_x_commit) +{ + static const char *const exact_begin_contract[] + = { "cluster_pcm_own_reservation_begin_exact", "PCM_OWN_FLAG_GRANT_PENDING", + "UnlockBufHdr", "result == CLUSTER_PCM_OWN_OK", + "cluster_pcm_x_stats_note_own_begin();" }; + static const char *const handoff_begin_contract[] + = { "flags == PCM_OWN_FLAG_GRANT_PENDING", "result = CLUSTER_PCM_OWN_OK", + "cluster_pcm_own_revoke_to_grant_handoff_exact", + "handoff_transitioned = result == CLUSTER_PCM_OWN_OK", "UnlockBufHdr", + "if (handoff_transitioned)", "cluster_pcm_x_stats_note_own_begin();" }; + static const char *const direct_init_begin_contract[] + = { "cluster_pcm_own_reservation_begin_exact", "PCM_OWN_FLAG_GRANT_PENDING", + "UnlockBufHdr", "pending_result == CLUSTER_PCM_OWN_OK", + "cluster_pcm_x_stats_note_own_begin();" }; + static const char *const x_commit_contract[] + = { "cluster_pcm_own_grant_commit_exact", "UnlockBufHdr", + "result == CLUSTER_PCM_OWN_OK && new_pcm_state == (uint8)PCM_STATE_X", + "cluster_pcm_x_stats_note_own_commit();" }; + char *source = read_bufmgr_source(); + + /* t/400 own-lifecycle counters: every counted begin sits after the header + * spinlock proved an exact GRANT_PENDING linearization (a fresh token, a + * REVOKING->GRANT_PENDING handoff, or a consumed direct-init proof), and + * a replayed/refused begin never counts. The idempotent duplicate + * PREPARE that merely observes GRANT_PENDING must stay uncounted, so the + * handoff increments only behind its transition evidence. */ + assert_ordered_in_function(source, "\ncluster_pcm_own_begin_grant_reservation(", + "\ncluster_bufmgr_pcm_own_begin_x_reservation(", + exact_begin_contract, lengthof(exact_begin_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_begin_x_reservation(", + "\ncluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation(", + exact_begin_contract, lengthof(exact_begin_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation(", + "\ncluster_bufmgr_pcm_own_handoff_s_revoke_to_x_reservation(", + handoff_begin_contract, lengthof(handoff_begin_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_gate_direct_init(", + "#define BUF_DROP_FULL_SCAN_THRESHOLD", direct_init_begin_contract, + lengthof(direct_init_begin_contract)); + + /* The X ownership commit counts exactly once, at the single exact-commit + * funnel shared by the queue finish, the LockBuffer finalize, and the + * direct-init grant; an S-grant finish and every refusal stay silent. */ + assert_ordered_in_function(source, "\ncluster_pcm_own_finish_grant_reservation(", + "\ncluster_bufmgr_pcm_own_finish_x_commit(", x_commit_contract, + lengthof(x_commit_contract)); + + /* Pin the production wiring against silent duplication or removal: four + * begin sites, one commit funnel, no counter call under LockBufHdr. */ + UT_ASSERT_EQ(count_occurrences(source, "cluster_pcm_x_stats_note_own_begin();"), 4); + UT_ASSERT_EQ(count_occurrences(source, "cluster_pcm_x_stats_note_own_commit();"), 1); + free(source); +} + int main(void) { - UT_PLAN(50); + UT_PLAN(51); UT_RUN(test_shmem_initializes_complete_entry); UT_RUN(test_begin_abort_is_exact_and_monotonic); UT_RUN(test_invalid_live_flag_shapes_are_corrupt_not_busy); @@ -2296,6 +2350,7 @@ main(void) UT_RUN(test_queue_writer_grant_snapshot_is_claim_and_generation_exact); UT_RUN(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_authority); UT_RUN(test_preflight_busy_waits_then_clean_resnapshot_begins_reservation); + UT_RUN(test_own_lifecycle_counters_land_on_exact_begin_and_x_commit); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From 94fe492c57527b7749399fbe80ccd39733cffcb2 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Mon, 20 Jul 2026 13:36:51 +0800 Subject: [PATCH 49/56] fix(cluster): retry post-handoff COMMIT_X --- src/backend/cluster/cluster_gcs_block.c | 18 ++ src/backend/cluster/cluster_pcm_x_convert.c | 118 ++++++++++ src/include/cluster/cluster_pcm_x_convert.h | 10 + .../cluster_unit/test_cluster_pcm_x_convert.c | 206 +++++++++++++++++- 4 files changed, 351 insertions(+), 1 deletion(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 0544a82c27..ce6e702532 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -9776,11 +9776,23 @@ gcs_block_pcm_x_stage_queue_invalidations(const PcmXMasterDriveSnapshot *snapsho } +static bool +gcs_block_pcm_x_stage_frame_callback(uint8 msg_type, int32 dest_node_id, const void *payload, + Size payload_len, void *callback_arg) +{ + (void)callback_arg; + return cluster_gcs_pcm_x_stage_frame(msg_type, dest_node_id, payload, payload_len); +} + + static PcmXQueueResult gcs_block_pcm_x_master_drive_transfer(const PcmXMasterDriveSnapshot *snapshot) { + PcmXMasterDriveSnapshot retried; PcmXRevokePayload revoke; PcmXQueueResult result; + uint64 now_ms; + uint64 retry_delay_ms; uint64 source_session = 0; uint32 source_bit; uint32 unacked; @@ -9788,6 +9800,12 @@ gcs_block_pcm_x_master_drive_transfer(const PcmXMasterDriveSnapshot *snapshot) if (snapshot == NULL || snapshot->ticket_state != PCM_XT_ACTIVE_TRANSFER) return PCM_X_QUEUE_INVALID; + if (snapshot->pending_opcode == PGRAC_IC_MSG_PCM_X_COMMIT_X) { + now_ms = (uint64)(GetCurrentTimestamp() / 1000); + retry_delay_ms = (uint64)Max(cluster_lmon_main_loop_interval, 1); + return cluster_pcm_x_master_commit_retry_drive( + snapshot, now_ms, retry_delay_ms, gcs_block_pcm_x_stage_frame_callback, NULL, &retried); + } if (!cluster_gcs_pcm_x_transfer_pre_handoff_phase(snapshot->pending_opcode)) return PCM_X_QUEUE_NOT_READY; if (!cluster_pcm_lock_queue_pending_x_exact(snapshot->ref.identity.tag, diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index bcd0f5782a..1d47ef5023 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -104,6 +104,10 @@ static bool pcm_x_stats_depth_increment(PcmXShmemHeader *header); static bool pcm_x_stats_depth_decrement(PcmXShmemHeader *header); static bool pcm_x_master_terminal_leg_is_clear(const PcmXReliableLegState *leg); static bool pcm_x_master_blocker_stage_metadata_valid(const PcmXMasterTicketSlot *ticket); +static bool pcm_x_transfer_leg_exact(const PcmXReliableLegState *leg, uint16 opcode, + int32 responder_node, uint64 responder_session); +static bool pcm_x_transfer_peer_exact(const PcmXMasterTicketSlot *ticket, int32 responder_node, + uint64 responder_session); static void pcm_x_master_blocker_snapshot_clear(PcmXMasterBlockerSnapshot *snapshot); static bool pcm_x_ticket_ref_is_zero(const PcmXTicketRef *ref); static bool pcm_x_local_terminal_ref_valid(const PcmXTicketRef *ref); @@ -4895,6 +4899,120 @@ cluster_pcm_x_master_drive_snapshot_exact(const BufferTag *tag, uint64 cluster_e } +PcmXQueueResult +cluster_pcm_x_master_commit_retry_exact(const PcmXMasterDriveSnapshot *expected, uint64 now_ms, + uint64 next_retry_deadline_ms, PcmXPhasePayload *commit_out, + PcmXMasterDriveSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXMasterDriveSnapshot current; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 partition; + + if (commit_out != NULL) + memset(commit_out, 0, sizeof(*commit_out)); + pcm_x_master_drive_snapshot_clear(snapshot_out); + if (header == NULL || expected == NULL || commit_out == NULL || snapshot_out == NULL + || next_retry_deadline_ms <= now_ms || expected->ticket_state != PCM_XT_ACTIVE_TRANSFER + || expected->pending_opcode != PGRAC_IC_MSG_PCM_X_COMMIT_X + || expected->phase != PGRAC_IC_MSG_PCM_X_COMMIT_X || expected->reserved != 0 + || expected->expected_responder_node != expected->ref.identity.node_id + || expected->expected_responder_session == 0 + || !pcm_x_wait_identity_valid(&expected->ref.identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(&expected->ref, &ticket_ref, &ticket); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&expected->ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TAG, tag_ref, + &expected->ref.identity.tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &expected->ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + result = pcm_x_master_drive_capture_locked(runtime, tag_slot, tag_ref, ticket, ticket_ref, + ¤t); + if (result != PCM_X_QUEUE_OK) + goto retry_done; + if (!pcm_x_master_drive_snapshot_equal(¤t, expected)) { + result = PCM_X_QUEUE_STALE; + goto retry_done; + } + if (!pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_COMMIT_X, + ticket->ref.identity.node_id, + ticket->reliable.expected_responder_session) + || !pcm_x_transfer_peer_exact(ticket, ticket->ref.identity.node_id, + ticket->reliable.expected_responder_session)) { + result = PCM_X_QUEUE_STALE; + goto retry_done; + } + if (ticket->reliable.retry_deadline_ms != 0 && now_ms < ticket->reliable.retry_deadline_ms) { + result = PCM_X_QUEUE_NOT_READY; + goto retry_done; + } + + if (ticket->reliable.retry_count != PG_UINT32_MAX) + ticket->reliable.retry_count++; + ticket->reliable.retry_deadline_ms = next_retry_deadline_ms; + pg_write_barrier(); + result = pcm_x_master_drive_capture_locked(runtime, tag_slot, tag_ref, ticket, ticket_ref, + snapshot_out); + if (result == PCM_X_QUEUE_OK) { + commit_out->ref = ticket->ref; + commit_out->phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; + } + +retry_done: + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_commit_retry_drive(const PcmXMasterDriveSnapshot *expected, uint64 now_ms, + uint64 retry_delay_ms, PcmXStageFrameCallback stage_frame, + void *callback_arg, PcmXMasterDriveSnapshot *snapshot_out) +{ + PcmXPhasePayload commit; + PcmXQueueResult result; + uint64 next_retry_deadline_ms; + + pcm_x_master_drive_snapshot_clear(snapshot_out); + if (expected == NULL || stage_frame == NULL || snapshot_out == NULL || retry_delay_ms == 0 + || now_ms == UINT64_MAX) + return PCM_X_QUEUE_INVALID; + next_retry_deadline_ms + = now_ms > UINT64_MAX - retry_delay_ms ? UINT64_MAX : now_ms + retry_delay_ms; + result = cluster_pcm_x_master_commit_retry_exact(expected, now_ms, next_retry_deadline_ms, + &commit, snapshot_out); + if (result != PCM_X_QUEUE_OK) + return result; + return stage_frame(PGRAC_IC_MSG_PCM_X_COMMIT_X, commit.ref.identity.node_id, &commit, + sizeof(commit), callback_arg) + ? PCM_X_QUEUE_OK + : PCM_X_QUEUE_BUSY; +} + + PcmXQueueResult cluster_pcm_x_master_drive_bitmap_replace_exact(const PcmXMasterDriveSnapshot *expected, uint32 pending_s_holders_bitmap, diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h index 0525036554..8c24baa055 100644 --- a/src/include/cluster/cluster_pcm_x_convert.h +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -1420,6 +1420,16 @@ extern PcmXQueueResult cluster_pcm_x_master_drive_work_next(Size *cursor_io, Siz extern PcmXQueueResult cluster_pcm_x_master_drive_snapshot_exact(const BufferTag *tag, uint64 cluster_epoch, PcmXMasterDriveSnapshot *snapshot_out); +extern PcmXQueueResult +cluster_pcm_x_master_commit_retry_exact(const PcmXMasterDriveSnapshot *expected, uint64 now_ms, + uint64 next_retry_deadline_ms, PcmXPhasePayload *commit_out, + PcmXMasterDriveSnapshot *snapshot_out); +typedef bool (*PcmXStageFrameCallback)(uint8 msg_type, int32 dest_node_id, const void *payload, + Size payload_len, void *callback_arg); +extern PcmXQueueResult +cluster_pcm_x_master_commit_retry_drive(const PcmXMasterDriveSnapshot *expected, uint64 now_ms, + uint64 retry_delay_ms, PcmXStageFrameCallback stage_frame, + void *callback_arg, PcmXMasterDriveSnapshot *snapshot_out); extern PcmXQueueResult cluster_pcm_x_master_drive_bitmap_replace_exact( const PcmXMasterDriveSnapshot *expected, uint32 pending_s_holders_bitmap, uint32 acked_s_holders_bitmap, PcmXMasterDriveSnapshot *snapshot_out); diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 152c06c282..27221f32ee 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -6432,6 +6432,209 @@ drive_master_ticket_to_prepared(PcmXMasterAdmission *admission, uint64 source_se *image_id_out = revoke.image_id; } +typedef struct CommitRetryStageCapture { + int calls; + bool accept; + uint8 msg_type; + int32 dest_node_id; + Size payload_len; + PcmXPhasePayload payload; +} CommitRetryStageCapture; + +static bool +capture_commit_retry_stage(uint8 msg_type, int32 dest_node_id, const void *payload, + Size payload_len, void *callback_arg) +{ + CommitRetryStageCapture *capture = (CommitRetryStageCapture *)callback_arg; + + capture->calls++; + capture->msg_type = msg_type; + capture->dest_node_id = dest_node_id; + capture->payload_len = payload_len; + memset(&capture->payload, 0, sizeof(capture->payload)); + if (payload != NULL && payload_len == sizeof(capture->payload)) + memcpy(&capture->payload, payload, sizeof(capture->payload)); + return capture->accept; +} + +/* + * P0-14: the master has crossed the handoff boundary and durably armed type + * 53, but the first COMMIT_X frame is dropped. The real bounded master-work + * scan must still select the ACTIVE_TRANSFER ticket; an exact deadline retry + * then reconstructs the same logical leg without minting a ticket or moving + * authority. FINAL_ACK remains the only exact successor that consumes it. + */ +UT_TEST(test_master_periodic_drive_replays_post_handoff_commit_x_exactly) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterDriveSnapshot armed; + PcmXMasterDriveSnapshot retried; + PcmXMasterDriveSnapshot retried_again; + PcmXMasterDriveSnapshot stale; + PcmXMasterTicketSlot before_blocked; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef transfer; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload first_commit; + PcmXPhasePayload retry_commit; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + PcmXFinalAckPayload final_ack; + PcmXMasterFinalAckToken final_ack_token; + PcmXQueueResult result; + CommitRetryStageCapture stage; + BufferTag drive_tag; + Size cursor; + uint64 drive_epoch; + uint64 image_id; + const uint64 requester_session = UINT64_C(7384); + const uint64 source_session = UINT64_C(8384); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe_with_base(813, 0, 23, UINT64_C(8384), requester_session, 1, UINT64_C(5), + &admission); + drive_master_ticket_to_prepared(&admission, source_session, UINT64_C(9384), &transfer, + &image_id); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + + memset(&install_ready, 0, sizeof(install_ready)); + install_ready.ref = transfer; + install_ready.image_id = image_id; + install_ready.result = PCM_X_QUEUE_OK; + install_ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, 0, 0, requester_session, + &first_commit), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(first_commit.phase, PGRAC_IC_MSG_PCM_X_COMMIT_X); + UT_ASSERT(ticket_refs_equal(&first_commit.ref, &transfer)); + UT_ASSERT_EQ(ticket->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_COMMIT_X); + UT_ASSERT_EQ(ticket->reliable.retry_count, 0); + UT_ASSERT_EQ(ticket->reliable.retry_deadline_ms, 0); + + /* Drop first_commit: no requester delivery and therefore no FINAL_ACK. */ + cursor = admission.tag_slot.slot_index; + UT_ASSERT_EQ(cluster_pcm_x_master_drive_work_next(&cursor, 1, &drive_tag, &drive_epoch), + PCM_X_QUEUE_OK); + UT_ASSERT(BufferTagsEqual(&drive_tag, &transfer.identity.tag)); + UT_ASSERT_EQ(drive_epoch, transfer.identity.cluster_epoch); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact(&drive_tag, drive_epoch, &armed), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(armed.ticket_state, PCM_XT_ACTIVE_TRANSFER); + UT_ASSERT_EQ(armed.pending_opcode, PGRAC_IC_MSG_PCM_X_COMMIT_X); + UT_ASSERT_EQ(armed.expected_responder_session, requester_session); + + memset(&stage, 0, sizeof(stage)); + stage.accept = true; + result = cluster_pcm_x_master_commit_retry_drive(&armed, UINT64_C(1000), UINT64_C(100), + capture_commit_retry_stage, &stage, &retried); + UT_ASSERT_EQ(result, PCM_X_QUEUE_OK); + if (result != PCM_X_QUEUE_OK) + return; + UT_ASSERT_EQ(stage.calls, 1); + UT_ASSERT_EQ(stage.msg_type, PGRAC_IC_MSG_PCM_X_COMMIT_X); + UT_ASSERT_EQ(stage.dest_node_id, transfer.identity.node_id); + UT_ASSERT_EQ(stage.payload_len, sizeof(PcmXPhasePayload)); + UT_ASSERT_EQ(stage.payload.phase, PGRAC_IC_MSG_PCM_X_COMMIT_X); + UT_ASSERT(ticket_refs_equal(&stage.payload.ref, &first_commit.ref)); + UT_ASSERT_EQ(retried.state_sequence, armed.state_sequence); + UT_ASSERT_EQ(retried.retry_count, 1); + UT_ASSERT_EQ(retried.retry_deadline_ms, UINT64_C(1100)); + UT_ASSERT_EQ(ticket->reliable.retry_count, 1); + + /* A periodic pass before the deadline is a byte-stable no-op. */ + memset(&retried_again, 0xA5, sizeof(retried_again)); + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_drive(&retried, UINT64_C(1099), UINT64_C(100), + capture_commit_retry_stage, &stage, + &retried_again), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(&retried_again, &(PcmXMasterDriveSnapshot){ 0 }, sizeof(retried_again)) == 0); + UT_ASSERT_EQ(stage.calls, 1); + UT_ASSERT_EQ(ticket->reliable.retry_count, 1); + UT_ASSERT_EQ(ticket->reliable.retry_deadline_ms, UINT64_C(1100)); + + /* The pre-retry token and a forged responder session cannot mutate it. */ + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_exact(&armed, UINT64_C(1100), UINT64_C(1200), + &retry_commit, &retried_again), + PCM_X_QUEUE_STALE); + stale = retried; + stale.expected_responder_session++; + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_exact(&stale, UINT64_C(1100), UINT64_C(1200), + &retry_commit, &retried_again), + PCM_X_QUEUE_STALE); + + /* A full outbound ring still advances the deadline and cannot busy-spin. */ + stage.accept = false; + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_drive(&retried, UINT64_C(1100), UINT64_C(100), + capture_commit_retry_stage, &stage, + &retried_again), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(stage.calls, 2); + UT_ASSERT(ticket_refs_equal(&stage.payload.ref, &transfer)); + UT_ASSERT_EQ(retried_again.state_sequence, armed.state_sequence); + UT_ASSERT_EQ(retried_again.retry_count, 2); + UT_ASSERT_EQ(retried_again.retry_deadline_ms, UINT64_C(1200)); + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_drive(&retried_again, UINT64_C(1100), + UINT64_C(100), capture_commit_retry_stage, + &stage, &stale), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(stage.calls, 2); + + /* Deadline expiry stages the same logical leg and advances scheduling. */ + stage.accept = true; + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_drive(&retried_again, UINT64_C(1200), + UINT64_C(100), capture_commit_retry_stage, + &stage, &stale), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(stage.calls, 3); + UT_ASSERT(ticket_refs_equal(&stage.payload.ref, &transfer)); + UT_ASSERT_EQ(stale.state_sequence, armed.state_sequence); + UT_ASSERT_EQ(stale.retry_count, 3); + UT_ASSERT_EQ(stale.retry_deadline_ms, UINT64_C(1300)); + + /* Old-session ACK is refused; exact FINAL_ACK advances and replays idempotently. */ + memset(&final_ack, 0, sizeof(final_ack)); + final_ack.ref = transfer; + final_ack.image_id = image_id; + final_ack.committed_own_generation = UINT64_C(6); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session + 1, + &final_ack_token), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(ticket->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_COMMIT_X); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session, + &final_ack_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_finalize_exact(&final_ack_token, &final_commit), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session, + &final_ack_token), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_finalize_exact(&final_ack_token, &final_commit), + PCM_X_QUEUE_DUPLICATE); + + memset(&final_confirm, 0, sizeof(final_confirm)); + final_confirm.ref = transfer; + final_confirm.phase = PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM; + UT_ASSERT_EQ(cluster_pcm_x_master_final_confirm_exact(&final_confirm, 0, requester_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_COMPLETE); + UT_ASSERT_EQ(ticket->reliable.pending_opcode, 0); + UT_ASSERT_EQ(ticket->reliable.retry_count, 0); + UT_ASSERT_EQ(ticket->reliable.retry_deadline_ms, 0); + + /* A non-ACTIVE runtime never resurrects or replays the consumed leg. */ + before_blocked = *ticket; + UT_ASSERT( + cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, PCM_X_RUNTIME_RECOVERY_BLOCKED)); + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_drive(&stale, UINT64_C(1300), UINT64_C(100), + capture_commit_retry_stage, &stage, + &armed), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(stage.calls, 3); + UT_ASSERT(memcmp(ticket, &before_blocked, sizeof(*ticket)) == 0); +} + /* * A' rebase (t/400 item 2): an interleaved revoke on the requester node * consumes the enqueue-time base_own_generation while the request is queued. @@ -15932,7 +16135,7 @@ UT_TEST(test_local_retire_episode_lock_errors_fail_closed) int main(void) { - UT_PLAN(269); + UT_PLAN(270); UT_RUN(test_image_id_domain_is_canonical_and_bounded); UT_RUN(test_wire_abi_sizes_are_exact); UT_RUN(test_wire_abi_offsets_are_exact); @@ -16052,6 +16255,7 @@ main(void) UT_RUN(test_master_drive_bitmap_replace_rejects_reliable_leg_drift); UT_RUN(test_master_drive_snapshot_fails_closed_on_bitmap_corruption); UT_RUN(test_master_transfer_wire_49_56_is_generation_exact); + UT_RUN(test_master_periodic_drive_replays_post_handoff_commit_x_exactly); UT_RUN(test_master_install_ready_rebase_grant_base_is_one_shot); UT_RUN(test_master_install_ready_rebase_conflict_fails_closed); UT_RUN(test_master_install_ready_no_drift_keeps_identity_base_math); From e56e2f1663a33ad471c4d6a96e6372f9b181dc76 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Mon, 20 Jul 2026 14:24:01 +0800 Subject: [PATCH 50/56] fix(cluster): count exact PCM-X requester waits --- src/backend/cluster/cluster_gcs_block.c | 48 ++++++++++++- src/backend/cluster/cluster_pcm_x_convert.c | 12 ++++ src/include/cluster/cluster_pcm_x_convert.h | 4 ++ .../cluster_unit/test_cluster_gcs_block.c | 2 +- .../cluster_unit/test_cluster_pcm_x_convert.c | 71 ++++++++++++++++++- 5 files changed, 134 insertions(+), 3 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index ce6e702532..4a438908ab 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -8277,6 +8277,33 @@ gcs_block_pcm_x_requester_wait(uint32 *wait_index) CHECK_FOR_INTERRUPTS(); } +typedef struct GcsBlockPcmXRequesterWaitContext { + const PcmXRuntimeSnapshot *request_runtime; + int32 master_node; + uint64 cluster_epoch; + uint64 master_session; + long timeout_ms; +} GcsBlockPcmXRequesterWaitContext; + +static bool +gcs_block_pcm_x_requester_pre_sleep_revalidate(void *callback_arg) +{ + GcsBlockPcmXRequesterWaitContext *context = (GcsBlockPcmXRequesterWaitContext *)callback_arg; + + return gcs_block_pcm_x_requester_authority_exact(context->request_runtime, context->master_node, + context->cluster_epoch, + context->master_session); +} + +static void +gcs_block_pcm_x_requester_wait_latch(void *callback_arg) +{ + GcsBlockPcmXRequesterWaitContext *context = (GcsBlockPcmXRequesterWaitContext *)callback_arg; + + (void)WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, context->timeout_ms, + WAIT_EVENT_PCM_BLOCK_CONVERT_WAIT); +} + /* Retry only inside the exact formation that admitted this requester. Both * sides of the sleep are checked: a runtime fence or peer-session change that * lands while the latch is asleep must be observed before the next protocol @@ -8285,10 +8312,29 @@ static bool gcs_block_pcm_x_requester_wait_exact(uint32 *wait_index, const PcmXRuntimeSnapshot *request_runtime, int32 master_node, uint64 cluster_epoch, uint64 master_session) { + GcsBlockPcmXRequesterWaitContext context; + uint32 current; + + if (wait_index == NULL) + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cluster PCM-X requester wait has no backoff state"))); if (!gcs_block_pcm_x_requester_authority_exact(request_runtime, master_node, cluster_epoch, master_session)) return false; - gcs_block_pcm_x_requester_wait(wait_index); + current = *wait_index; + memset(&context, 0, sizeof(context)); + context.request_runtime = request_runtime; + context.master_node = master_node; + context.cluster_epoch = cluster_epoch; + context.master_session = master_session; + context.timeout_ms = cluster_pcm_x_holder_retry_delay_ms(current); + CHECK_FOR_INTERRUPTS(); + if (!cluster_pcm_x_requester_wait_once(gcs_block_pcm_x_requester_pre_sleep_revalidate, + gcs_block_pcm_x_requester_wait_latch, &context)) + return false; + ResetLatch(MyLatch); + *wait_index = cluster_gcs_pcm_x_requester_wait_index_advance(current); + CHECK_FOR_INTERRUPTS(); return gcs_block_pcm_x_requester_authority_exact(request_runtime, master_node, cluster_epoch, master_session); } diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index 1d47ef5023..02d1dc8e94 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -986,6 +986,18 @@ cluster_pcm_x_stats_note_wait(void) } +bool +cluster_pcm_x_requester_wait_once(PcmXPreSleepRevalidateCallback revalidate, PcmXWaitCallback wait, + void *callback_arg) +{ + if (revalidate == NULL || wait == NULL || !revalidate(callback_arg)) + return false; + cluster_pcm_x_stats_note_wait(); + wait(callback_arg); + return true; +} + + void cluster_pcm_x_stats_note_queue_result(PcmXQueueResult result) { diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h index 8c24baa055..d42e02c0e0 100644 --- a/src/include/cluster/cluster_pcm_x_convert.h +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -1342,6 +1342,10 @@ extern PcmXRuntimeSnapshot cluster_pcm_x_runtime_snapshot(void); extern bool cluster_pcm_x_stats_snapshot(PcmXStatsSnapshot *snapshot_out); extern void cluster_pcm_x_stats_note_enqueue(void); extern void cluster_pcm_x_stats_note_wait(void); +typedef bool (*PcmXPreSleepRevalidateCallback)(void *callback_arg); +typedef void (*PcmXWaitCallback)(void *callback_arg); +extern bool cluster_pcm_x_requester_wait_once(PcmXPreSleepRevalidateCallback revalidate, + PcmXWaitCallback wait, void *callback_arg); extern void cluster_pcm_x_stats_note_queue_result(PcmXQueueResult result); extern void cluster_pcm_x_stats_note_own_begin(void); extern void cluster_pcm_x_stats_note_own_commit(void); diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 8b0b14199b..f2d5883f0f 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -2859,7 +2859,7 @@ UT_TEST(test_pcm_x_requester_driver_owns_fifo_and_transfer_lifecycles) : NULL; wait_exact_physical = wait_exact_first_authority != NULL - ? strstr(wait_exact_first_authority, "gcs_block_pcm_x_requester_wait(wait_index)") + ? strstr(wait_exact_first_authority, "cluster_pcm_x_requester_wait_once(") : NULL; wait_exact_second_authority = wait_exact_physical != NULL diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 27221f32ee..1e005e2e0c 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -15683,6 +15683,74 @@ UT_TEST(test_stats_initialize_zero_and_narrow_note_apis_are_exact) } +typedef struct RequesterWaitCapture { + bool revalidate_ok; + int revalidate_calls; + int wait_calls; + uint64 count_seen_at_wait; +} RequesterWaitCapture; + +static bool +requester_wait_revalidate(void *callback_arg) +{ + RequesterWaitCapture *capture = (RequesterWaitCapture *)callback_arg; + + capture->revalidate_calls++; + return capture->revalidate_ok; +} + +static void +requester_wait_block(void *callback_arg) +{ + RequesterWaitCapture *capture = (RequesterWaitCapture *)callback_arg; + PcmXStatsSnapshot stats; + + capture->wait_calls++; + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + capture->count_seen_at_wait = stats.wait_count; +} + +UT_TEST(test_queue_wait_count_tracks_only_linearized_requester_waits) +{ + RequesterWaitCapture capture = { 0 }; + PcmXStatsSnapshot stats; + + init_active_pcm_x(UINT64_C(81)); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.wait_count, 0); + + /* Fast-path and polling iterations never enter the wait producer. */ + UT_ASSERT_EQ(capture.revalidate_calls, 0); + UT_ASSERT_EQ(capture.wait_calls, 0); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.wait_count, 0); + + /* A failed pre-sleep authority check cannot bump or invoke the wait. */ + UT_ASSERT(!cluster_pcm_x_requester_wait_once(requester_wait_revalidate, requester_wait_block, + &capture)); + UT_ASSERT_EQ(capture.revalidate_calls, 1); + UT_ASSERT_EQ(capture.wait_calls, 0); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.wait_count, 0); + + /* Each actual wait callback observes its own count already linearized. */ + capture.revalidate_ok = true; + UT_ASSERT(cluster_pcm_x_requester_wait_once(requester_wait_revalidate, requester_wait_block, + &capture)); + UT_ASSERT_EQ(capture.wait_calls, 1); + UT_ASSERT_EQ(capture.count_seen_at_wait, 1); + UT_ASSERT(cluster_pcm_x_requester_wait_once(requester_wait_revalidate, requester_wait_block, + &capture)); + UT_ASSERT(cluster_pcm_x_requester_wait_once(requester_wait_revalidate, requester_wait_block, + &capture)); + UT_ASSERT_EQ(capture.revalidate_calls, 4); + UT_ASSERT_EQ(capture.wait_calls, 3); + UT_ASSERT_EQ(capture.count_seen_at_wait, 3); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.wait_count, 3); +} + + UT_TEST(test_stats_follow_exact_master_and_local_success_transitions) { PcmXMasterAdmission admission[2]; @@ -16135,7 +16203,7 @@ UT_TEST(test_local_retire_episode_lock_errors_fail_closed) int main(void) { - UT_PLAN(270); + UT_PLAN(271); UT_RUN(test_image_id_domain_is_canonical_and_bounded); UT_RUN(test_wire_abi_sizes_are_exact); UT_RUN(test_wire_abi_offsets_are_exact); @@ -16398,6 +16466,7 @@ main(void) UT_RUN(test_exec_backend_init_fails_closed_on_layout_mismatch); UT_RUN(test_registration_exposes_one_exact_region); UT_RUN(test_stats_initialize_zero_and_narrow_note_apis_are_exact); + UT_RUN(test_queue_wait_count_tracks_only_linearized_requester_waits); UT_RUN(test_stats_follow_exact_master_and_local_success_transitions); UT_RUN(test_stats_count_fail_closed_and_exact_activating_reset_once); UT_RUN(test_master_admit_lock_window_error_releases_gate_and_fails_closed); From 3417a62d4ad6df078d2471eb8196631a3b19b328 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Mon, 20 Jul 2026 14:54:31 +0800 Subject: [PATCH 51/56] fix(cluster): unwind requester waits on late revoke barriers --- src/backend/cluster/cluster_gcs_block.c | 100 +++++++++++------- src/backend/cluster/cluster_pcm_x_convert.c | 17 +++ src/include/cluster/cluster_pcm_x_convert.h | 4 + .../cluster_unit/test_cluster_gcs_block.c | 40 +++++-- .../cluster_unit/test_cluster_pcm_x_convert.c | 43 +++++++- 5 files changed, 156 insertions(+), 48 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 4a438908ab..8c463d7e41 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -8285,14 +8285,17 @@ typedef struct GcsBlockPcmXRequesterWaitContext { long timeout_ms; } GcsBlockPcmXRequesterWaitContext; -static bool +static PcmXQueueResult gcs_block_pcm_x_requester_pre_sleep_revalidate(void *callback_arg) { GcsBlockPcmXRequesterWaitContext *context = (GcsBlockPcmXRequesterWaitContext *)callback_arg; - return gcs_block_pcm_x_requester_authority_exact(context->request_runtime, context->master_node, - context->cluster_epoch, - context->master_session); + if (context == NULL + || !gcs_block_pcm_x_requester_authority_exact( + context->request_runtime, context->master_node, context->cluster_epoch, + context->master_session)) + return PCM_X_QUEUE_NOT_READY; + return cluster_pcm_x_nested_wait_guard_before_block(); } static void @@ -8308,11 +8311,12 @@ gcs_block_pcm_x_requester_wait_latch(void *callback_arg) * sides of the sleep are checked: a runtime fence or peer-session change that * lands while the latch is asleep must be observed before the next protocol * read or mutation. */ -static bool +static PcmXQueueResult gcs_block_pcm_x_requester_wait_exact(uint32 *wait_index, const PcmXRuntimeSnapshot *request_runtime, int32 master_node, uint64 cluster_epoch, uint64 master_session) { GcsBlockPcmXRequesterWaitContext context; + PcmXQueueResult result; uint32 current; if (wait_index == NULL) @@ -8320,7 +8324,7 @@ gcs_block_pcm_x_requester_wait_exact(uint32 *wait_index, const PcmXRuntimeSnapsh errmsg("cluster PCM-X requester wait has no backoff state"))); if (!gcs_block_pcm_x_requester_authority_exact(request_runtime, master_node, cluster_epoch, master_session)) - return false; + return PCM_X_QUEUE_NOT_READY; current = *wait_index; memset(&context, 0, sizeof(context)); context.request_runtime = request_runtime; @@ -8329,14 +8333,15 @@ gcs_block_pcm_x_requester_wait_exact(uint32 *wait_index, const PcmXRuntimeSnapsh context.master_session = master_session; context.timeout_ms = cluster_pcm_x_holder_retry_delay_ms(current); CHECK_FOR_INTERRUPTS(); - if (!cluster_pcm_x_requester_wait_once(gcs_block_pcm_x_requester_pre_sleep_revalidate, - gcs_block_pcm_x_requester_wait_latch, &context)) - return false; + result = cluster_pcm_x_requester_wait_once_result( + gcs_block_pcm_x_requester_pre_sleep_revalidate, gcs_block_pcm_x_requester_wait_latch, + &context); + if (result != PCM_X_QUEUE_OK) + return result; ResetLatch(MyLatch); *wait_index = cluster_gcs_pcm_x_requester_wait_index_advance(current); CHECK_FOR_INTERRUPTS(); - return gcs_block_pcm_x_requester_authority_exact(request_runtime, master_node, cluster_epoch, - master_session); + return gcs_block_pcm_x_requester_pre_sleep_revalidate(&context); } static void @@ -8706,9 +8711,11 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); if (!cluster_gcs_pcm_x_nested_guard_retryable(result)) GCS_BLOCK_PCM_X_REQUESTER_DONE(); - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; + result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } @@ -8734,9 +8741,11 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (result == PCM_X_QUEUE_CORRUPT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); if (initial_own.pcm_state == (uint8)PCM_STATE_READ_IMAGE || result == PCM_X_QUEUE_BUSY) { - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; + result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); GCS_BLOCK_PCM_X_REQUESTER_DONE(); } continue; @@ -8772,9 +8781,11 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim = cluster_gcs_pcm_x_requester_retry_action(GCS_BLOCK_PCM_X_RETRY_SITE_JOIN, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; + result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } @@ -8797,9 +8808,11 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim GCS_BLOCK_PCM_X_RETRY_SITE_LEADER_REKEY, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; + result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } @@ -8816,9 +8829,11 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim GCS_BLOCK_PCM_X_RETRY_SITE_CLAIM, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; + result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } @@ -8836,9 +8851,11 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); gcs_block_pcm_x_requester_cleanup_context.cutoff_started = false; - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; + result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } @@ -8889,10 +8906,11 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_ROLE_REFRESH, result); if (retry_action == GCS_BLOCK_PCM_X_RETRY_WAIT) { - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, - master_node, cluster_epoch, - master_session)) { - result = PCM_X_QUEUE_NOT_READY; + result = gcs_block_pcm_x_requester_wait_exact( + &wait_index, &request_runtime, master_node, cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); GCS_BLOCK_PCM_X_REQUESTER_DONE(); } continue; @@ -8967,9 +8985,11 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; + result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } @@ -9227,9 +9247,11 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); requester_wait: - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; + result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index 02d1dc8e94..0ec2289f73 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -998,6 +998,23 @@ cluster_pcm_x_requester_wait_once(PcmXPreSleepRevalidateCallback revalidate, Pcm } +PcmXQueueResult +cluster_pcm_x_requester_wait_once_result(PcmXPreSleepResultCallback revalidate, + PcmXWaitCallback wait, void *callback_arg) +{ + PcmXQueueResult result; + + if (revalidate == NULL || wait == NULL) + return PCM_X_QUEUE_INVALID; + result = revalidate(callback_arg); + if (result != PCM_X_QUEUE_OK) + return result; + cluster_pcm_x_stats_note_wait(); + wait(callback_arg); + return PCM_X_QUEUE_OK; +} + + void cluster_pcm_x_stats_note_queue_result(PcmXQueueResult result) { diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h index d42e02c0e0..39a1efd3e7 100644 --- a/src/include/cluster/cluster_pcm_x_convert.h +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -1343,9 +1343,13 @@ extern bool cluster_pcm_x_stats_snapshot(PcmXStatsSnapshot *snapshot_out); extern void cluster_pcm_x_stats_note_enqueue(void); extern void cluster_pcm_x_stats_note_wait(void); typedef bool (*PcmXPreSleepRevalidateCallback)(void *callback_arg); +typedef PcmXQueueResult (*PcmXPreSleepResultCallback)(void *callback_arg); typedef void (*PcmXWaitCallback)(void *callback_arg); extern bool cluster_pcm_x_requester_wait_once(PcmXPreSleepRevalidateCallback revalidate, PcmXWaitCallback wait, void *callback_arg); +extern PcmXQueueResult +cluster_pcm_x_requester_wait_once_result(PcmXPreSleepResultCallback revalidate, + PcmXWaitCallback wait, void *callback_arg); extern void cluster_pcm_x_stats_note_queue_result(PcmXQueueResult result); extern void cluster_pcm_x_stats_note_own_begin(void); extern void cluster_pcm_x_stats_note_own_commit(void); diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index f2d5883f0f..3cb28212d5 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -2725,7 +2725,11 @@ UT_TEST(test_pcm_x_requester_driver_owns_fifo_and_transfer_lifecycles) const char *wait_exact_end; const char *wait_exact_first_authority; const char *wait_exact_physical; - const char *wait_exact_second_authority; + const char *wait_exact_second_revalidate; + const char *wait_revalidate; + const char *wait_revalidate_end; + const char *wait_revalidate_authority; + const char *wait_revalidate_nested_guard; const char *wait_scan; int raw_wait_count = 0; int exact_wait_count = 0; @@ -2748,6 +2752,9 @@ UT_TEST(test_pcm_x_requester_driver_owns_fifo_and_transfer_lifecycles) authority_helper_end = authority_helper != NULL ? strstr(authority_helper, "\n}\n") : NULL; wait_exact = strstr(source, "\ngcs_block_pcm_x_requester_wait_exact("); wait_exact_end = wait_exact != NULL ? strstr(wait_exact, "\n}\n") : NULL; + wait_revalidate = strstr(source, "\ngcs_block_pcm_x_requester_pre_sleep_revalidate("); + wait_revalidate_end + = wait_revalidate != NULL ? strstr(wait_revalidate, "\n}\n") : NULL; UT_ASSERT_NOT_NULL(driver); UT_ASSERT_NOT_NULL(driver_end); UT_ASSERT_NOT_NULL(rekey_helper); @@ -2762,6 +2769,8 @@ UT_TEST(test_pcm_x_requester_driver_owns_fifo_and_transfer_lifecycles) UT_ASSERT_NOT_NULL(authority_helper_end); UT_ASSERT_NOT_NULL(wait_exact); UT_ASSERT_NOT_NULL(wait_exact_end); + UT_ASSERT_NOT_NULL(wait_revalidate); + UT_ASSERT_NOT_NULL(wait_revalidate_end); if (driver == NULL || driver_end == NULL || wrapper == NULL || wrapper_end == NULL) { free(source); return; @@ -2859,28 +2868,43 @@ UT_TEST(test_pcm_x_requester_driver_owns_fifo_and_transfer_lifecycles) : NULL; wait_exact_physical = wait_exact_first_authority != NULL - ? strstr(wait_exact_first_authority, "cluster_pcm_x_requester_wait_once(") + ? strstr(wait_exact_first_authority, "cluster_pcm_x_requester_wait_once_result(") : NULL; - wait_exact_second_authority + wait_exact_second_revalidate = wait_exact_physical != NULL - ? strstr(wait_exact_physical + 1, "gcs_block_pcm_x_requester_authority_exact(") + ? strstr(wait_exact_physical + 1, + "gcs_block_pcm_x_requester_pre_sleep_revalidate(&context)") + : NULL; + wait_revalidate_authority + = wait_revalidate != NULL + ? strstr(wait_revalidate, "gcs_block_pcm_x_requester_authority_exact(") + : NULL; + wait_revalidate_nested_guard + = wait_revalidate_authority != NULL + ? strstr(wait_revalidate_authority, "cluster_pcm_x_nested_wait_guard_before_block()") : NULL; UT_ASSERT_NOT_NULL(authority_runtime); UT_ASSERT_NOT_NULL(authority_runtime_exact); UT_ASSERT_NOT_NULL(authority_peer); UT_ASSERT_NOT_NULL(wait_exact_first_authority); UT_ASSERT_NOT_NULL(wait_exact_physical); - UT_ASSERT_NOT_NULL(wait_exact_second_authority); + UT_ASSERT_NOT_NULL(wait_exact_second_revalidate); + UT_ASSERT_NOT_NULL(wait_revalidate_authority); + UT_ASSERT_NOT_NULL(wait_revalidate_nested_guard); if (authority_runtime != NULL && authority_runtime_exact != NULL && authority_peer != NULL && authority_helper_end != NULL) UT_ASSERT(authority_runtime < authority_runtime_exact && authority_runtime_exact < authority_peer && authority_peer < authority_helper_end); if (wait_exact_first_authority != NULL && wait_exact_physical != NULL - && wait_exact_second_authority != NULL && wait_exact_end != NULL) + && wait_exact_second_revalidate != NULL && wait_exact_end != NULL) UT_ASSERT(wait_exact_first_authority < wait_exact_physical - && wait_exact_physical < wait_exact_second_authority - && wait_exact_second_authority < wait_exact_end); + && wait_exact_physical < wait_exact_second_revalidate + && wait_exact_second_revalidate < wait_exact_end); + if (wait_revalidate_authority != NULL && wait_revalidate_nested_guard != NULL + && wait_revalidate_end != NULL) + UT_ASSERT(wait_revalidate_authority < wait_revalidate_nested_guard + && wait_revalidate_nested_guard < wait_revalidate_end); formation = strstr(driver, "cluster_gcs_pcm_x_requester_formation_action("); formation_wait = formation != NULL ? strstr(formation, "gcs_block_pcm_x_requester_wait(&wait_index)") diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 1e005e2e0c..eb87be8844 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -15699,6 +15699,15 @@ requester_wait_revalidate(void *callback_arg) return capture->revalidate_ok; } +static PcmXQueueResult +requester_wait_revalidate_result(void *callback_arg) +{ + RequesterWaitCapture *capture = (RequesterWaitCapture *)callback_arg; + + capture->revalidate_calls++; + return capture->revalidate_ok ? PCM_X_QUEUE_OK : PCM_X_QUEUE_BARRIER_CLOSED; +} + static void requester_wait_block(void *callback_arg) { @@ -15751,6 +15760,37 @@ UT_TEST(test_queue_wait_count_tracks_only_linearized_requester_waits) } +UT_TEST(test_requester_wait_preserves_late_revoke_barrier_refusal) +{ + RequesterWaitCapture capture = { 0 }; + PcmXStatsSnapshot stats; + + init_active_pcm_x(UINT64_C(82)); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.wait_count, 0); + + /* A type-49 barrier that freezes after initial admission but before the + * next WaitLatch must escape as BARRIER_CLOSED. Sleeping here would keep + * the source holder registered and deadlock IMAGE_READY forever. */ + UT_ASSERT_EQ(cluster_pcm_x_requester_wait_once_result( + requester_wait_revalidate_result, requester_wait_block, &capture), + PCM_X_QUEUE_BARRIER_CLOSED); + UT_ASSERT_EQ(capture.revalidate_calls, 1); + UT_ASSERT_EQ(capture.wait_calls, 0); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.wait_count, 0); + + /* An exact OK proof still linearizes the count before the physical wait. */ + capture.revalidate_ok = true; + UT_ASSERT_EQ(cluster_pcm_x_requester_wait_once_result( + requester_wait_revalidate_result, requester_wait_block, &capture), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(capture.revalidate_calls, 2); + UT_ASSERT_EQ(capture.wait_calls, 1); + UT_ASSERT_EQ(capture.count_seen_at_wait, 1); +} + + UT_TEST(test_stats_follow_exact_master_and_local_success_transitions) { PcmXMasterAdmission admission[2]; @@ -16203,7 +16243,7 @@ UT_TEST(test_local_retire_episode_lock_errors_fail_closed) int main(void) { - UT_PLAN(271); + UT_PLAN(272); UT_RUN(test_image_id_domain_is_canonical_and_bounded); UT_RUN(test_wire_abi_sizes_are_exact); UT_RUN(test_wire_abi_offsets_are_exact); @@ -16467,6 +16507,7 @@ main(void) UT_RUN(test_registration_exposes_one_exact_region); UT_RUN(test_stats_initialize_zero_and_narrow_note_apis_are_exact); UT_RUN(test_queue_wait_count_tracks_only_linearized_requester_waits); + UT_RUN(test_requester_wait_preserves_late_revoke_barrier_refusal); UT_RUN(test_stats_follow_exact_master_and_local_success_transitions); UT_RUN(test_stats_count_fail_closed_and_exact_activating_reset_once); UT_RUN(test_master_admit_lock_window_error_releases_gate_and_fails_closed); From 3bcf48496f62ff5213ac7299effe5be3dd41ba50 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Mon, 20 Jul 2026 14:54:52 +0800 Subject: [PATCH 52/56] fix(cluster): reuse stable successor S cover on reverify --- src/backend/storage/buffer/bufmgr.c | 20 +++++------ src/include/cluster/cluster_pcm_x_bufmgr.h | 27 ++++++++++++++ src/test/cluster_unit/test_cluster_pcm_own.c | 38 +++++++++++++++++--- 3 files changed, 70 insertions(+), 15 deletions(-) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index d4299b3145..78bd8fbf8d 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -8489,14 +8489,13 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) * PGRAC ownership-generation wave (W1) — cached-cover * re-verify. The cover fast path decided we already held the mode * on a raw, unlocked pcm_state read. A BAST X->S downgrade (or - * any ownership round) can have raced this content-lock window - * and revoked the cover. Re-read the state/generation/flags - * projection of the coherent ownership tuple under the header - * spinlock: if the generation moved, the mode no longer covers, - * or a grant/revoke is in flight, the cover is STALE -- - * writing/reading a block we no longer own is the Rule 8.A - * violation. Release, do a real master re-acquire, re-take the - * content lock. Bounded: after a real acquire we hold the + * any ownership round) can have raced this content-lock window. + * Re-read the coherent ownership tuple now that content authority + * serializes page-image transitions. A stable successor S/X still + * covers an S read; an X writer stays generation-exact. A mode + * loss or live lifecycle is STALE for both. Release, do a real + * master/queue re-acquire, and re-take the content lock. Bounded: + * after a real acquire we hold the * content lock and the downgrade path serializes under it, so no * further downgrade can intervene -- at most one fallback. */ @@ -8507,9 +8506,8 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) uint32 cur_flags; cluster_pcm_own_read(buf, &cur_state, &cur_gen, &cur_flags); - if (cur_gen != pcm_covered_gen - || !cluster_pcm_mode_covers((PcmLockMode) cur_state, pcm_mode) - || (cur_flags & (PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING)) != 0) + if (!cluster_pcm_x_cached_cover_reverify_accepts( + (uint8) pcm_mode, pcm_covered_gen, cur_gen, cur_state, cur_flags)) { cluster_pcm_note_writer_cover_stale_detected(); pcm_covered = false; diff --git a/src/include/cluster/cluster_pcm_x_bufmgr.h b/src/include/cluster/cluster_pcm_x_bufmgr.h index 3f8cdbdc44..9c0a58e330 100644 --- a/src/include/cluster/cluster_pcm_x_bufmgr.h +++ b/src/include/cluster/cluster_pcm_x_bufmgr.h @@ -172,6 +172,33 @@ cluster_pcm_x_cached_cover_bypasses_queue(bool local_cache, bool requested_x, ui return local_cache && requested_x && pcm_state == (uint8)PCM_STATE_X && flags == 0; } +/* Revalidate a cached cover after the caller has acquired content authority. + * A stable current S/X is the node-level authority for an S read even when a + * complete ownership round advanced the generation while this backend waited: + * the page-image install/transition is serialized by the content lock already + * held by the caller. Treating that successor S as stale would open a fresh + * legacy reservation from S (the forbidden S_NEW shape). X writers retain + * the stricter generation-exact rule and re-enter the convert queue after any + * ownership round. */ +static inline bool +cluster_pcm_x_cached_cover_reverify_accepts(uint8 requested_state, uint64 captured_generation, + uint64 current_generation, uint8 current_state, + uint32 current_flags) +{ + bool covers; + + if (requested_state != (uint8)PCM_STATE_S && requested_state != (uint8)PCM_STATE_X) + return false; + if (current_flags != 0) + return false; + covers = current_state == (uint8)PCM_STATE_X + || (requested_state == (uint8)PCM_STATE_S + && current_state == (uint8)PCM_STATE_S); + return covers + && (requested_state == (uint8)PCM_STATE_S + || current_generation == captured_generation); +} + /* ConditionalLockBuffer cannot initiate a PCM conversion. Preserve native * PostgreSQL behavior while PCM is inactive and for relations outside the * coherence domain; an active tracked page must already hold exact X. Live diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index 443ecd31fc..5c030bc333 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -358,6 +358,35 @@ UT_TEST(test_s_new_fresh_token_finish_shape_stays_invalid) CLUSTER_PCM_X_GRANT_RESERVATION_INVALID); } +UT_TEST(test_share_cover_reverify_accepts_stable_successor_grant) +{ + /* Once content authority is held, a stable current S/X successor is the + * exact node-level grant for a read. Generation drift alone must not open + * a fresh legacy reservation from S (the forbidden S_NEW shape). */ + UT_ASSERT(cluster_pcm_x_cached_cover_reverify_accepts( + (uint8)PCM_LOCK_MODE_S, UINT64_C(13), UINT64_C(14), (uint8)PCM_STATE_S, 0)); + UT_ASSERT(cluster_pcm_x_cached_cover_reverify_accepts( + (uint8)PCM_LOCK_MODE_S, UINT64_C(13), UINT64_C(14), (uint8)PCM_STATE_X, 0)); + + /* A writer keeps the stricter generation-exact gate and must re-enter the + * convert queue after any ownership round. A non-covering or live + * lifecycle remains closed for both modes. */ + UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts( + (uint8)PCM_LOCK_MODE_X, UINT64_C(13), UINT64_C(14), (uint8)PCM_STATE_X, 0)); + UT_ASSERT(cluster_pcm_x_cached_cover_reverify_accepts( + (uint8)PCM_LOCK_MODE_X, UINT64_C(14), UINT64_C(14), (uint8)PCM_STATE_X, 0)); + UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts( + (uint8)PCM_LOCK_MODE_X, UINT64_C(14), UINT64_C(14), (uint8)PCM_STATE_S, 0)); + UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts( + (uint8)PCM_LOCK_MODE_S, UINT64_C(14), UINT64_C(14), (uint8)PCM_STATE_S, + PCM_OWN_FLAG_GRANT_PENDING)); + UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts( + (uint8)PCM_LOCK_MODE_S, UINT64_C(14), UINT64_C(14), (uint8)PCM_STATE_S, + PCM_OWN_FLAG_REVOKING)); + UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts( + (uint8)PCM_LOCK_MODE_N, UINT64_C(14), UINT64_C(14), (uint8)PCM_STATE_X, 0)); +} + UT_TEST(test_revoke_commit_is_exact_and_classifies_live_races) { uint64 committed = UINT64_MAX; @@ -1363,12 +1392,12 @@ UT_TEST(test_retained_image_release_and_writeback_gates_are_exact) = strstr(lockbuffer, "cluster_bufmgr_pcm_begin_grant_reservation_wait("); const char *content = strstr(lockbuffer, "LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED)"); - const char *w1_revoke - = strstr(lockbuffer, "PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING"); + const char *w1_reverify + = strstr(lockbuffer, "cluster_pcm_x_cached_cover_reverify_accepts("); UT_ASSERT_NOT_NULL(reserve); UT_ASSERT_NOT_NULL(content); - UT_ASSERT_NOT_NULL(w1_revoke); + UT_ASSERT_NOT_NULL(w1_reverify); if (reserve != NULL && content != NULL) UT_ASSERT(reserve < content); } @@ -2299,7 +2328,7 @@ UT_TEST(test_own_lifecycle_counters_land_on_exact_begin_and_x_commit) int main(void) { - UT_PLAN(51); + UT_PLAN(52); UT_RUN(test_shmem_initializes_complete_entry); UT_RUN(test_begin_abort_is_exact_and_monotonic); UT_RUN(test_invalid_live_flag_shapes_are_corrupt_not_busy); @@ -2307,6 +2336,7 @@ main(void) UT_RUN(test_s_revoke_handoff_reuses_exact_token_and_bumps_once); UT_RUN(test_revoke_handoff_kinds_cover_n_s_x_with_one_lifecycle); UT_RUN(test_s_new_fresh_token_finish_shape_stays_invalid); + UT_RUN(test_share_cover_reverify_accepts_stable_successor_grant); UT_RUN(test_retained_release_retag_respects_pin_contract); UT_RUN(test_retained_release_and_finish_never_cover_invalid_bytes); UT_RUN(test_legacy_byte_proof_republishes_kept_pi_mirror); From e59cc4bf0043e94bba794cbe879957aee30e55ff Mon Sep 17 00:00:00 2001 From: SqlRush Date: Mon, 20 Jul 2026 15:30:49 +0800 Subject: [PATCH 53/56] fix(cluster): stage pending-X denial replies on DATA owner --- src/backend/cluster/cluster_gcs_block.c | 23 +++- src/backend/cluster/cluster_lms_outbound.c | 104 +++++++++++++++++- src/include/cluster/cluster_gcs_block.h | 2 + src/include/cluster/cluster_lms.h | 3 + .../cluster_unit/test_cluster_lms_outbound.c | 94 +++++++++++++++- 5 files changed, 219 insertions(+), 7 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 8c463d7e41..6fde68e916 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -1116,6 +1116,21 @@ gcs_block_send_direct_reply_sge(int32 dest_node, const GcsBlockReplyHeader *hdr, return rc; } +ClusterICSendResult +cluster_gcs_block_send_direct_zero_reply(int32 dest_node, const GcsBlockReplyHeader *header) +{ + ClusterICSendResult rc; + + if (header == NULL || dest_node < 0 || dest_node >= CLUSTER_MAX_NODES + || dest_node == cluster_node_id + || header->status != (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X) + return CLUSTER_IC_SEND_HARD_ERROR; + rc = gcs_block_send_direct_reply_sge(dest_node, header, NULL, 0, NULL, NULL); + if (rc == CLUSTER_IC_SEND_DONE && ClusterGcsBlock != NULL) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_reply_count, 1); + return rc; +} + static bool gcs_block_try_send_direct_reply(int32 dest_node, bool direct_armed, GcsBlockReplyHeader *hdr, const char *block_payload, uint32 block_lkey, @@ -9798,7 +9813,13 @@ gcs_block_pcm_x_deny_legacy_readers(const PcmXMasterDriveSnapshot *snapshot) && deny_result != GCS_BLOCK_PENDING_X_DENY_REPLAY) return PCM_X_QUEUE_CORRUPT; if (denied.key.origin_node_id >= PCM_X_PROTOCOL_NODE_LIMIT - || !gcs_block_resend_cached_reply((int32)denied.key.origin_node_id, &denied)) + || denied.status != GCS_BLOCK_REPLY_DENIED_PENDING_X + || denied.reply_header.status != (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X) + return PCM_X_QUEUE_CORRUPT; + if (!cluster_lms_outbound_enqueue_zero_block_reply( + worker_id, denied.key.origin_node_id, &denied.reply_header, + (denied.request_flags & GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND) != 0 + && (int32)denied.key.origin_node_id != cluster_node_id)) return PCM_X_QUEUE_BUSY; if (deny_result == GCS_BLOCK_PENDING_X_DENY_REPLAY) return PCM_X_QUEUE_OK; diff --git a/src/backend/cluster/cluster_lms_outbound.c b/src/backend/cluster/cluster_lms_outbound.c index db9697fe54..a46ba15071 100644 --- a/src/backend/cluster/cluster_lms_outbound.c +++ b/src/backend/cluster/cluster_lms_outbound.c @@ -54,13 +54,28 @@ #define PGRAC_LMS_OUTBOUND_CAPACITY 256 #define PGRAC_LMS_OUTBOUND_PAYLOAD_MAX 128 +typedef enum ClusterLmsOutboundKind { + CLUSTER_LMS_OUTBOUND_FRAME = 0, + CLUSTER_LMS_OUTBOUND_ZERO_BLOCK_REPLY = 1, + CLUSTER_LMS_OUTBOUND_DIRECT_ZERO_BLOCK_REPLY = 2 +} ClusterLmsOutboundKind; + typedef struct ClusterLmsOutboundSlot { uint32 dest_node_id; uint8 msg_type; + uint8 kind; uint16 payload_len; uint8 payload[PGRAC_LMS_OUTBOUND_PAYLOAD_MAX]; } ClusterLmsOutboundSlot; +typedef struct ClusterLmsZeroBlockReplyWire { + GcsBlockReplyHeader header; + char block_data[GCS_BLOCK_DATA_SIZE]; +} ClusterLmsZeroBlockReplyWire; + +StaticAssertDecl(sizeof(ClusterLmsZeroBlockReplyWire) == GCS_BLOCK_REPLY_PAYLOAD_TOTAL_SIZE, + "staged zero-block reply must preserve the GCS reply wire size"); + typedef struct ClusterLmsOutboundState { uint32 head; /* next slot to fill */ uint32 tail; /* next slot to drain */ @@ -182,6 +197,7 @@ cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, slot = &ring->ring[ring->head]; slot->dest_node_id = dest_node_id; slot->msg_type = msg_type; + slot->kind = (uint8)CLUSTER_LMS_OUTBOUND_FRAME; slot->payload_len = payload_len; if (payload_len > 0) memcpy(slot->payload, payload, payload_len); @@ -193,6 +209,51 @@ cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, return true; } +/* + * Stage a header-only GCS denial from a CONTROL-plane producer. The DATA + * owner expands the ABI-mandated zero block immediately before transport + * admission. Keep this surface narrow: only the Shape-B pending-X denial + * may use it, and callers must choose worker[shard(tag)] so it stays on the + * same per-tag stream as the request it terminates. + */ +bool +cluster_lms_outbound_enqueue_zero_block_reply(int worker_id, uint32 dest_node_id, + const GcsBlockReplyHeader *header, bool direct_land) +{ + ClusterLmsOutboundState *ring; + LWLock *lock; + ClusterLmsOutboundSlot *slot; + + if (worker_id < 0 || worker_id >= CLUSTER_LMS_MAX_WORKERS + || dest_node_id >= CLUSTER_MAX_NODES || header == NULL + || (direct_land && (int32)dest_node_id == cluster_node_id) + || header->status != (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X) + return false; + if (cluster_lms_outbound_rings == NULL || OB_LOCK(worker_id) == NULL) + return false; + + ring = OB_RING(worker_id); + lock = OB_LOCK(worker_id); + LWLockAcquire(lock, LW_EXCLUSIVE); + if (ring->count >= PGRAC_LMS_OUTBOUND_CAPACITY) { + LWLockRelease(lock); + return false; + } + slot = &ring->ring[ring->head]; + slot->dest_node_id = dest_node_id; + slot->msg_type = PGRAC_IC_MSG_GCS_BLOCK_REPLY; + slot->kind = (uint8)(direct_land ? CLUSTER_LMS_OUTBOUND_DIRECT_ZERO_BLOCK_REPLY + : CLUSTER_LMS_OUTBOUND_ZERO_BLOCK_REPLY); + slot->payload_len = sizeof(*header); + memcpy(slot->payload, header, sizeof(*header)); + ring->head = (ring->head + 1) % PGRAC_LMS_OUTBOUND_CAPACITY; + ring->count++; + LWLockRelease(lock); + + cluster_lms_wakeup(worker_id); + return true; +} + /* * cluster_lms_outbound_drain_send — one worker drains + sends its own ring. * @@ -247,6 +308,9 @@ cluster_lms_outbound_drain_send(int worker_id) while (scanned < 64) { ClusterLmsOutboundSlot slot; + ClusterLmsZeroBlockReplyWire zero_reply; + const void *send_payload; + uint32 send_payload_len; ClusterICSendResult rc; bool got = false; @@ -261,6 +325,29 @@ cluster_lms_outbound_drain_send(int worker_id) if (!got) break; scanned++; + send_payload = slot.payload_len > 0 ? slot.payload : NULL; + send_payload_len = slot.payload_len; + if (slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_ZERO_BLOCK_REPLY + || slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_DIRECT_ZERO_BLOCK_REPLY) { + if (slot.msg_type != PGRAC_IC_MSG_GCS_BLOCK_REPLY + || slot.payload_len != sizeof(GcsBlockReplyHeader)) { + rc = CLUSTER_IC_SEND_HARD_ERROR; + goto handle_send_result; + } + memset(&zero_reply, 0, sizeof(zero_reply)); + memcpy(&zero_reply.header, slot.payload, sizeof(zero_reply.header)); + if (zero_reply.header.status != (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X) { + rc = CLUSTER_IC_SEND_HARD_ERROR; + goto handle_send_result; + } + zero_reply.header.checksum + = cluster_gcs_block_compute_checksum(zero_reply.block_data); + send_payload = &zero_reply; + send_payload_len = sizeof(zero_reply); + } else if (slot.kind != (uint8)CLUSTER_LMS_OUTBOUND_FRAME) { + rc = CLUSTER_IC_SEND_HARD_ERROR; + goto handle_send_result; + } /* A peer that refused a frame this batch keeps its later frames * queued BEHIND the refused one (per-peer order). */ @@ -296,21 +383,28 @@ cluster_lms_outbound_drain_send(int worker_id) * staged back onto this tag's ring, preserving the one-worker FIFO and * avoiding recursive handler execution. */ - if ((int32)slot.dest_node_id == cluster_node_id) { + if (slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_DIRECT_ZERO_BLOCK_REPLY) + rc = cluster_gcs_block_send_direct_zero_reply((int32)slot.dest_node_id, + &zero_reply.header); + else if ((int32)slot.dest_node_id == cluster_node_id) { ClusterICEnvelope env; if (cluster_ic_envelope_build( &env, slot.msg_type, (uint32)cluster_node_id, slot.dest_node_id, - slot.payload_len > 0 ? slot.payload : NULL, slot.payload_len) - && cluster_ic_dispatch_envelope(&env, slot.payload_len > 0 ? slot.payload : NULL, + send_payload, send_payload_len) + && cluster_ic_dispatch_envelope(&env, send_payload, cluster_node_id)) rc = CLUSTER_IC_SEND_DONE; else rc = CLUSTER_IC_SEND_HARD_ERROR; } else rc = cluster_ic_send_envelope(slot.msg_type, (int32)slot.dest_node_id, - slot.payload_len > 0 ? slot.payload : NULL, - slot.payload_len); + send_payload, send_payload_len); + +handle_send_result: + if (slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_ZERO_BLOCK_REPLY + || slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_DIRECT_ZERO_BLOCK_REPLY) + cluster_gcs_block_note_send_outcome(GCS_BLOCK_SEND_FAMILY_REPLY, rc); switch (rc) { case CLUSTER_IC_SEND_DONE: case CLUSTER_IC_SEND_WOULD_BLOCK: diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index 5385f493f8..07d2d83b44 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -3431,6 +3431,8 @@ typedef enum GcsBlockSendFamily { } GcsBlockSendFamily; extern void cluster_gcs_block_note_send_outcome(GcsBlockSendFamily family, ClusterICSendResult rc); +extern ClusterICSendResult cluster_gcs_block_send_direct_zero_reply( + int32 dest_node, const GcsBlockReplyHeader *header); extern uint64 cluster_gcs_get_reply_send_queued_count(void); extern uint64 cluster_gcs_get_reply_send_not_admitted_count(void); diff --git a/src/include/cluster/cluster_lms.h b/src/include/cluster/cluster_lms.h index c716f5d677..7277c4dca9 100644 --- a/src/include/cluster/cluster_lms.h +++ b/src/include/cluster/cluster_lms.h @@ -401,6 +401,9 @@ extern void cluster_lms_outbound_shmem_register(void); extern void cluster_lms_outbound_request_lwlocks(void); extern bool cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, const void *payload, uint16 payload_len); +struct GcsBlockReplyHeader; +extern bool cluster_lms_outbound_enqueue_zero_block_reply( + int worker_id, uint32 dest_node_id, const struct GcsBlockReplyHeader *header, bool direct_land); extern int cluster_lms_outbound_drain_send(int worker_id); extern uint32 cluster_lms_outbound_depth(int worker_id); diff --git a/src/test/cluster_unit/test_cluster_lms_outbound.c b/src/test/cluster_unit/test_cluster_lms_outbound.c index a94623d562..fd42a223ac 100644 --- a/src/test/cluster_unit/test_cluster_lms_outbound.c +++ b/src/test/cluster_unit/test_cluster_lms_outbound.c @@ -193,6 +193,13 @@ cluster_lms_obs_note_outbound_requeue_drop(int worker_id) ut_requeue_drop_count++; } +void +cluster_gcs_block_note_send_outcome(GcsBlockSendFamily family, ClusterICSendResult rc) +{ + (void)family; + (void)rc; +} + static int ut_prepare_hook_count = 0; void @@ -215,6 +222,8 @@ typedef struct UtSentRec { uint8 msg_type; int32 dest; uint8 marker; /* first payload byte identifies the frame */ + uint32 payload_len; + GcsBlockReplyHeader reply_header; } UtSentRec; static UtSentRec ut_sent_log[64]; @@ -222,6 +231,8 @@ static int ut_sent_n = 0; static ClusterICSendResult ut_peer_rc[CLUSTER_MAX_NODES]; static int ut_local_dispatch_count = 0; static uint8 ut_local_dispatch_marker = 0; +static int ut_direct_zero_reply_count = 0; +static GcsBlockReplyHeader ut_direct_zero_reply_header; bool cluster_ic_envelope_build(ClusterICEnvelope *out_env, uint8 msg_type, uint32 source_node_id, @@ -256,12 +267,31 @@ cluster_ic_send_envelope(uint8 msg_type, int32 dest_node_id, const void *payload ut_sent_log[ut_sent_n].msg_type = msg_type; ut_sent_log[ut_sent_n].dest = dest_node_id; ut_sent_log[ut_sent_n].marker = payload_len > 0 ? *(const uint8 *)payload : 0; + ut_sent_log[ut_sent_n].payload_len = payload_len; + if (msg_type == PGRAC_IC_MSG_GCS_BLOCK_REPLY + && payload_len >= sizeof(GcsBlockReplyHeader)) + memcpy(&ut_sent_log[ut_sent_n].reply_header, payload, sizeof(GcsBlockReplyHeader)); } ut_sent_n++; UT_ASSERT(dest_node_id >= 0 && dest_node_id < CLUSTER_MAX_NODES); return ut_peer_rc[dest_node_id]; } +uint32 +cluster_gcs_block_compute_checksum(const char *block_data) +{ + (void)block_data; + return UINT32_C(0xA55A7E11); +} + +ClusterICSendResult +cluster_gcs_block_send_direct_zero_reply(int32 dest_node, const GcsBlockReplyHeader *header) +{ + ut_direct_zero_reply_count++; + ut_direct_zero_reply_header = *header; + return ut_peer_rc[dest_node]; +} + static int ut_count_marker(uint8 marker) { @@ -280,6 +310,8 @@ ut_reset_log(void) ut_sent_n = 0; ut_local_dispatch_count = 0; ut_local_dispatch_marker = 0; + ut_direct_zero_reply_count = 0; + memset(&ut_direct_zero_reply_header, 0, sizeof(ut_direct_zero_reply_header)); ut_pcm_x_runtime_state = PCM_X_RUNTIME_ACTIVE; ut_write_fence_enforcing = false; ut_write_fence_allowed = true; @@ -498,10 +530,68 @@ UT_TEST(test_pcm_x_grant_frame_waits_behind_write_fence) UT_ASSERT_EQ(cluster_lms_outbound_depth(7), 0); } +/* + * Shape-B denial replay is driven by LMON, which owns only plane 0. The + * reply is an ABI-sized header + zero block, so LMON stages its compact + * header on the tag's DATA ring and the owning LMS worker expands and sends + * it. A direct LMON send is a production FATAL under the plane guard. + */ +UT_TEST(test_zero_block_reply_is_expanded_by_data_owner) +{ + GcsBlockReplyHeader hdr; + + ut_reset_log(); + memset(&hdr, 0, sizeof(hdr)); + hdr.request_id = UINT64_C(0x1122334455667788); + hdr.epoch = UINT64_C(41); + hdr.sender_node = 1; + hdr.requester_backend_id = 17; + hdr.transition_id = PCM_TRANS_N_TO_X; + hdr.status = (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X; + GcsBlockReplyHeaderSetForwardingMasterNode(&hdr, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + + UT_ASSERT(cluster_lms_outbound_enqueue_zero_block_reply(2, UT_PEER_X, &hdr, false)); + UT_ASSERT_EQ(cluster_lms_outbound_depth(2), 1); + ut_peer_rc[UT_PEER_X] = CLUSTER_IC_SEND_NOT_ADMITTED; + UT_ASSERT_EQ(cluster_lms_outbound_drain_send(2), 0); + UT_ASSERT_EQ(cluster_lms_outbound_depth(2), 1); + UT_ASSERT_EQ(ut_sent_n, 1); + UT_ASSERT_EQ((int)ut_sent_log[0].payload_len, (int)GCS_BLOCK_REPLY_PAYLOAD_TOTAL_SIZE); + + ut_peer_rc[UT_PEER_X] = CLUSTER_IC_SEND_DONE; + UT_ASSERT_EQ(cluster_lms_outbound_drain_send(2), 1); + UT_ASSERT_EQ(cluster_lms_outbound_depth(2), 0); + UT_ASSERT_EQ(ut_sent_n, 2); + UT_ASSERT_EQ((int)ut_sent_log[1].msg_type, (int)PGRAC_IC_MSG_GCS_BLOCK_REPLY); + UT_ASSERT_EQ((int)ut_sent_log[1].payload_len, (int)GCS_BLOCK_REPLY_PAYLOAD_TOTAL_SIZE); + UT_ASSERT_EQ(ut_sent_log[1].reply_header.request_id, hdr.request_id); + UT_ASSERT_EQ((int)ut_sent_log[1].reply_header.status, + (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ(ut_sent_log[1].reply_header.checksum, UINT32_C(0xA55A7E11)); +} + +UT_TEST(test_direct_zero_block_reply_uses_data_owner_direct_lane) +{ + GcsBlockReplyHeader hdr; + + ut_reset_log(); + memset(&hdr, 0, sizeof(hdr)); + hdr.request_id = UINT64_C(0x8877665544332211); + hdr.status = (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X; + + UT_ASSERT(cluster_lms_outbound_enqueue_zero_block_reply(3, UT_PEER_Y, &hdr, true)); + ut_peer_rc[UT_PEER_Y] = CLUSTER_IC_SEND_DONE; + UT_ASSERT_EQ(cluster_lms_outbound_drain_send(3), 1); + UT_ASSERT_EQ(ut_direct_zero_reply_count, 1); + UT_ASSERT_EQ(ut_sent_n, 0); + UT_ASSERT_EQ(ut_direct_zero_reply_header.request_id, hdr.request_id); + UT_ASSERT_EQ(ut_direct_zero_reply_header.checksum, UINT32_C(0xA55A7E11)); +} + int main(void) { - UT_PLAN(8); + UT_PLAN(10); UT_RUN(test_ring_shmem_init); UT_RUN(test_admitted_frame_is_never_resubmitted); @@ -511,6 +601,8 @@ main(void) UT_RUN(test_self_frame_dispatches_on_owning_worker); UT_RUN(test_pcm_x_grant_frame_waits_for_active_runtime); UT_RUN(test_pcm_x_grant_frame_waits_behind_write_fence); + UT_RUN(test_zero_block_reply_is_expanded_by_data_owner); + UT_RUN(test_direct_zero_block_reply_uses_data_owner_direct_lane); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; From 1b2c3c422cbf99d976da2844c1717316164e3ee0 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Mon, 20 Jul 2026 19:30:33 +0800 Subject: [PATCH 54/56] fix(cluster): keep PCM-X retained fallback current --- src/backend/cluster/cluster_gcs_block.c | 161 ++++++++++++--- src/backend/storage/buffer/bufmgr.c | 190 +++++++++++++++--- .../t/400_pcm_x_queue_4node_liveness.pl | 61 +++++- .../cluster_unit/test_cluster_gcs_block.c | 85 +++++++- src/test/cluster_unit/test_cluster_pcm_own.c | 59 ++++-- 5 files changed, 471 insertions(+), 85 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 6fde68e916..63a8f66552 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -9486,6 +9486,10 @@ gcs_block_pcm_x_collect_formation(PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_L static void gcs_block_pcm_x_master_drive_retry_tick(void); static void gcs_block_pcm_x_terminal_retry_tick(void); +static void gcs_block_pcm_x_master_retry_observe(const char *stage, PcmXQueueResult result, + int32 peer_node, Size cursor_before, + Size cursor_after, const BufferTag *tag, + uint64 cluster_epoch); static PcmXQueueResult gcs_block_pcm_x_cancel_claimed_probe_exact(const PcmXMasterPendingXReleaseToken *token); @@ -9518,14 +9522,23 @@ cluster_gcs_block_pcm_x_formation_tick(void) * including an unrelated membership flip, is a no-op for this tick. */ if (!gcs_block_pcm_x_collect_formation(bindings_before, &epoch_before, &self_session_before, - NULL)) + NULL)) { + gcs_block_pcm_x_master_retry_observe("collect-before", PCM_X_QUEUE_NOT_READY, -1, 0, + 0, NULL, 0); return; + } if (!gcs_block_pcm_x_collect_formation(bindings_after, &epoch_after, &self_session_after, - NULL)) + NULL)) { + gcs_block_pcm_x_master_retry_observe("collect-after", PCM_X_QUEUE_NOT_READY, -1, 0, + 0, NULL, 0); return; + } if (!cluster_gcs_pcm_x_formation_samples_stable(true, bindings_before, true, bindings_after) - || epoch_before != epoch_after || self_session_before != self_session_after) + || epoch_before != epoch_after || self_session_before != self_session_after) { + gcs_block_pcm_x_master_retry_observe("stability", PCM_X_QUEUE_NOT_READY, -1, 0, 0, + NULL, epoch_before); return; + } for (i = 0; i < PCM_X_PROTOCOL_NODE_LIMIT; i++) { if (bindings_before[i].peer_session_incarnation == 0) continue; @@ -9533,14 +9546,20 @@ cluster_gcs_block_pcm_x_formation_tick(void) i, bindings_before[i].cluster_epoch, bindings_before[i].peer_session_incarnation); if (result == PCM_X_QUEUE_STALE || result == PCM_X_QUEUE_CORRUPT) goto fail_closed; - if (result != PCM_X_QUEUE_OK) + if (result != PCM_X_QUEUE_OK) { + gcs_block_pcm_x_master_retry_observe("peer-revalidate", result, i, 0, 0, NULL, + epoch_before); return; + } } runtime_after = cluster_pcm_x_runtime_snapshot(); if (runtime_after.state != PCM_X_RUNTIME_ACTIVE || runtime_after.gate_generation != runtime.gate_generation - || runtime_after.master_session_incarnation != runtime.master_session_incarnation) + || runtime_after.master_session_incarnation != runtime.master_session_incarnation) { + gcs_block_pcm_x_master_retry_observe("runtime-resample", PCM_X_QUEUE_NOT_READY, -1, + 0, 0, NULL, epoch_before); return; + } gcs_block_pcm_x_master_drive_retry_tick(); gcs_block_pcm_x_terminal_retry_tick(); return; @@ -9560,7 +9579,8 @@ cluster_gcs_block_pcm_x_formation_tick(void) return; fail_closed: - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, PCM_X_RUNTIME_RECOVERY_BLOCKED); + (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, + PCM_X_RUNTIME_RECOVERY_BLOCKED); } @@ -9658,7 +9678,7 @@ gcs_block_pcm_x_master_drive_fail_closed(PcmXQueueResult result) || result == PCM_X_QUEUE_BAD_STATE || result == PCM_X_QUEUE_INVALID || result == PCM_X_QUEUE_NO_CAPACITY) (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + PCM_X_RUNTIME_RECOVERY_BLOCKED); } @@ -9892,8 +9912,13 @@ gcs_block_pcm_x_master_drive_transfer(const PcmXMasterDriveSnapshot *snapshot) if (snapshot->pending_opcode == PGRAC_IC_MSG_PCM_X_COMMIT_X) { now_ms = (uint64)(GetCurrentTimestamp() / 1000); retry_delay_ms = (uint64)Max(cluster_lmon_main_loop_interval, 1); - return cluster_pcm_x_master_commit_retry_drive( + result = cluster_pcm_x_master_commit_retry_drive( snapshot, now_ms, retry_delay_ms, gcs_block_pcm_x_stage_frame_callback, NULL, &retried); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + gcs_block_pcm_x_master_retry_observe("commit-retry", result, -1, 0, 0, + &snapshot->ref.identity.tag, + snapshot->ref.identity.cluster_epoch); + return result; } if (!cluster_gcs_pcm_x_transfer_pre_handoff_phase(snapshot->pending_opcode)) return PCM_X_QUEUE_NOT_READY; @@ -10132,6 +10157,49 @@ gcs_block_pcm_x_master_drive_note(const char *stage, PcmXQueueResult result, uin } +/* One breadcrumb per stable pre-mutation refusal shape. A healthy idle LMON + * may observe NOT_FOUND forever, so the first observation logs and identical + * repeats stay silent until the exit shape changes. */ +static void +gcs_block_pcm_x_master_retry_observe(const char *stage, PcmXQueueResult result, int32 peer_node, + Size cursor_before, Size cursor_after, const BufferTag *tag, + uint64 cluster_epoch) +{ + static const char *last_stage = NULL; + static PcmXQueueResult last_result = PCM_X_QUEUE_OK; + static int32 last_peer_node = -1; + PcmXStatsSnapshot stats; + bool same; + + if (stage == NULL || result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + last_stage = NULL; + return; + } + if (!cluster_pcm_x_stats_snapshot(&stats) || stats.live_tickets == 0) { + last_stage = NULL; + return; + } + same = stage == last_stage && result == last_result && peer_node == last_peer_node; + if (same) + return; + last_stage = stage; + last_result = result; + last_peer_node = peer_node; + if (tag != NULL) + ereport(LOG, + (errmsg("PCM-X periodic retry exit (stage %s, result %d, peer %d, " + "cursor %zu->%zu, epoch %llu, rel %u, block %u)", + stage, (int)result, peer_node, cursor_before, cursor_after, + (unsigned long long)cluster_epoch, tag->relNumber, tag->blockNum))); + else + ereport(LOG, + (errmsg("PCM-X periodic retry exit (stage %s, result %d, peer %d, " + "cursor %zu->%zu, epoch %llu)", + stage, (int)result, peer_node, cursor_before, cursor_after, + (unsigned long long)cluster_epoch))); +} + + static void gcs_block_pcm_x_master_drive_tag(const BufferTag *tag, uint64 cluster_epoch) { @@ -10140,12 +10208,41 @@ gcs_block_pcm_x_master_drive_tag(const BufferTag *tag, uint64 cluster_epoch) PcmXQueueResult result; const char *drive_stage; - if (tag == NULL || cluster_epoch != cluster_epoch_get_current() || cluster_node_id < 0 - || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT - || cluster_gcs_lookup_master(*tag) != cluster_node_id - || cluster_gcs_block_phase_for_tag(*tag) == GCS_BLOCK_RECOVERING - || !cluster_qvotec_in_quorum() || !cluster_membership_is_member(cluster_node_id)) + if (tag == NULL) { + gcs_block_pcm_x_master_retry_observe("drive-precheck-tag", PCM_X_QUEUE_INVALID, -1, 0, 0, + NULL, cluster_epoch); + return; + } + if (cluster_epoch != cluster_epoch_get_current()) { + gcs_block_pcm_x_master_retry_observe("drive-precheck-epoch", PCM_X_QUEUE_STALE, -1, 0, + 0, tag, cluster_epoch); + return; + } + if (cluster_node_id < 0 || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) { + gcs_block_pcm_x_master_retry_observe("drive-precheck-node", PCM_X_QUEUE_INVALID, + cluster_node_id, 0, 0, tag, cluster_epoch); return; + } + if (cluster_gcs_lookup_master(*tag) != cluster_node_id) { + gcs_block_pcm_x_master_retry_observe("drive-precheck-master", PCM_X_QUEUE_STALE, + cluster_node_id, 0, 0, tag, cluster_epoch); + return; + } + if (cluster_gcs_block_phase_for_tag(*tag) == GCS_BLOCK_RECOVERING) { + gcs_block_pcm_x_master_retry_observe("drive-precheck-recovering", PCM_X_QUEUE_NOT_READY, + -1, 0, 0, tag, cluster_epoch); + return; + } + if (!cluster_qvotec_in_quorum()) { + gcs_block_pcm_x_master_retry_observe("drive-precheck-quorum", PCM_X_QUEUE_NOT_READY, -1, + 0, 0, tag, cluster_epoch); + return; + } + if (!cluster_membership_is_member(cluster_node_id)) { + gcs_block_pcm_x_master_retry_observe("drive-precheck-member", PCM_X_QUEUE_NOT_READY, + cluster_node_id, 0, 0, tag, cluster_epoch); + return; + } result = cluster_pcm_x_master_promote_head_exact(tag, cluster_epoch, &active); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_BUSY) { @@ -10653,11 +10750,11 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en if (result == PCM_X_QUEUE_DUPLICATE && snapshot.graph_generation != 0) goto complete_blocker_probe; result = cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&snapshot, entries, - commit->nblockers); + commit->nblockers); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK) { (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + PCM_X_RUNTIME_RECOVERY_BLOCKED); goto blocker_commit_done; } for (i = 0; i < commit->nblockers; i++) @@ -10669,7 +10766,7 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en if (graph_generation == 0) { cluster_lmd_pcm_convert_wfg_note_replace_fail(); (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + PCM_X_RUNTIME_RECOVERY_BLOCKED); goto blocker_commit_done; } cluster_lmd_pcm_convert_wfg_note_replace(); @@ -10681,7 +10778,7 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en * could delete a concurrently newer graph generation; retain evidence * and close the runtime for recovery instead. */ (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + PCM_X_RUNTIME_RECOVERY_BLOCKED); goto blocker_commit_done; } @@ -10694,7 +10791,7 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en if (!cluster_gcs_pcm_x_blocker_ack_build(&commit->ref, commit->set_generation, &ack)) { Assert(false); (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + PCM_X_RUNTIME_RECOVERY_BLOCKED); goto blocker_commit_done; } if (!cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK, source_node, &ack, @@ -10713,7 +10810,7 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en * classifier keeps BAD_STATE/STALE/NOT_READY/BUSY benign in that replay * case while structural CORRUPT still halts the runtime. */ (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + PCM_X_RUNTIME_RECOVERY_BLOCKED); goto blocker_commit_done; } gcs_block_pcm_x_master_drive_tag(&commit->ref.identity.tag, commit->ref.identity.cluster_epoch); @@ -11529,6 +11626,8 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage } if (!source_is_n && !cluster_bufmgr_copy_block_for_gcs(work->tag, &page_lsn, block_data)) { + gcs_block_pcm_x_revoke_refusal_note("materialize-copy", (int)CLUSTER_PCM_OWN_BUSY, + &revoking); gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &work->binding, buf, &revoking, true); return; @@ -11587,6 +11686,8 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage = cluster_bufmgr_pcm_own_finish_revoke_retain(buf, &revoking, page_lsn, &retained); gcs_block_pcm_x_note_own_result(own_result); if (own_result != CLUSTER_PCM_OWN_OK) { + gcs_block_pcm_x_revoke_refusal_note("materialize-finish", (int)own_result, + &revoking); gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &ready_binding, buf, &revoking, true); return; @@ -11626,6 +11727,7 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage ready_work.binding = ready_binding; ready_work.entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE; + gcs_block_pcm_x_revoke_refusal_note(NULL, 0, NULL); gcs_block_pcm_x_stage_ready_work(worker_id, &ready_work); } } @@ -11851,7 +11953,8 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi result = cluster_pcm_x_local_holder_revoke_apply_exact(revoke, source_node, source_session); cluster_pcm_x_stats_note_queue_result(result); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { - gcs_block_pcm_x_revoke_refusal_note(NULL, 0, NULL); + /* Ingress only installs/re-arms holder work. It does not prove the + * materializer advanced, so only a READY image resets the refusal streak. */ gcs_block_pcm_x_wake_registered_holders(&revoke->ref.identity.tag); } else { gcs_block_pcm_x_revoke_refusal_note("apply", (int)result, NULL); @@ -12051,13 +12154,13 @@ cluster_gcs_handle_pcm_x_final_ack_envelope(const ClusterICEnvelope *env, const if (!cluster_pcm_lock_authority_snapshot(final_ack->ref.identity.tag, &authority) || !cluster_gcs_pcm_x_grd_handoff_token_build(&token, &authority, &handoff)) { (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + PCM_X_RUNTIME_RECOVERY_BLOCKED); return; } handoff_result = cluster_pcm_lock_queue_handoff_x_exact(&handoff); if (handoff_result != PCM_X_GRD_HANDOFF_OK && handoff_result != PCM_X_GRD_HANDOFF_DUPLICATE) { (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + PCM_X_RUNTIME_RECOVERY_BLOCKED); return; } result = cluster_pcm_x_master_final_ack_finalize_exact(&token, &final_commit); @@ -12066,7 +12169,7 @@ cluster_gcs_handle_pcm_x_final_ack_envelope(const ClusterICEnvelope *env, const /* GRD already committed; retaining the engine/image evidence and * closing the runtime is the only 8.A-safe outcome. */ (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + PCM_X_RUNTIME_RECOVERY_BLOCKED); return; } (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK, @@ -12683,12 +12786,18 @@ gcs_block_pcm_x_master_drive_retry_tick(void) { static Size cursor = 0; BufferTag tag; + PcmXQueueResult result; + Size cursor_before; uint64 cluster_epoch; - if (cluster_pcm_x_master_drive_work_next(&cursor, PCM_X_MASTER_DRIVE_SCAN_BUDGET, &tag, - &cluster_epoch) - == PCM_X_QUEUE_OK) + cursor_before = cursor; + result = cluster_pcm_x_master_drive_work_next(&cursor, PCM_X_MASTER_DRIVE_SCAN_BUDGET, &tag, + &cluster_epoch); + if (result == PCM_X_QUEUE_OK) gcs_block_pcm_x_master_drive_tag(&tag, cluster_epoch); + else + gcs_block_pcm_x_master_retry_observe("work-next", result, -1, cursor_before, cursor, NULL, + cluster_epoch); } diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 78bd8fbf8d..3cfcb1a1e7 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -150,6 +150,8 @@ cluster_bufmgr_should_pcm_track(BufferDesc *buf) static XLogRecPtr cluster_gcs_clamp_ship_flush_lsn(XLogRecPtr page_lsn); static void cluster_bufmgr_pin_for_gcs_locked(BufferDesc *buf, uint32 buf_state); static void cluster_bufmgr_unpin_for_gcs(BufferDesc *buf); +static void FlushBuffer(BufferDesc *buf, SMgrRelation reln, + IOObject io_object, IOContext io_context); static ClusterPcmOwnResult cluster_pcm_own_bump_failure(BufferDesc *buf, uint64 generation, uint32 *out_flags) @@ -516,7 +518,7 @@ cluster_bufmgr_pcm_own_finish_s_release_to_n( * ordinary invalidate path nevertheless cannot unmap a pinned descriptor, * which makes every hot-block writer wait on its own pin. Bind tag->buffer * under the mapping lock, add one raw pin, then serialize all byte users with - * content EXCLUSIVE. Two exact tuple checks around WAL flush close retag, + * content EXCLUSIVE. Two exact tuple checks around the data flush close retag, * generation, reservation, and page-LSN ABA windows. Success keeps the clean * bytes BM_VALID as an N mirror; every later LockBuffer must reacquire PCM * before consuming them. VM/FSM never enter this shape because their @@ -543,7 +545,7 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, uint32 flags; uint64 token; int buf_id; - bool dirty = false; + bool needs_flush = false; volatile bool content_locked = false; if (out_page_lsn != NULL) @@ -611,20 +613,24 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, buf_state = LockBufHdr(buf); if (!cluster_pcm_own_snapshot_matches_locked(buf, &expected_s) || (buf_state & BM_VALID) == 0 - || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state) - || (buf_state & BM_IO_IN_PROGRESS) != 0) + || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) result = CLUSTER_PCM_OWN_STALE; + else if ((buf_state & BM_IO_ERROR) != 0) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + result = CLUSTER_PCM_OWN_BUSY; else { page = (Page) BufHdrGetBlock(buf); page_lsn = PageGetLSN(page); page_scn = (uint64) ((PageHeader) page)->pd_block_scn; - dirty = (buf_state & BM_DIRTY) != 0; + needs_flush = (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | + BM_CHECKPOINT_NEEDED)) != 0; } UnlockBufHdr(buf, buf_state); - if (result == CLUSTER_PCM_OWN_OK && dirty && !XLogRecPtrIsInvalid(page_lsn)) - XLogFlush(cluster_gcs_clamp_ship_flush_lsn(page_lsn)); + if (result == CLUSTER_PCM_OWN_OK && needs_flush) + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); if (result == CLUSTER_PCM_OWN_OK) { @@ -633,9 +639,13 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, if (!cluster_pcm_own_snapshot_matches_locked(buf, &expected_s) || (buf_state & BM_VALID) == 0 || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state) - || (buf_state & BM_IO_IN_PROGRESS) != 0 || PageGetLSN(page) != page_lsn) result = CLUSTER_PCM_OWN_STALE; + else if ((buf_state & BM_IO_ERROR) != 0) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0 + || (buf_state & BM_IO_IN_PROGRESS) != 0) + result = CLUSTER_PCM_OWN_BUSY; else { result = cluster_pcm_own_bump_locked(buf, 0, 0, NULL, NULL); @@ -3140,8 +3150,6 @@ static BufferDesc *BufferAlloc(SMgrRelation smgr, BufferAccessStrategy strategy, bool *foundPtr, IOContext io_context); static Buffer GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context); -static void FlushBuffer(BufferDesc *buf, SMgrRelation reln, - IOObject io_object, IOContext io_context); static void FindAndDropRelationBuffers(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber nForkBlock, @@ -8061,12 +8069,16 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) #ifdef USE_PGRAC_CLUSTER /* - * Hint changes are optional. A retained image may already have had an - * in-memory hint bit touched by its caller, but it must never regain - * dirty state and become eligible for stale output. + * Hint changes are optional. A tracked buffer without live S/X current + * authority may already have had its in-memory hint bit touched by the + * caller, but it must never gain writeback eligibility. In particular, + * DRAIN can leave a pinned retained PI as a clean N mirror; dirtying that + * stale mirror would both block the next storage refresh and make it + * eligible to overwrite newer shared-storage bytes. */ retained_state = LockBufHdr(bufHdr); - if (cluster_bufmgr_pcm_x_retained_image_locked(bufHdr, retained_state)) + if (cluster_pcm_is_active() && cluster_bufmgr_should_pcm_track(bufHdr) + && !cluster_bufmgr_pcm_current_image_locked(bufHdr, retained_state)) { UnlockBufHdr(bufHdr, retained_state); return; @@ -9999,7 +10011,8 @@ cluster_bufmgr_read_storage_scn_for_gcs(BufferTag tag, SCN *out_page_scn) /* * Copy the 8KB block bytes for `tag` into *dst, flushing WAL up to the - * page's LSN before reading the bytes (HC82 I-WAL-before-ship). Sets + * page's LSN before reading the bytes (HC82 I-WAL-before-ship), then making + * any dirty source current in shared storage before it may be retired. Sets * *out_page_lsn to the page LSN observed at the second-stable revalidation. * * Returns false on: @@ -10010,8 +10023,12 @@ cluster_bufmgr_read_storage_scn_for_gcs(BufferTag tag, SCN *out_page_scn) * * Concurrency: caller does NOT hold any buffer/partition locks. We take a * SHARED partition lock to look up, raw-pin the shared buffer refcount, then - * drop the partition lock and operate on the buffer's content_lock. The raw - * pin is always released before return. + * drop the partition lock and conditionally acquire the buffer's + * content_lock. GCS DATA workers must keep draining replies while a backend + * owns the content lock and waits for one of those replies; waiting here + * would deadlock that backend against its own receive worker. A busy lock is + * therefore a retryable no-image result. The raw pin is always released + * before return. */ bool cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char *dst) @@ -10025,6 +10042,8 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char XLogRecPtr second_lsn; int retries; bool stable; + bool needs_flush; + bool storage_current; Page page; Assert(dst != NULL); @@ -10076,8 +10095,10 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char stable = false; for (retries = 0; retries < 2; retries++) { - /* Read page_lsn under content_lock SHARED. */ - LWLockAcquire(content_lock, LW_SHARED); + /* Read page_lsn under content_lock SHARED. Never park a GCS DATA + * worker behind a backend that may itself be waiting for this worker. */ + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) + break; first_lsn = PageGetLSN(page); LWLockRelease(content_lock); @@ -10100,14 +10121,20 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char * Either signals concurrent mutation that would break HC82's "ship * the bytes that I just flushed WAL for" contract. */ - LWLockAcquire(content_lock, LW_SHARED); + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) + break; { uint32 buf_state = LockBufHdr(buf); bool current = BufferTagsEqual(&buf->tag, &tag) && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); + needs_flush = current + && (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | + BM_CHECKPOINT_NEEDED)) != 0; + storage_current = current && (buf_state & BM_IO_ERROR) == 0; + UnlockBufHdr(buf, buf_state); - if (!current) + if (!storage_current) { LWLockRelease(content_lock); break; @@ -10131,9 +10158,46 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char } #endif - if (BufferTagsEqual(&buf->tag, &tag) && first_lsn == second_lsn) + if (first_lsn == second_lsn) { + uint32 buf_state; + + /* A queue handoff may retire this descriptor immediately after + * copying it. Make the shared-storage fallback at least as current + * as the shipped image before publishing that handoff watermark. */ + if (needs_flush) + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + + buf_state = LockBufHdr(buf); + storage_current = BufferTagsEqual(&buf->tag, &tag) + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state) + && PageGetLSN(page) == second_lsn + && (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | + BM_IO_ERROR | BM_IO_IN_PROGRESS)) == 0; + UnlockBufHdr(buf, buf_state); + if (!storage_current) + { + LWLockRelease(content_lock); + continue; + } + memcpy(dst, page, BLCKSZ); + + /* Hint-bit dirties may occur under a shared content lock. Do not + * certify a copy if one raced the memcpy after the first clean check. */ + buf_state = LockBufHdr(buf); + storage_current = BufferTagsEqual(&buf->tag, &tag) + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state) + && PageGetLSN(page) == second_lsn + && (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | + BM_IO_ERROR | BM_IO_IN_PROGRESS)) == 0; + UnlockBufHdr(buf, buf_state); + if (!storage_current) + { + LWLockRelease(content_lock); + continue; + } + *out_page_lsn = second_lsn; LWLockRelease(content_lock); stable = true; @@ -10212,7 +10276,8 @@ cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page for (retries = 0; retries < 2; retries++) { - LWLockAcquire(content_lock, LW_SHARED); + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) + break; first_lsn = PageGetLSN(page); LWLockRelease(content_lock); @@ -10223,7 +10288,8 @@ cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page if (!XLogRecPtrIsInvalid(first_lsn)) XLogFlush(cluster_gcs_clamp_ship_flush_lsn(first_lsn)); - LWLockAcquire(content_lock, LW_SHARED); + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) + break; { uint32 buf_state = LockBufHdr(buf); bool current = BufferTagsEqual(&buf->tag, &tag) @@ -10397,7 +10463,14 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) LWLockRelease(partition_lock); content_lock = BufferDescriptorGetContentLock(buf); - LWLockAcquire(content_lock, LW_EXCLUSIVE); + /* This helper is called by the GCS DATA worker for FORWARD/BAST. A + * blocking content-lock wait can prevent the same worker from receiving + * the reply awaited by the lock owner, so refusal must be immediate. */ + if (!LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) + { + cluster_bufmgr_unpin_for_gcs(buf); + return false; + } if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) { LWLockRelease(content_lock); @@ -10621,7 +10694,14 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) LWLockRelease(partition_lock); content_lock = BufferDescriptorGetContentLock(buf); - LWLockAcquire(content_lock, LW_EXCLUSIVE); + /* Same receive-worker liveness rule as the master==holder downgrade: a + * busy local writer makes this downgrade request retryable, never a reason + * to park the DATA dispatch loop. */ + if (!LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) + { + cluster_bufmgr_unpin_for_gcs(buf); + return false; + } if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) { LWLockRelease(content_lock); @@ -10792,7 +10872,8 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa stable = false; for (retries = 0; retries < 2; retries++) { - LWLockAcquire(content_lock, LW_SHARED); + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) + break; { uint32 buf_state = LockBufHdr(buf); bool current = BufferTagsEqual(&buf->tag, &tag) @@ -12148,6 +12229,7 @@ cluster_bufmgr_pcm_own_finish_revoke_retain( uint32 buf_state; bool source_is_s; bool source_is_x; + bool needs_flush = false; ClusterPcmXRevokeFinishMode finish_mode; if (out_retained != NULL) @@ -12184,6 +12266,8 @@ cluster_bufmgr_pcm_own_finish_revoke_retain( result = CLUSTER_PCM_OWN_CORRUPT; else if ((buf_state & BM_IO_IN_PROGRESS) != 0) result = CLUSTER_PCM_OWN_BUSY; + else if ((buf_state & BM_IO_ERROR) != 0) + result = CLUSTER_PCM_OWN_CORRUPT; else { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); @@ -12199,13 +12283,57 @@ cluster_bufmgr_pcm_own_finish_revoke_retain( result = CLUSTER_PCM_OWN_STALE; else if (PageGetLSN((Page) BufHdrGetBlock(buf)) != expected_lsn) result = CLUSTER_PCM_OWN_STALE; + else + needs_flush + = (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0; } + UnlockBufHdr(buf, buf_state); if (result == CLUSTER_PCM_OWN_OK) { - result = cluster_pcm_own_revoke_retain_commit_exact( - buf->buf_id, expected_revoking->generation, - expected_revoking->reservation_token, &committed_generation); + /* A hint-bit update may legally dirty the page under the shared + * content lock after the immutable A-record copy. Under EXCLUSIVE no + * further byte change can race us: flush that same-LSN image, then + * re-prove both the ownership token and a clean storage fallback before + * retiring the descriptor. */ + if (needs_flush) + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, &tag) + || (buf_state & BM_VALID) == 0 + || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation + || buf->pcm_state != expected_revoking->pcm_state) + result = CLUSTER_PCM_OWN_STALE; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + result = CLUSTER_PCM_OWN_BUSY; + else if ((buf_state & BM_IO_ERROR) != 0) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) + result = CLUSTER_PCM_OWN_BUSY; + else + { + live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if (flags == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (flags != PCM_OWN_FLAG_REVOKING) + result = CLUSTER_PCM_OWN_BUSY; + else if (live_token != expected_revoking->reservation_token) + result = CLUSTER_PCM_OWN_STALE; + else if (PageGetLSN((Page) BufHdrGetBlock(buf)) != expected_lsn) + result = CLUSTER_PCM_OWN_STALE; + } + + if (result == CLUSTER_PCM_OWN_OK) + result = cluster_pcm_own_revoke_retain_commit_exact( + buf->buf_id, expected_revoking->generation, + expected_revoking->reservation_token, &committed_generation); if (result == CLUSTER_PCM_OWN_OK) { buf->pcm_state = (uint8) PCM_STATE_N; @@ -12214,9 +12342,9 @@ cluster_bufmgr_pcm_own_finish_revoke_retain( BM_CHECKPOINT_NEEDED | BM_IO_ERROR); cluster_pcm_own_snapshot_locked(buf, out_retained); } + UnlockBufHdr(buf, buf_state); } - UnlockBufHdr(buf, buf_state); LWLockRelease(content_lock); return result; } diff --git a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl index cf8b4d67a6..e2152aec0b 100644 --- a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl +++ b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl @@ -98,8 +98,6 @@ sub state_int my @zero_pcm_failure_keys = qw( pcm_x_queue_cancel_count pcm_x_queue_full_count - pcm_x_queue_stale_count - pcm_x_queue_miss_count pcm_x_queue_recovery_blocked_count pcm_x_queue_activating_reset_count pcm_x_own_abort_count @@ -108,7 +106,6 @@ sub state_int my @positive_wfg_keys = qw( pcm_convert_wfg_replace_count - pcm_convert_wfg_remove_count ); sub exact_key_count @@ -591,9 +588,24 @@ sub write_file 'x_vs_s_no_carrier_denied_count', 'block_invalidate_broadcast_count', 'block_invalidate_ack_received_count', + 'invalidate_send_queued_count', 'invalidate_passive_s_release_count', + 'invalidate_parked_count', + 'invalidate_busy_sent_count', + 'invalidate_busy_received_count', + 'invalidate_park_expired_count', + 'invalidate_park_overflow_count', 'pcm_x_self_handoff_count', 'pcm_x_self_handoff_drain_count', + 'dedup_pcm_x_stage_count', + 'dedup_pcm_x_replay_count', + 'dedup_pcm_x_release_count', + 'dedup_pcm_x_failclosed_count', + 'stale_reply_drop_count', + 'block_checksum_fail_count', + 'block_forward_received_count', + 'block_from_holder_ship_count', + 'cf_xheld_read_ship_count', 'invalidate_send_not_admitted_count', 'forward_send_not_admitted_count', 'reply_send_not_admitted_count') @@ -604,11 +616,37 @@ sub write_file ', ' ORDER BY category, key) FROM pg_cluster_state WHERE (category = 'cr' AND key IN ( + 'rtvis_undo_fetch_wire_count', + 'rtvis_undo_fetch_cache_hit_count', + 'rtvis_undo_fetch_failclosed_count', + 'rtvis_resolve_committed_count', + 'rtvis_resolve_aborted_count', + 'rtvis_resolve_failclosed_count', 'rtvis_verdict_wire_count', 'rtvis_verdict_failclosed_count', 'rtvis_verdict_exact_count', 'rtvis_verdict_below_horizon_count', - 'rtvis_verdict_inadmissible_count')) + 'rtvis_verdict_inadmissible_count', + 'rtvis_underivable_failclosed_count', + 'cr_server_verdict_served_count', + 'cr_server_verdict_denied_count', + 'cr_server_fence_refused_count', + 'undo_authority_serve_hit_count', + 'undo_authority_fail_closed_count', + 'undo_authority_epoch_stale_reject_count', + 'undo_authority_scan_incomplete_reject_count', + 'undo_authority_multi_match_reject_count', + 'vis53r97_leg_invalid_scn_refuse_count', + 'vis53r97_leg_zero_match_refuse_count', + 'vis53r97_leg_srv_other_refuse_count', + 'vis53r97_leg_covers_refuse_count', + 'vis53r97_leg_multi_unresolvable_count', + 'vis53r97_leg_xmax_unprovable_count', + 'vis53r97_leg_xmin_overlay_verdict_ask_count', + 'vis53r97_leg_xmin_overlay_verdict_hit_count', + 'vis53r97_leg_multi_member_serve_ask_count', + 'vis53r97_leg_multi_member_serve_hit_count', + 'vis53r97_leg_live_upgrade_hit_count')) OR (category = 'xnode' AND key IN ( 'c_resolve_count', 'c_tt_lookup_count', 'c_memo_hit_count', 'c_memo_install_count')) @@ -623,12 +661,12 @@ sub write_file is($runs[$i]->{errors}, 0, "L3 node$i writer surfaced zero client errors"); cmp_ok($runs[$i]->{transactions}, '>', 0, "L3 node$i writer made progress"); - cmp_ok( - $pcm_after_by_node[$i]->{pcm_x_queue_enqueue_count} - - $pcm_before_by_node[$i]->{pcm_x_queue_enqueue_count}, - '>', 0, "L3 node$i PCM-X enqueue delta proves this requester used the queue"); } +# Queue counters live on the static tag master, not on each requester. The +# aggregate lifecycle plus every writer's committed progress proves that all +# four requesters traversed the protocol; requiring a local enqueue delta on +# non-master nodes is a false topology assumption. cmp_ok($queue_after - $queue_before, '>=', 4, 'L3 all four node writers entered the PCM-X queue protocol'); cmp_ok($denied_after - $denied_before, '>', 0, @@ -645,11 +683,18 @@ sub write_file is($pcm_after{$key} - $pcm_before{$key}, 0, "L3 PCM-X failure counter $key stayed zero"); } +# STALE/NOT_FOUND are retry classifications used by generation-exact role +# refresh and idempotent replay. They are observable churn, not terminal +# failures; zero terminal gauges, zero client errors, and L4 conservation are +# the authoritative safety/liveness verdicts. for my $key (@positive_wfg_keys) { cmp_ok($lmd_after{$key} - $lmd_before{$key}, '>', 0, "L3 PCM-X WFG lifecycle $key advanced"); } +# Atomic replacement with a zero-blocker set removes the old waiter edges but +# intentionally counts as a replace, so an explicit remove call is not +# required on every successful run. is($lmd_after{pcm_convert_wfg_replace_fail_count} - $lmd_before{pcm_convert_wfg_replace_fail_count}, 0, 'L3 PCM-X WFG atomic replace failures stayed zero'); diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 3cb28212d5..0506d5dec9 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -1200,6 +1200,68 @@ UT_TEST(test_pcm_x_formation_transient_or_inconsistent_sample_is_tick_noop) } +/* P0-20: a post-handoff COMMIT_X ticket can remain freshly armed while the + * production LMON path silently exits before retry mutation. Every + * pre-mutation boundary must therefore leave one bounded, stage-specific + * breadcrumb; otherwise t/400 cannot distinguish formation rejection from a + * missed work scan or a drive precheck refusal. */ +UT_TEST(test_pcm_x_periodic_retry_reports_pre_mutation_exit_stage) +{ + char *source = read_gcs_block_source(); + const char *formation; + const char *formation_end; + const char *retry_tick; + const char *retry_tick_end; + const char *drive; + const char *drive_end; + const char *transfer; + const char *transfer_end; + + UT_ASSERT_NOT_NULL(source); + if (source != NULL) { + formation = strstr(source, "\ncluster_gcs_block_pcm_x_formation_tick("); + formation_end = formation != NULL ? strstr(formation, "\nfail_closed:") : NULL; + UT_ASSERT_NOT_NULL(formation); + UT_ASSERT_NOT_NULL(formation_end); + if (formation != NULL && formation_end != NULL) { + UT_ASSERT_NOT_NULL(strstr(formation, "\"collect-before\"")); + UT_ASSERT_NOT_NULL(strstr(formation, "\"collect-after\"")); + UT_ASSERT_NOT_NULL(strstr(formation, "\"stability\"")); + UT_ASSERT_NOT_NULL(strstr(formation, "\"peer-revalidate\"")); + UT_ASSERT_NOT_NULL(strstr(formation, "\"runtime-resample\"")); + } + + retry_tick = strstr(source, "\ngcs_block_pcm_x_master_drive_retry_tick("); + retry_tick_end = retry_tick != NULL ? strstr(retry_tick, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(retry_tick); + UT_ASSERT_NOT_NULL(retry_tick_end); + if (retry_tick != NULL && retry_tick_end != NULL) { + UT_ASSERT_NOT_NULL(strstr(retry_tick, "\"work-next\"")); + UT_ASSERT_NOT_NULL(strstr(retry_tick, "cursor_before")); + } + UT_ASSERT_NOT_NULL(strstr(source, "cluster_pcm_x_stats_snapshot(&stats)")); + UT_ASSERT_NOT_NULL(strstr(source, "stats.live_tickets == 0")); + + drive = strstr(source, "\ngcs_block_pcm_x_master_drive_tag("); + drive_end = drive != NULL ? strstr(drive, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(drive); + UT_ASSERT_NOT_NULL(drive_end); + if (drive != NULL && drive_end != NULL) { + UT_ASSERT_NOT_NULL(strstr(drive, "\"drive-precheck-")); + } + + transfer = strstr(source, "\ngcs_block_pcm_x_master_drive_transfer("); + transfer_end = transfer != NULL ? strstr(transfer, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(transfer); + UT_ASSERT_NOT_NULL(transfer_end); + if (transfer != NULL && transfer_end != NULL) { + UT_ASSERT_NOT_NULL(strstr(transfer, "\"commit-retry\"")); + } + free(source); + } +} + + /* review R2 P0-1: a SELF-loopback V2 INSTALL_READY (master == requester * node) has no HELLO capability record; the handler must treat self as * rebase-capable (the local binary) while still requiring the @@ -3666,15 +3728,31 @@ UT_TEST(test_revoke_handler_silent_refusal_arms_all_note) { char *source = read_gcs_block_source(); const char *handler = strstr(source, "\ncluster_gcs_handle_pcm_x_revoke_envelope("); + const char *materialize = strstr(source, "\ngcs_block_pcm_x_materialize_reserved_work("); + const char *publish; + const char *reset; /* The master re-sends REVOKE forever, so every refusal arm in the source * handler must name itself through the log-once streak note (t/400 form-B * wedges previously refused silently at un-instrumented arms): ingress * auth, holder-ledger stale, holder-progress error, image reserve, apply, - * ingress snapshot, plus the progress reset. */ + * and ingress snapshot. Merely accepting/re-arming work is not progress; + * the streak resets only after exact READY publication. */ UT_ASSERT_NOT_NULL(handler); if (handler != NULL) - UT_ASSERT(count_occurrences(handler, "gcs_block_pcm_x_revoke_refusal_note(") >= 7); + UT_ASSERT(count_occurrences(handler, "gcs_block_pcm_x_revoke_refusal_note(") >= 6); + UT_ASSERT_NOT_NULL(materialize); + if (materialize != NULL) { + UT_ASSERT_NOT_NULL(strstr(materialize, "\"materialize-copy\"")); + UT_ASSERT_NOT_NULL(strstr(materialize, "\"materialize-finish\"")); + } + publish = materialize != NULL + ? strstr(materialize, "cluster_gcs_block_dedup_pcm_x_publish_ready_exact(") + : NULL; + reset = publish != NULL ? strstr(publish, "gcs_block_pcm_x_revoke_refusal_note(NULL, 0, NULL)") + : NULL; + UT_ASSERT_NOT_NULL(publish); + UT_ASSERT_NOT_NULL(reset); free(source); } @@ -3682,7 +3760,7 @@ UT_TEST(test_revoke_handler_silent_refusal_arms_all_note) int main(void) { - UT_PLAN(82); + UT_PLAN(83); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -3721,6 +3799,7 @@ main(void) UT_RUN(test_pcm_x_blocker_ack_carries_full_generation_and_binds_master_source); UT_RUN(test_pcm_x_formation_identical_complete_samples_may_revalidate); UT_RUN(test_pcm_x_formation_transient_or_inconsistent_sample_is_tick_noop); + UT_RUN(test_pcm_x_periodic_retry_reports_pre_mutation_exit_stage); UT_RUN(test_pcm_x_install_ready_v2_self_loopback_is_admissible); UT_RUN(test_pcm_x_formation_samples_capability_family_atomically); UT_RUN(test_pcm_x_confirm_publish_then_stale_requires_exact_graph_close); diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index 5c030bc333..7932f3a59f 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -1203,6 +1203,14 @@ UT_TEST(test_queue_revoke_retains_main_but_drops_unpinned_vm_fsm) "cluster_bufmgr_pcm_current_image_locked", "BM_IO_IN_PROGRESS", "PageGetLSN", + "UnlockBufHdr", + "FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL)", + "LockBufHdr", + "cluster_pcm_own_gen_get", + "cluster_bufmgr_pcm_current_image_locked", + "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED", + "result = CLUSTER_PCM_OWN_BUSY", + "PageGetLSN", "cluster_pcm_own_revoke_retain_commit_exact", "buf->pcm_state = (uint8) PCM_STATE_N", "buf->buffer_type = (uint8) BUF_TYPE_PI", @@ -1384,9 +1392,22 @@ UT_TEST(test_retained_image_release_and_writeback_gates_are_exact) if (dirty != NULL) UT_ASSERT(strstr(dirty, "cluster_bufmgr_pcm_x_retained_image_locked") < strstr(dirty, "buf_state |= BM_DIRTY")); - if (hint != NULL) - UT_ASSERT(strstr(hint, "cluster_bufmgr_pcm_x_retained_image_locked") - < strstr(hint, "BM_DIRTY | BM_JUST_DIRTIED")); + if (hint != NULL) { + const char *tracked = strstr(hint, "cluster_bufmgr_should_pcm_track(bufHdr)"); + const char *current = strstr(hint, "cluster_bufmgr_pcm_current_image_locked"); + const char *refuse = current != NULL ? strstr(current, "return;") : NULL; + const char *dirty_flags = strstr(hint, "BM_DIRTY | BM_JUST_DIRTIED"); + + /* Hint dirt is optional. A kept pinned N/PI mirror may have its + * in-memory hint byte touched, but without live S/X current authority it + * must not regain writeback eligibility and block a later storage refresh. */ + UT_ASSERT_NOT_NULL(tracked); + UT_ASSERT_NOT_NULL(current); + UT_ASSERT_NOT_NULL(refuse); + UT_ASSERT_NOT_NULL(dirty_flags); + if (tracked != NULL && current != NULL && refuse != NULL && dirty_flags != NULL) + UT_ASSERT(tracked < current && current < refuse && refuse < dirty_flags); + } if (lockbuffer != NULL) { const char *reserve = strstr(lockbuffer, "cluster_bufmgr_pcm_begin_grant_reservation_wait("); @@ -1816,9 +1837,11 @@ UT_TEST(test_queue_passive_pinned_s_release_serializes_bytes_and_ownership) "LWLockAcquire(content_lock, LW_EXCLUSIVE)", "cluster_pcm_own_snapshot_matches_locked", "PageGetLSN", - "XLogFlush", + "FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL)", "LockBufHdr", "cluster_pcm_own_snapshot_matches_locked", + "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED", + "result = CLUSTER_PCM_OWN_BUSY", "cluster_pcm_own_bump_locked", "buf->pcm_state = (uint8) PCM_STATE_N", "buf->buffer_type = (uint8) BUF_TYPE_PI", @@ -1864,23 +1887,25 @@ UT_TEST(test_queue_passive_n_mirror_is_never_gcs_ship_authority) static const char *const probe_contract[] = { "LockBufHdr", "cluster_bufmgr_pcm_current_image_locked", "UnlockBufHdr" }; static const char *const copy_contract[] = { "LockBufHdr", + "cluster_bufmgr_pcm_current_image_locked", + "cluster_bufmgr_pin_for_gcs_locked", + "LWLockConditionalAcquire(content_lock, LW_SHARED)", + "cluster_bufmgr_pcm_current_image_locked", + "FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL)", + "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED", + "memcpy(dst, page, BLCKSZ)" }; + static const char *const live_sge_contract[] = { "LockBufHdr", "cluster_bufmgr_pcm_current_image_locked", "cluster_bufmgr_pin_for_gcs_locked", - "LWLockAcquire(content_lock, LW_SHARED)", + "LWLockConditionalAcquire(content_lock, LW_SHARED)", "cluster_bufmgr_pcm_current_image_locked", - "memcpy(dst, page, BLCKSZ)" }; - static const char *const live_sge_contract[] = { "LockBufHdr", - "cluster_bufmgr_pcm_current_image_locked", - "cluster_bufmgr_pin_for_gcs_locked", - "LWLockAcquire(content_lock, LW_SHARED)", - "cluster_bufmgr_pcm_current_image_locked", - "*out_page_addr = page" }; + "*out_page_addr = page" }; static const char *const smart_contract[] = { "LockBufHdr", - "cluster_bufmgr_pcm_current_image_locked", - "cluster_bufmgr_pin_for_gcs_locked", - "LWLockAcquire(content_lock, LW_SHARED)", - "cluster_bufmgr_pcm_current_image_locked", - "memcpy(dst, page, BLCKSZ)" }; + "cluster_bufmgr_pcm_current_image_locked", + "cluster_bufmgr_pin_for_gcs_locked", + "LWLockConditionalAcquire(content_lock, LW_SHARED)", + "cluster_bufmgr_pcm_current_image_locked", + "memcpy(dst, page, BLCKSZ)" }; char *source = read_bufmgr_source(); assert_ordered_in_function(source, "\ncluster_bufmgr_probe_block_for_gcs(", From 505517fe6faf6e383006d5f7623d2a49c3807697 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 22 Jul 2026 20:32:16 +0800 Subject: [PATCH 55/56] fix(cluster): close four-node PCM-X liveness campaign Stabilize exact PCM-X queue ownership, transfer, retirement, visibility, and fail-closed handling across concurrent four-node writers. Add deterministic unit and TAP coverage for data-plane routing, dirty-image retention, queue drainage, data conservation, and injected FlushBuffer failures. Validation: t/400 228/228 PASS and ten consecutive acceptance runs. --- src/backend/access/heap/heapam.c | 114 +- src/backend/access/heap/heapam_visibility.c | 73 +- src/backend/access/heap/hio.c | 46 +- src/backend/access/heap/visibilitymap.c | 26 + src/backend/cluster/cluster_cr_server.c | 2 +- src/backend/cluster/cluster_debug.c | 9 +- src/backend/cluster/cluster_gcs_block.c | 2226 ++++++++++++++--- src/backend/cluster/cluster_gcs_block_dedup.c | 362 ++- src/backend/cluster/cluster_gcs_block_shard.c | 7 +- src/backend/cluster/cluster_ic.c | 3 + src/backend/cluster/cluster_inject.c | 23 + src/backend/cluster/cluster_lms.c | 61 + src/backend/cluster/cluster_lms_outbound.c | 81 +- src/backend/cluster/cluster_pcm_lock.c | 288 ++- src/backend/cluster/cluster_pcm_own.c | 100 +- src/backend/cluster/cluster_pcm_x_convert.c | 330 ++- .../cluster/cluster_pcm_x_image_fetch.c | 107 +- src/backend/cluster/cluster_sf_dep.c | 57 + src/backend/cluster/cluster_tt_local.c | 231 +- .../cluster/cluster_visibility_verdict.c | 25 + src/backend/storage/buffer/bufmgr.c | 1914 +++++++++++--- src/include/access/visibilitymap.h | 2 + src/include/cluster/cluster_gcs_block.h | 123 +- src/include/cluster/cluster_gcs_block_dedup.h | 64 +- src/include/cluster/cluster_ic.h | 3 + src/include/cluster/cluster_inject.h | 3 + src/include/cluster/cluster_lms.h | 13 + src/include/cluster/cluster_pcm_lock.h | 35 +- src/include/cluster/cluster_pcm_own.h | 25 +- src/include/cluster/cluster_pcm_x_bufmgr.h | 65 +- src/include/cluster/cluster_pcm_x_convert.h | 61 +- .../cluster/cluster_pcm_x_image_fetch.h | 16 + src/include/cluster/cluster_sf_dep.h | 19 + src/include/cluster/cluster_tt_local.h | 10 + .../cluster/cluster_visibility_resolve.h | 4 + src/test/cluster_tap/t/015_inject.pl | 6 +- src/test/cluster_tap/t/017_debug.pl | 8 +- src/test/cluster_tap/t/030_acceptance.pl | 8 +- .../t/400_pcm_x_queue_4node_liveness.pl | 375 ++- src/test/cluster_unit/Makefile | 17 +- .../test_cluster_bufmgr_pcm_hook.c | 10 +- src/test/cluster_unit/test_cluster_debug.c | 14 +- .../cluster_unit/test_cluster_gcs_block.c | 1562 +++++++++++- .../test_cluster_gcs_block_dedup.c | 270 +- .../test_cluster_gcs_block_shard.c | 33 +- src/test/cluster_unit/test_cluster_ic.c | 16 +- .../cluster_unit/test_cluster_lms_outbound.c | 153 +- .../test_cluster_pcm_direct_init.c | 157 +- src/test/cluster_unit/test_cluster_pcm_lock.c | 589 ++++- src/test/cluster_unit/test_cluster_pcm_own.c | 473 +++- .../cluster_unit/test_cluster_pcm_x_convert.c | 439 +++- .../test_cluster_pcm_x_image_fetch.c | 17 +- src/test/cluster_unit/test_cluster_sf_dep.c | 18 + .../test_cluster_visibility_fork.c | 138 + .../test_cluster_visibility_variants.c | 40 + 55 files changed, 9632 insertions(+), 1239 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 4cd6a60003..20572529c6 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -1719,12 +1719,12 @@ heap_hot_search_buffer(ItemPointer tid, Relation relation, Buffer buffer, if ( #ifdef USE_PGRAC_CLUSTER - /* spec-3.14 D5b: HeapTupleIsSurelyDead has no Buffer and - * bypasses VacuumHorizon; guard the HOT all_dead probe here - * so a remote-evidence chain is never declared all-dead. */ - (cluster_storage_mode_enabled() - && cluster_tuple_has_remote_evidence(buffer, heapTuple->t_data)) - ? true : /* remote evidence -> not surely dead -> *all_dead=false */ + /* P0-28: no cluster-wide prune horizon; even a recycled ITL + * slot cannot turn a shared-storage HOT member surely dead. */ + cluster_vis_prune_must_defer(cluster_storage_mode_enabled() + && !RelationUsesLocalBuffers(relation), + false) + ? true : #endif !HeapTupleIsSurelyDead(heapTuple, vistest)) *all_dead = false; @@ -2323,6 +2323,11 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, { ClusterItlTouchHandle handle; + /* P0-33: publish the exact data-writer binding as IN_PROGRESS only + * after the tuple + ACTIVE ITL stamp is WAL-protected, and before the + * content lock is released. */ + cluster_tt_local_record_data_active(xid, cluster_itl_uba); + handle.rloc = relation->rd_locator; handle.block = ItemPointerGetBlockNumber(&heaptup->t_self); handle.forknum = MAIN_FORKNUM; @@ -2851,6 +2856,10 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, { ClusterItlTouchHandle handle; + /* P0-33: pair this page's fresh ACTIVE data ref with its exact + * IN_PROGRESS overlay/hint before releasing the content lock. */ + cluster_tt_local_record_data_active(xid, cluster_mi_uba); + handle.rloc = relation->rd_locator; handle.block = BufferGetBlockNumber(buffer); handle.forknum = MAIN_FORKNUM; @@ -3463,6 +3472,50 @@ cluster_heap_vm_barrier_warm(Buffer vmbuf) LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK); } +/* + * Acquire heap content X without carrying a visibility-map pin across the + * cluster PCM wait. VM/FSM pages cannot use the passive-pin retained-PI + * shape because their callers may read pinned bytes without a content lock. + * Keeping this pin while another node transfers the VM page therefore makes + * the source's exact-drop finish return BUSY forever. + * + * visibilitymap_pin() runs before heap content authority and may do I/O. We + * then release that pin before LockBuffer can wait. Once heap X is held, + * visibilitymap_pin_recent() may only pin the exact still-resident descriptor; + * it cannot do I/O. A retag/drop race releases heap X and repeats the safe + * pre-read sequence. + */ +static void +cluster_heap_lock_with_vm_repin(Relation relation, BlockNumber heap_block, + Buffer heap_buffer, Buffer *vmbuffer) +{ + Buffer recent_vm = InvalidBuffer; + Page heap_page; + + Assert(relation != NULL); + Assert(BufferIsValid(heap_buffer)); + Assert(vmbuffer != NULL); + heap_page = BufferGetPage(heap_buffer); + + for (;;) + { + if (PageIsAllVisible(heap_page) && !BufferIsValid(*vmbuffer)) + visibilitymap_pin(relation, heap_block, vmbuffer); + if (BufferIsValid(*vmbuffer)) + { + recent_vm = *vmbuffer; + ReleaseBuffer(*vmbuffer); + *vmbuffer = InvalidBuffer; + } + + LockBuffer(heap_buffer, BUFFER_LOCK_EXCLUSIVE); + if (!PageIsAllVisible(heap_page) + || visibilitymap_pin_recent(relation, heap_block, recent_vm, vmbuffer)) + return; + LockBuffer(heap_buffer, BUFFER_LOCK_UNLOCK); + } +} + TM_Result heap_delete(Relation relation, ItemPointer tid, @@ -3907,7 +3960,9 @@ heap_delete(Relation relation, ItemPointer tid, } LockBuffer(buffer, BUFFER_LOCK_UNLOCK); cluster_heap_vm_barrier_warm(vmbuffer); - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + ReleaseBuffer(vmbuffer); + vmbuffer = InvalidBuffer; + cluster_heap_lock_with_vm_repin(relation, block, buffer, &vmbuffer); goto l1; } vm_locked = true; @@ -4066,6 +4121,10 @@ heap_delete(Relation relation, ItemPointer tid, { ClusterItlTouchHandle handle; + /* P0-33: publish the exact ACTIVE data-writer identity while the + * freshly stamped page is still protected by its content lock. */ + cluster_tt_local_record_data_active(xid, cluster_itl_uba); + handle.rloc = relation->rd_locator; handle.block = BufferGetBlockNumber(buffer); handle.forknum = MAIN_FORKNUM; @@ -4301,10 +4360,14 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, * in the middle of changing this, so we'll need to recheck after we have * the lock. */ +#ifdef USE_PGRAC_CLUSTER + cluster_heap_lock_with_vm_repin(relation, block, buffer, &vmbuffer); +#else if (PageIsAllVisible(page)) visibilitymap_pin(relation, block, &vmbuffer); LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); +#endif lp = PageGetItemId(page, ItemPointerGetOffsetNumber(otid)); @@ -4732,8 +4795,7 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, if (vmbuffer == InvalidBuffer && PageIsAllVisible(page)) { LockBuffer(buffer, BUFFER_LOCK_UNLOCK); - visibilitymap_pin(relation, block, &vmbuffer); - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + cluster_heap_lock_with_vm_repin(relation, block, buffer, &vmbuffer); goto l2; } @@ -4890,7 +4952,9 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, iscombo = false; LockBuffer(buffer, BUFFER_LOCK_UNLOCK); cluster_heap_vm_barrier_warm(vmbuffer); - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + ReleaseBuffer(vmbuffer); + vmbuffer = InvalidBuffer; + cluster_heap_lock_with_vm_repin(relation, block, buffer, &vmbuffer); goto l2; } vm_locked = true; @@ -5351,7 +5415,9 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, iscombo = false; LockBuffer(buffer, BUFFER_LOCK_UNLOCK); cluster_heap_vm_barrier_warm(refused_vm); - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + ReleaseBuffer(vmbuffer); + vmbuffer = InvalidBuffer; + cluster_heap_lock_with_vm_repin(relation, block, buffer, &vmbuffer); goto l2; } @@ -5506,7 +5572,29 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, if (vm_locked_new) LockBuffer(vmbuffer_new, BUFFER_LOCK_UNLOCK); + /* VM pages require a zero-pin exact DROP at remote source handoff. Do not + * carry these pins into the heap writer's queue release: that release may + * wait for DRAIN after a VM successor has already frozen this node as its + * source, creating a VM-drop -> pin -> heap-DRAIN cycle. Every VM byte use + * is complete once the VM content locks above are released. */ + if (BufferIsValid(vmbuffer_new)) + { + ReleaseBuffer(vmbuffer_new); + vmbuffer_new = InvalidBuffer; + } + if (BufferIsValid(vmbuffer)) + { + ReleaseBuffer(vmbuffer); + vmbuffer = InvalidBuffer; + } + #ifdef USE_PGRAC_CLUSTER + /* P0-33: old and new versions share one xact-local TT binding. Publish + * that exact binding once, after both possible ACTIVE stamps are WAL- + * protected and before either heap content lock is released. */ + if (cluster_itl_old_active || cluster_itl_new_active) + cluster_tt_local_record_data_active(xid, cluster_itl_uba); + /* * PGRAC (spec-3.4a D4): register touched ITL handle(s). Outside * CRIT so palloc is safe. Same-page UPDATE registers one handle @@ -5554,10 +5642,6 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, if (newbuf != buffer) ReleaseBuffer(newbuf); ReleaseBuffer(buffer); - if (BufferIsValid(vmbuffer_new)) - ReleaseBuffer(vmbuffer_new); - if (BufferIsValid(vmbuffer)) - ReleaseBuffer(vmbuffer); /* * Release the lmgr tuple lock, if we had it. diff --git a/src/backend/access/heap/heapam_visibility.c b/src/backend/access/heap/heapam_visibility.c index db14f79967..4520189f19 100644 --- a/src/backend/access/heap/heapam_visibility.c +++ b/src/backend/access/heap/heapam_visibility.c @@ -828,37 +828,42 @@ cluster_satisfies_update_fork(HeapTuple htup, Buffer buffer, TM_Result *res) * striping-off/below-floor collision with a remote inserter's xid; the * pre-round-6 early return handed that collision to the PG-native cmin * path -> "attempted to update invisible tuple". Only the no-remote- - * evidence route treats the raw match as our own insert. + * evidence route treats the raw match as our own insert. VACUUM freeze is + * stronger: it proves xmin committed even after that historical ITL slot is + * recycled, so only the exact FROZEN bit pair skips xmin resolution. */ - cluster_visibility_resolve_tuple(buffer, tuple, raw_xmin, CLUSTER_VIS_XMIN, &r); - switch (cluster_vis_evidence_route(r.evidence, TransactionIdIsCurrentTransactionId(raw_xmin))) { - case CLUSTER_VIS_ROUTE_FAILCLOSED_UNKNOWN: - ereport(ERROR, (errcode(ERRCODE_CLUSTER_TT_STATUS_UNKNOWN), - errmsg("cluster TT slot recycled for xmin %u", raw_xmin), - errhint("ITL slot no longer maps to this xid; retry."))); - break; - case CLUSTER_VIS_ROUTE_NATIVE_SELF: - /* Our own insert: PG-native handles cmin / self-modified correctly. */ - return false; - case CLUSTER_VIS_ROUTE_REMOTE_VERDICT: - switch (cluster_vis_update_xmin_verdict(r.status)) { - case CVV_INVISIBLE: - *res = TM_Invisible; - return true; - case CVV_FAILCLOSED_UNKNOWN: + if (cluster_vis_xmin_needs_resolution(tuple->t_infomask)) { + cluster_visibility_resolve_tuple(buffer, tuple, raw_xmin, CLUSTER_VIS_XMIN, &r); + switch ( + cluster_vis_evidence_route(r.evidence, TransactionIdIsCurrentTransactionId(raw_xmin))) { + case CLUSTER_VIS_ROUTE_FAILCLOSED_UNKNOWN: ereport(ERROR, (errcode(ERRCODE_CLUSTER_TT_STATUS_UNKNOWN), - errmsg("cluster TT status unknown for xmin %u", raw_xmin), - errhint("Remote commit_scn not yet propagated; retry or abort."))); + errmsg("cluster TT slot recycled for xmin %u", raw_xmin), + errhint("ITL slot no longer maps to this xid; retry."))); break; - case CVV_VISIBLE: - xmin_remote_visible = true; + case CLUSTER_VIS_ROUTE_NATIVE_SELF: + /* Our own insert: PG-native handles cmin / self-modified correctly. */ + return false; + case CLUSTER_VIS_ROUTE_REMOTE_VERDICT: + switch (cluster_vis_update_xmin_verdict(r.status)) { + case CVV_INVISIBLE: + *res = TM_Invisible; + return true; + case CVV_FAILCLOSED_UNKNOWN: + ereport(ERROR, (errcode(ERRCODE_CLUSTER_TT_STATUS_UNKNOWN), + errmsg("cluster TT status unknown for xmin %u", raw_xmin), + errhint("Remote commit_scn not yet propagated; retry or abort."))); + break; + case CVV_VISIBLE: + xmin_remote_visible = true; + break; + default: + break; + } break; - default: + case CLUSTER_VIS_ROUTE_NATIVE: break; } - break; - case CLUSTER_VIS_ROUTE_NATIVE: - break; } /* xmin NONE/LOCAL: leave xmin to PG unless we must intercept the xmax. */ @@ -2465,17 +2470,15 @@ HeapTupleSatisfiesVacuumHorizon(HeapTuple htup, Buffer buffer, TransactionId *de #ifdef USE_PGRAC_CLUSTER /* - * spec-3.14 D5 (hole #2): a tuple with REMOTE writer evidence must never - * be judged DEAD by this node's local horizon -- overlapping xid spaces - * mean local oldest_xmin can pass a still-live remote xid (false-dead). - * Return HEAPTUPLE_LIVE: the RECENTLY_DEAD path re-tests dead_after - * against the LOCAL oldest_xmin / vistest (heap_prune_satisfies_vacuum) - * and would re-introduce the false-dead, so LIVE (unconditional keep) is - * the only safe verdict. Real reclamation waits for the cross-node - * vacuum-coordination spec (spec-3.14 §10). spec wrote RECENTLY_DEAD; - * LIVE is the implementation-time correction (dead_after re-test hazard). + * P0-28 / spec-3.14 §10: a peer may retain this exact TID across a + * PCM-X wait. ITL evidence is not a durable prune-horizon token: its slot + * may be reused before that peer reacquires the page, after which native + * xmin/CLOG checks can misclassify and recycle the target line pointer. + * Keep every shared-storage tuple until a real cluster horizon exists. */ - if (cluster_storage_mode_enabled() && cluster_tuple_has_remote_evidence(buffer, tuple)) { + if (cluster_vis_prune_must_defer(cluster_storage_mode_enabled() + && BufferIsValid(buffer) && !BufferIsLocal(buffer), + false)) { cluster_vis_bump_prune_remote_keep_count(); return HEAPTUPLE_LIVE; } diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c index bc2bc1741f..f3e4a027cc 100644 --- a/src/backend/access/heap/hio.c +++ b/src/backend/access/heap/hio.c @@ -66,6 +66,30 @@ cluster_hio_lease_target_block(Relation relation) } #endif +/* P0-20: acquire an ordered pair without ever sleeping on the second PCM-X + * conversion while the first content lock is held. BARRIER_CLOSED means the + * first page was frozen before this nested edge became visible. Release our + * first lock, resolve the second conversion alone, then retry in the original + * block order; pins remain owned by the caller throughout. */ +static void +cluster_hio_lock_buffer_pair(Buffer first, Buffer second) +{ + Assert(BufferIsValid(first)); + Assert(BufferIsValid(second)); + Assert(first != second); + + for (;;) + { + LockBuffer(first, BUFFER_LOCK_EXCLUSIVE); + if (ClusterLockBufferExclusiveBarrierAware(second)) + return; + + LockBuffer(first, BUFFER_LOCK_UNLOCK); + LockBuffer(second, BUFFER_LOCK_EXCLUSIVE); + LockBuffer(second, BUFFER_LOCK_UNLOCK); + } +} + /* * RelationPutHeapTuple - place tuple at specified page * @@ -233,10 +257,12 @@ GetVisibilityMapPins(Relation relation, Buffer buffer1, Buffer buffer2, if (need_to_pin_buffer2) visibilitymap_pin(relation, block2, vmbuffer2); - /* Relock buffers. */ - LockBuffer(buffer1, BUFFER_LOCK_EXCLUSIVE); + /* Relock buffers. A cross-page relock keeps the same block order, + * but must unwind a frozen-barrier refusal from the second page. */ if (buffer2 != InvalidBuffer && buffer2 != buffer1) - LockBuffer(buffer2, BUFFER_LOCK_EXCLUSIVE); + cluster_hio_lock_buffer_pair(buffer1, buffer2); + else + LockBuffer(buffer1, BUFFER_LOCK_EXCLUSIVE); /* * If there are two buffers involved and we pinned just one of them, @@ -716,8 +742,7 @@ RelationGetBufferForTuple(Relation relation, Size len, buffer = ReadBuffer(relation, targetBlock); if (PageIsAllVisible(BufferGetPage(buffer))) visibilitymap_pin(relation, targetBlock, vmbuffer); - LockBuffer(otherBuffer, BUFFER_LOCK_EXCLUSIVE); - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + cluster_hio_lock_buffer_pair(otherBuffer, buffer); } else { @@ -725,8 +750,7 @@ RelationGetBufferForTuple(Relation relation, Size len, buffer = ReadBuffer(relation, targetBlock); if (PageIsAllVisible(BufferGetPage(buffer))) visibilitymap_pin(relation, targetBlock, vmbuffer); - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); - LockBuffer(otherBuffer, BUFFER_LOCK_EXCLUSIVE); + cluster_hio_lock_buffer_pair(buffer, otherBuffer); } /* @@ -900,8 +924,9 @@ RelationGetBufferForTuple(Relation relation, Size len, { /* released lock on target buffer above */ if (otherBuffer != InvalidBuffer) - LockBuffer(otherBuffer, BUFFER_LOCK_EXCLUSIVE); - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + cluster_hio_lock_buffer_pair(otherBuffer, buffer); + else + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); recheckVmPins = true; } else if (otherBuffer != InvalidBuffer) @@ -926,8 +951,7 @@ RelationGetBufferForTuple(Relation relation, Size len, { unlockedTargetBuffer = true; LockBuffer(buffer, BUFFER_LOCK_UNLOCK); - LockBuffer(otherBuffer, BUFFER_LOCK_EXCLUSIVE); - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + cluster_hio_lock_buffer_pair(otherBuffer, buffer); } recheckVmPins = true; } diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index f98a30191e..e1a97c0ee6 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -255,6 +255,32 @@ visibilitymap_pin(Relation rel, BlockNumber heapBlk, Buffer *vmbuf) *vmbuf = vm_readbuf(rel, mapBlock, true); } +/* + * Repin the exact visibility-map block in a recently observed descriptor. + * + * Unlike visibilitymap_pin(), this never performs a mapping lookup, storage + * read, or relation extension. A heap mutation can therefore release its VM + * pin before a potentially blocking cluster PCM acquire, take heap content + * authority, and use this helper without introducing I/O below that lock. A + * false result means the descriptor was invalidated or retagged; the caller + * must release heap content authority and retry through visibilitymap_pin(). + */ +bool +visibilitymap_pin_recent(Relation rel, BlockNumber heapBlk, Buffer recent_buffer, + Buffer *vmbuf) +{ + BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk); + + if (rel == NULL || vmbuf == NULL || BufferIsValid(*vmbuf) + || !BufferIsValid(recent_buffer)) + return false; + if (!ReadRecentBuffer(RelationGetSmgr(rel)->smgr_rlocator.locator, + VISIBILITYMAP_FORKNUM, mapBlock, recent_buffer)) + return false; + *vmbuf = recent_buffer; + return true; +} + /* * visibilitymap_pin_ok - do we already have the correct page pinned? * diff --git a/src/backend/cluster/cluster_cr_server.c b/src/backend/cluster/cluster_cr_server.c index 94656cc045..3007bde432 100644 --- a/src/backend/cluster/cluster_cr_server.c +++ b/src/backend/cluster/cluster_cr_server.c @@ -1049,7 +1049,7 @@ cr_serve_slot(ClusterLmsCrSlot *slot) CLUSTER_INJECTION_POINT("cluster-lms-cr-construct"); if (!cluster_injection_should_skip("cluster-lms-cr-construct") && cluster_crossnode_cr_data_plane - && cluster_bufmgr_copy_block_for_gcs(slot->tag, &page_lsn, cur_copy.data)) { + && cluster_bufmgr_copy_block_for_gcs(slot->tag, &page_lsn, cur_copy.data, NULL)) { PG_TRY(); { cluster_cr_construct_page_for_server(cur_copy.data, slot->read_scn, slot->tag, diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 1e86fe5d8d..fc28d67d81 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -1451,6 +1451,8 @@ dump_lms(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_lms_obs_get_outbound_not_admitted(-1))); emit_row(rsinfo, "lms", "lms_outbound_requeue_drop_count", fmt_int64((int64)cluster_lms_obs_get_outbound_requeue_drop(-1))); + emit_row(rsinfo, "lms", "lms_outbound_cap_guard_drop_count", + fmt_int64((int64)cluster_lms_obs_get_outbound_cap_guard_drop(-1))); { int w, hb; @@ -1474,6 +1476,9 @@ dump_lms(ReturnSetInfo *rsinfo) snprintf(wkey, sizeof(wkey), "lms_outbound_requeue_drop_count_w%d", w); emit_row(rsinfo, "lms", wkey, fmt_int64((int64)cluster_lms_obs_get_outbound_requeue_drop(w))); + snprintf(wkey, sizeof(wkey), "lms_outbound_cap_guard_drop_count_w%d", w); + emit_row(rsinfo, "lms", wkey, + fmt_int64((int64)cluster_lms_obs_get_outbound_cap_guard_drop(w))); } /* Inline-serve duration histogram: aggregate 16 rows + per live @@ -2017,7 +2022,7 @@ dump_pcm(ReturnSetInfo *rsinfo) { Size cursor = 0; Size index = 0; - char line[384]; + char line[768]; char key[64]; while (cluster_pcm_x_master_tag_debug_next(&cursor, &index, line, sizeof(line))) { @@ -2401,6 +2406,8 @@ dump_gcs(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_gcs_get_pi_watermark_advance_count())); emit_row(rsinfo, "gcs", "pi_watermark_retire_count", fmt_int64((int64)cluster_gcs_get_pi_watermark_retire_count())); + emit_row(rsinfo, "gcs", "pi_durable_note_apply_count", + fmt_int64((int64)cluster_gcs_get_pi_durable_note_apply_count())); emit_row(rsinfo, "gcs", "lost_write_detected_count", fmt_int64((int64)cluster_gcs_get_lost_write_detected_count())); emit_row(rsinfo, "gcs", "lost_write_avoid_count", diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 63a8f66552..1e83e03684 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -262,6 +262,7 @@ typedef struct ClusterGcsBlockShared { /* PGRAC: spec-2.37 D12 — 4 NEW counters for PI watermark + lost-write detection. */ pg_atomic_uint64 pi_watermark_advance_count; /* X→N/S downgrade caller advance ticks */ pg_atomic_uint64 pi_watermark_retire_count; /* tag lifecycle + durable-confirm retire */ + pg_atomic_uint64 pi_durable_note_apply_count; /* target accepted status-3 DATA/CONTROL note */ pg_atomic_uint64 lost_write_detected_count; /* master direct OR holder forward detect */ pg_atomic_uint64 lost_write_avoid_count; /* durable-confirm retire avoided false-pos */ /* PGRAC: spec-2.41 D7 — SCN lost-write detector + redo-coverage observability. @@ -447,7 +448,8 @@ static bool gcs_block_get_ship_image(BufferTag tag, int32 dest_node, bool allow_ const char **out_block_payload, uint32 *out_block_lkey, ClusterICSgeReleaseCallback *out_release_cb, void **out_release_arg, ClusterSfDepVec *out_sf_dep_vec, - bool *out_sf_dep_valid); + bool *out_sf_dep_valid, + ClusterBufmgrGcsCopyRefusal *out_copy_refusal); static void gcs_block_release_ship_image(ClusterICSgeReleaseCallback release_cb, void *release_arg); static uint32 gcs_block_compute_checksum(const char *block_data); static uint32 gcs_block_compute_invalidate_checksum(const GcsBlockInvalidatePayload *inv); @@ -585,6 +587,7 @@ cluster_gcs_block_shmem_init(void) /* PGRAC: spec-2.37 D12 — 4 NEW counters init. */ pg_atomic_init_u64(&ClusterGcsBlock->pi_watermark_advance_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->pi_watermark_retire_count, 0); + pg_atomic_init_u64(&ClusterGcsBlock->pi_durable_note_apply_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->lost_write_detected_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->lost_write_avoid_count, 0); /* PGRAC: spec-2.41 D7 — 4 NEW SCN detector + redo-coverage counters init. */ @@ -858,8 +861,24 @@ static void gcs_block_release_slot(ClusterGcsBlockOutstandingSlot *slot) { ClusterGcsBlockBackendBlock *blk = gcs_block_my_block(); + BufferTag released_tag; + uint64 released_request_id = 0; + uint64 released_epoch = 0; + int released_slot_index = -1; + int released_direct_state = (int)GCS_BLOCK_DIRECT_UNARMED; + bool released_reply_received = false; + bool released_live_direct = false; LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); + if (slot->in_use && slot->direct_target_prepared) { + released_tag = slot->tag; + released_request_id = slot->request_id; + released_epoch = slot->request_epoch; + released_slot_index = (int)(slot - &blk->slots[0]); + released_direct_state = (int)slot->direct_state; + released_reply_received = slot->reply_received; + released_live_direct = true; + } slot->in_use = false; slot->reply_received = false; slot->request_id = 0; @@ -881,6 +900,15 @@ gcs_block_release_slot(ClusterGcsBlockOutstandingSlot *slot) slot->direct_target_prepared = false; slot->direct_abort_reason = GCS_BLOCK_DIRECT_ABORT_NONE; LWLockRelease(&blk->lock.lock); + if (released_live_direct) + elog(LOG, + "cluster GCS block slot released with live direct target observation: " + "backend=%d slot=%d request_id=%llu epoch=%llu rel=%u fork=%d blk=%u " + "reply_received=%d direct_state=%d direct_prepared=1", + (int)MyBackendId - 1, released_slot_index, + (unsigned long long)released_request_id, (unsigned long long)released_epoch, + released_tag.relNumber, (int)released_tag.forkNum, released_tag.blockNum, + released_reply_received ? 1 : 0, released_direct_state); } static uint32 @@ -949,11 +977,23 @@ gcs_block_direct_prepare_attempt(ClusterGcsBlockOutstandingSlot *slot, BufferDes int backend_idx = MyBackendId - 1; int slot_idx; int32 holder_node; + bool slot_eligible; if (slot == NULL || buf == NULL) return false; if (transition_id != PCM_TRANS_N_TO_S) return false; + /* A current-master INVALIDATE can now deliver a local exact denial before + * this attempt is sent. Close both sides of the target-preparation window: + * do not begin after that denial, and recheck after the bufmgr prepare in + * case it landed while the target was being pinned/prepared. */ + LWLockAcquire(&blk->lock.lock, LW_SHARED); + slot_eligible = slot->in_use && !slot->reply_received + && slot->direct_state == GCS_BLOCK_DIRECT_UNARMED + && !slot->direct_target_prepared; + LWLockRelease(&blk->lock.lock); + if (!slot_eligible) + return false; holder_node = cluster_pcm_master_holder_node_by_tag(tag); if (!GcsBlockDirectCanArmExpectedPeer(holder_node, expected_peer)) return false; @@ -964,6 +1004,13 @@ gcs_block_direct_prepare_attempt(ClusterGcsBlockOutstandingSlot *slot, BufferDes slot_idx = gcs_block_slot_index(blk, slot); LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); + if (!slot->in_use || slot->reply_received + || slot->direct_state != GCS_BLOCK_DIRECT_UNARMED + || slot->direct_target_prepared) { + LWLockRelease(&blk->lock.lock); + gcs_block_direct_finish_target(buf, true, false, InvalidXLogRecPtr); + return false; + } slot->direct_generation = cluster_ic_rdma_direct_land_next_generation(slot->direct_generation); slot->direct_state = GCS_BLOCK_DIRECT_ARMING; slot->direct_expected_peer = expected_peer; @@ -1026,6 +1073,54 @@ cluster_gcs_block_compute_checksum(const char *block_data) return gcs_block_compute_checksum(block_data); } +/* + * A requester-side status=6 only says that some producer refused the image; + * the historical aggregate counter is also bumped by requesters and therefore + * cannot identify the producer branch. Emit one reason-tagged record at the + * producer with the unchanged wire identity and ONE entry-lock-coherent PCM + * authority snapshot. Do not reconstruct authority with separate state / + * holder / bitmap queries here: that would turn diagnostics into another + * observer race under the hot-block handoff this record is meant to explain. + */ +static void +gcs_block_log_master_not_holder_producer(const char *reason, BufferTag tag, uint64 request_id, + uint64 request_epoch, int32 requester_node, + uint8 transition_id) +{ + PcmAuthoritySnapshot authority; + bool authority_found; + + authority_found = cluster_pcm_lock_authority_snapshot(tag, &authority); + ereport( + LOG, + (errmsg_internal("cluster_gcs_block: master-not-holder reply produced"), + errdetail_internal( + "reason=%s producer=%d requester=%d request_id=" UINT64_FORMAT + " request_epoch=" UINT64_FORMAT + " transition=%u tag spc=%u db=%u relNumber=%u fork=%d block=%u " + "authority_found=%d state=%u x_holder=%d s_holders=0x%08x " + "master_holder_node=%u master_holder_procno=%u master_holder_epoch=" UINT64_FORMAT + " master_holder_request_id=" UINT64_FORMAT + " pending_x_requester=%d pending_x_since_lsn=" UINT64_FORMAT + " transition_count=" UINT64_FORMAT, + reason != NULL ? reason : "unknown", cluster_node_id, requester_node, request_id, + request_epoch, (unsigned int)transition_id, tag.spcOid, tag.dbOid, + (unsigned int)BufTagGetRelNumber(&tag), (int)tag.forkNum, + (unsigned int)tag.blockNum, (int)authority_found, (unsigned int)authority.state, + authority.x_holder_node, (unsigned int)authority.s_holders_bitmap, + authority.master_holder.node_id, authority.master_holder.procno, + authority.master_holder.cluster_epoch, authority.master_holder.request_id, + authority.pending_x_requester_node, authority.pending_x_since_lsn, + authority.transition_count))); +} + +#define GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, reason) \ + gcs_block_log_master_not_holder_producer((reason), (req)->tag, (req)->request_id, (req)->epoch, \ + (req)->sender_node, (req)->transition_id) +#define GCS_BLOCK_LOG_MASTER_NOT_HOLDER_FORWARD(fwd, reason) \ + gcs_block_log_master_not_holder_producer((reason), (fwd)->tag, (fwd)->request_id, (fwd)->epoch, \ + (fwd)->original_requester_node, (fwd)->transition_id) + static void gcs_block_release_ship_image(ClusterICSgeReleaseCallback release_cb, void *release_arg) { @@ -1169,7 +1264,8 @@ gcs_block_get_ship_image(BufferTag tag, int32 dest_node, bool allow_live_sge, XLogRecPtr *out_page_lsn, char *copy_buf, const char **out_block_payload, uint32 *out_block_lkey, ClusterICSgeReleaseCallback *out_release_cb, void **out_release_arg, ClusterSfDepVec *out_sf_dep_vec, - bool *out_sf_dep_valid) + bool *out_sf_dep_valid, + ClusterBufmgrGcsCopyRefusal *out_copy_refusal) { void *scratch = NULL; uint32 scratch_lkey = 0; @@ -1186,6 +1282,8 @@ gcs_block_get_ship_image(BufferTag tag, int32 dest_node, bool allow_live_sge, *out_release_arg = NULL; if (out_sf_dep_valid != NULL) *out_sf_dep_valid = false; + if (out_copy_refusal != NULL) + *out_copy_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE; if (out_sf_dep_vec != NULL) cluster_sf_dep_vec_reset(out_sf_dep_vec); @@ -1228,8 +1326,12 @@ gcs_block_get_ship_image(BufferTag tag, int32 dest_node, bool allow_live_sge, copied = cluster_bufmgr_copy_block_for_gcs_smart_fusion( tag, out_page_lsn, (char *)scratch, out_sf_dep_vec); else - copied = cluster_bufmgr_copy_block_for_gcs(tag, out_page_lsn, (char *)scratch); + copied = cluster_bufmgr_copy_block_for_gcs(tag, out_page_lsn, (char *)scratch, + out_copy_refusal); if (!copied) { + if (smart_fusion_reply && out_copy_refusal != NULL) + *out_copy_refusal + = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_SMART_FUSION_UNCLASSIFIED; if (release_cb != NULL) release_cb(release_arg); return false; @@ -1252,12 +1354,17 @@ gcs_block_get_ship_image(BufferTag tag, int32 dest_node, bool allow_live_sge, if (smart_fusion_reply) { if (!cluster_bufmgr_copy_block_for_gcs_smart_fusion(tag, out_page_lsn, copy_buf, - out_sf_dep_vec)) + out_sf_dep_vec)) { + if (out_copy_refusal != NULL) + *out_copy_refusal + = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_SMART_FUSION_UNCLASSIFIED; return false; + } if (out_sf_dep_valid != NULL) *out_sf_dep_valid = true; gcs_block_note_scratch_copy(); - } else if (!cluster_bufmgr_copy_block_for_gcs(tag, out_page_lsn, copy_buf)) + } else if (!cluster_bufmgr_copy_block_for_gcs(tag, out_page_lsn, copy_buf, + out_copy_refusal)) return false; else gcs_block_note_scratch_copy(); @@ -3212,21 +3319,61 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle volatile PcmXQueueResult result = PCM_X_QUEUE_NOT_READY; int max_retries; int retry_attempt; + bool diagnostic; if (buf == NULL || leader == NULL || reservation_base == NULL || request_runtime == NULL || MyBackendId <= 0) return PCM_X_QUEUE_INVALID; + diagnostic = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); + memset(&progress_before, 0, sizeof(progress_before)); queue_result = cluster_pcm_x_local_progress_exact(leader, &progress_before); cluster_pcm_x_stats_note_queue_result(queue_result); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch requester boundary: entry"), + errdetail("progress_result=%d local=%d backend=%d source=%u master=%d " + "member_state=%u pending_opcode=%u last_response=%u request_id=%llu " + "ticket=%llu grant_generation=%llu image_id=%llu", + (int)queue_result, cluster_node_id, (int)MyBackendId, + progress_before.image.source_node, progress_before.master_node, + (unsigned)progress_before.member_state, + (unsigned)progress_before.pending_opcode, + (unsigned)progress_before.last_response_opcode, + (unsigned long long)progress_before.identity.request_id, + (unsigned long long)progress_before.ref.handle.ticket_id, + (unsigned long long)progress_before.ref.grant_generation, + (unsigned long long)progress_before.image.image_id))); if (queue_result != PCM_X_QUEUE_OK) return queue_result; if (!BufferTagsEqual(&buf->tag, &reservation_base->tag) || !BufferTagsEqual(&leader->identity.tag, &reservation_base->tag) || reservation_base->generation != (progress_before.grant_base_own_generation != 0 - ? progress_before.grant_base_own_generation - : leader->identity.base_own_generation)) + ? progress_before.grant_base_own_generation + : leader->identity.base_own_generation)) { + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch requester boundary: pre-slot reject"), + errdetail("branch=identity-base buf_tag_exact=%s leader_tag_exact=%s " + "base_generation=%llu progress_identity_base=%llu " + "progress_grant_base=%llu effective_grant_base=%llu " + "base_reservation_token=%llu base_writer_activation_token=%llu " + "base_flags=0x%x base_state=%u", + BufferTagsEqual(&buf->tag, &reservation_base->tag) ? "true" : "false", + BufferTagsEqual(&leader->identity.tag, &reservation_base->tag) + ? "true" + : "false", + (unsigned long long)reservation_base->generation, + (unsigned long long)progress_before.identity.base_own_generation, + (unsigned long long)progress_before.grant_base_own_generation, + (unsigned long long)(progress_before.grant_base_own_generation != 0 + ? progress_before.grant_base_own_generation + : leader->identity.base_own_generation), + (unsigned long long)reservation_base->reservation_token, + (unsigned long long)reservation_base->writer_activation_token, + reservation_base->flags, (unsigned)reservation_base->pcm_state))); return PCM_X_QUEUE_STALE; + } if (!cluster_pcm_x_image_fetch_build_request(&progress_before, cluster_node_id, (int32)MyBackendId, &request)) return PCM_X_QUEUE_NOT_READY; @@ -3237,8 +3384,31 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle if (queue_result != PCM_X_QUEUE_OK) return queue_result; if (!cluster_pcm_x_image_fetch_reservation_exact(&live_own, reservation_base, - reservation_token)) - return gcs_block_pcm_x_fetch_reservation_mismatch(&live_own); + reservation_token)) { + queue_result = gcs_block_pcm_x_fetch_reservation_mismatch(&live_own); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch requester boundary: pre-slot reject"), + errdetail("branch=reservation-live result=%d live_tag_exact=%s " + "base_generation=%llu live_generation=%llu " + "base_reservation_token=%llu reservation_token=%llu " + "live_reservation_token=%llu base_writer_activation_token=%llu " + "live_writer_activation_token=%llu base_flags=0x%x live_flags=0x%x " + "base_state=%u live_state=%u", + (int)queue_result, + BufferTagsEqual(&live_own.tag, &reservation_base->tag) ? "true" + : "false", + (unsigned long long)reservation_base->generation, + (unsigned long long)live_own.generation, + (unsigned long long)reservation_base->reservation_token, + (unsigned long long)reservation_token, + (unsigned long long)live_own.reservation_token, + (unsigned long long)reservation_base->writer_activation_token, + (unsigned long long)live_own.writer_activation_token, + reservation_base->flags, live_own.flags, + (unsigned)reservation_base->pcm_state, (unsigned)live_own.pcm_state))); + return queue_result; + } cluster_gcs_block_dedup_register_backend_exit_hook(); slot = gcs_block_try_reserve_exact_slot(request.tag, request.transition_id, @@ -3260,6 +3430,7 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle TimestampTz deadline; bool fence_lost = false; bool got_reply = false; + bool enqueue_result; bool slot_stale = false; if (retry_attempt > 0) { @@ -3334,9 +3505,20 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_request_count, 1); else pg_atomic_fetch_add_u64(&ClusterGcsBlock->retransmit_send_count, 1); - if (!cluster_grd_outbound_enqueue_backend_msg(PGRAC_IC_MSG_GCS_BLOCK_REQUEST, - progress_now.image.source_node, &request, - sizeof(request))) { + enqueue_result = cluster_grd_outbound_enqueue_backend_msg( + PGRAC_IC_MSG_GCS_BLOCK_REQUEST, progress_now.image.source_node, &request, + sizeof(request)); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch requester boundary: send"), + errdetail("attempt=%d enqueue_result=%s source=%u request_id=%llu " + "backend=%d transition=%u", + retry_attempt, enqueue_result ? "true" : "false", + progress_now.image.source_node, + (unsigned long long)request.request_id, + request.requester_backend_id, + (unsigned)request.transition_id))); + if (!enqueue_result) { ConditionVariableCancelSleep(); if (retry_attempt < max_retries) continue; @@ -3377,6 +3559,16 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle } } ConditionVariableCancelSleep(); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch requester boundary: receive"), + errdetail("attempt=%d got_reply=%s slot_stale=%s fence_lost=%s " + "status=%d request_id=%llu", + retry_attempt, got_reply ? "true" : "false", + slot_stale ? "true" : "false", + fence_lost ? "true" : "false", + got_reply ? (int)slot->reply_header.status : -1, + (unsigned long long)request.request_id))); if (fence_lost) { result = PCM_X_QUEUE_NOT_READY; break; @@ -3477,6 +3669,13 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle PG_END_TRY(); gcs_block_release_slot(slot); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch requester boundary: complete"), + errdetail("result=%d requester=%d backend=%d source=%u request_id=%llu", + (int)result, cluster_node_id, (int)MyBackendId, + progress_before.image.source_node, + (unsigned long long)request.request_id))); return (PcmXQueueResult)result; } @@ -3505,111 +3704,166 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle * be obtained — never a silent stale read (Rule 8.A). */ bool -cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, int32 holder_node) +cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, + const PcmAuthoritySnapshot *expected, + bool *out_retry_denied) { ClusterGcsBlockOutstandingSlot *slot; uint64 request_id = 0; BufferTag tag; + int max_retries; + int retry_attempt; + int attempts = 0; + int last_status = -1; GcsBlockForwardPayload fwd; bool got_reply = false; bool installed = false; bool durable_s = false; /* spec-6.12a ㉕ — holder downgraded, we registered */ + int32 holder_node; - if (buf == NULL) - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("cluster_gcs_local_master_read_image_and_wait: NULL BufferDesc"))); + Assert(out_retry_denied != NULL); + if (buf == NULL || expected == NULL || out_retry_denied == NULL) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cluster_gcs_local_master_read_image_and_wait: invalid exact input"))); + *out_retry_denied = false; + holder_node = expected->x_holder_node; + if (expected->state != PCM_STATE_X || holder_node < 0 || holder_node >= 32 + || holder_node == cluster_node_id || expected->s_holders_bitmap != 0 + || expected->master_holder.node_id != (uint32)holder_node) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cluster_gcs_block: invalid remote-X authority for exact read image"))); tag = buf->tag; + /* P0-21 residual: the holder is an exact authority identity, not a route + * hint. A queue winner may have displaced a caller's optimistic sample + * before this helper starts; let bufmgr abort/rearm GRANT_PENDING and + * drive the new authority rather than sending to the stale node. */ + if (!cluster_pcm_lock_authority_matches(tag, expected)) { + *out_retry_denied = true; + return false; + } cluster_gcs_block_dedup_register_backend_exit_hook(); /* expected_master == self: the holder's reply carries forwarding_master = * self, which the HC108 authorized chain validates against this slot. */ slot = gcs_block_reserve_slot(tag, (uint8)PCM_TRANS_N_TO_S, cluster_node_id, &request_id); + max_retries = cluster_gcs_block_retransmit_max_retries >= 0 + ? cluster_gcs_block_retransmit_max_retries + : 4; PG_TRY(); { ClusterGcsBlockBackendBlock *blk = gcs_block_my_block(); TimestampTz deadline; - LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); - slot->reply_received = false; - memset(&slot->reply_header, 0, sizeof(slot->reply_header)); - memset(slot->reply_block_data, 0, sizeof(slot->reply_block_data)); - slot->reply_sf_dep_valid = false; - slot->reply_sf_flags = 0; - cluster_sf_dep_vec_reset(&slot->reply_sf_dep_vec); - slot->request_epoch = cluster_epoch_get_current(); - slot->expected_master_node = cluster_node_id; - slot->stale = false; - LWLockRelease(&blk->lock.lock); + for (retry_attempt = 0; retry_attempt <= max_retries; retry_attempt++) { + attempts = retry_attempt + 1; + got_reply = false; + if (retry_attempt > 0) { + long backoff_ms = gcs_block_backoff_ms_for_retry(retry_attempt); - memset(&fwd, 0, sizeof(fwd)); - fwd.request_id = request_id; - fwd.epoch = cluster_epoch_get_current(); - fwd.tag = tag; - fwd.original_requester_node = cluster_node_id; /* reply returns to us */ - fwd.requester_backend_id = (int32)MyBackendId; - fwd.master_node = cluster_node_id; - fwd.transition_id = (uint8)PCM_TRANS_N_TO_S; - GcsBlockForwardPayloadSetExpectedPiWatermarkScn( - &fwd, cluster_pcm_lock_pi_watermark_scn_query(tag)); - GcsBlockForwardPayloadSetReadImage(&fwd, true); - /* PGRAC: spec-6.12a ㉕ — ask the remote X holder to TRY the quiescent + pg_atomic_fetch_add_u64(&ClusterGcsBlock->retransmit_attempt_count, 1); + (void)WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, backoff_ms, + WAIT_EVENT_GCS_BLOCK_RETRANSMIT_WAIT); + ResetLatch(MyLatch); + /* The bounded retry belongs only to the exact remote-X authority + * sampled by the caller. Do not spend another request identity on + * a holder displaced while we backed off. */ + if (!cluster_pcm_lock_authority_matches(tag, expected)) { + *out_retry_denied = true; + break; + } + if (!gcs_block_pcm_x_next_request_id(&request_id)) + ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("cluster_gcs_block: read-image request id exhausted"))); + } + + LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); + slot->request_id = request_id; + slot->reply_received = false; + memset(&slot->reply_header, 0, sizeof(slot->reply_header)); + memset(slot->reply_block_data, 0, sizeof(slot->reply_block_data)); + slot->reply_sf_dep_valid = false; + slot->reply_sf_flags = 0; + cluster_sf_dep_vec_reset(&slot->reply_sf_dep_vec); + slot->request_epoch = cluster_epoch_get_current(); + slot->expected_master_node = cluster_node_id; + slot->stale = false; + LWLockRelease(&blk->lock.lock); + + memset(&fwd, 0, sizeof(fwd)); + fwd.request_id = request_id; + fwd.epoch = cluster_epoch_get_current(); + fwd.tag = tag; + fwd.original_requester_node = cluster_node_id; /* reply returns to us */ + fwd.requester_backend_id = (int32)MyBackendId; + fwd.master_node = cluster_node_id; + fwd.transition_id = (uint8)PCM_TRANS_N_TO_S; + GcsBlockForwardPayloadSetExpectedPiWatermarkScn( + &fwd, cluster_pcm_lock_pi_watermark_scn_query(tag)); + GcsBlockForwardPayloadSetReadImage(&fwd, true); + /* PGRAC: spec-6.12a ㉕ — ask the remote X holder to TRY the quiescent * X->S downgrade so this read (and every later one) becomes a durable * cached S. We ARE the master here, so on the holder's durable reply * the registration is a local transition apply — no ACK wire. */ - if (cluster_read_scache) - GcsBlockForwardPayloadSetDowngradeRequest(&fwd, true); - - pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_forward_sent_count, 1); - if (!cluster_grd_outbound_enqueue_backend_msg(PGRAC_IC_MSG_GCS_BLOCK_FORWARD, - (uint32)holder_node, &fwd, sizeof(fwd))) - ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), - errmsg("cluster_gcs_block: failed to enqueue read-image FORWARD " - "to X holder %d", - holder_node))); + if (cluster_read_scache) + GcsBlockForwardPayloadSetDowngradeRequest(&fwd, true); - deadline = GetCurrentTimestamp() - + ((TimestampTz)cluster_gcs_reply_timeout_ms) * (TimestampTz)1000; + if (retry_attempt > 0) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->retransmit_send_count, 1); - ConditionVariablePrepareToSleep(&slot->reply_cv); - for (;;) { - TimestampTz now; - long timeout_ms; - bool have_reply; + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_forward_sent_count, 1); + if (!cluster_grd_outbound_enqueue_backend_msg(PGRAC_IC_MSG_GCS_BLOCK_FORWARD, + (uint32)holder_node, &fwd, sizeof(fwd))) + ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("cluster_gcs_block: failed to enqueue read-image FORWARD " + "to X holder %d", + holder_node))); - LWLockAcquire(&blk->lock.lock, LW_SHARED); - have_reply = slot->in_use && slot->reply_received; - LWLockRelease(&blk->lock.lock); - if (have_reply) { - got_reply = true; - break; - } - now = GetCurrentTimestamp(); - if (now >= deadline) - break; - timeout_ms = (long)((deadline - now) / 1000); - if (timeout_ms <= 0) - timeout_ms = 1; - (void)ConditionVariableTimedSleep(&slot->reply_cv, timeout_ms, - WAIT_EVENT_GCS_BLOCK_SHIP_WAIT); - } - ConditionVariableCancelSleep(); + deadline = GetCurrentTimestamp() + + ((TimestampTz)cluster_gcs_reply_timeout_ms) * (TimestampTz)1000; - if (got_reply - && (slot->reply_header.status == (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER - || slot->reply_header.status - == (uint8)GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE)) { - uint32 expected = slot->reply_header.checksum; - uint32 got = gcs_block_compute_checksum(slot->reply_block_data); + ConditionVariablePrepareToSleep(&slot->reply_cv); + for (;;) { + TimestampTz now; + long timeout_ms; + bool have_reply; - if (expected == got) { - gcs_block_install_reply_block(buf, slot->reply_block_data, - (XLogRecPtr)slot->reply_header.page_lsn, slot); - /* spec-5.14 D2 class 2: this node (local master) consumed the + LWLockAcquire(&blk->lock.lock, LW_SHARED); + have_reply = slot->in_use && slot->reply_received; + LWLockRelease(&blk->lock.lock); + if (have_reply) { + got_reply = true; + break; + } + now = GetCurrentTimestamp(); + if (now >= deadline) + break; + timeout_ms = (long)((deadline - now) / 1000); + if (timeout_ms <= 0) + timeout_ms = 1; + (void)ConditionVariableTimedSleep(&slot->reply_cv, timeout_ms, + WAIT_EVENT_GCS_BLOCK_SHIP_WAIT); + } + ConditionVariableCancelSleep(); + last_status = got_reply ? (int)slot->reply_header.status : -1; + + if (got_reply + && (slot->reply_header.status == (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER + || slot->reply_header.status + == (uint8)GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE)) { + uint32 expected = slot->reply_header.checksum; + uint32 got = gcs_block_compute_checksum(slot->reply_block_data); + + if (expected == got) { + gcs_block_install_reply_block(buf, slot->reply_block_data, + (XLogRecPtr)slot->reply_header.page_lsn, slot); + /* spec-5.14 D2 class 2: this node (local master) consumed the * remote holder's volatile image. */ - gcs_block_stamp_touched(holder_node, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); - installed = true; - /* + gcs_block_stamp_touched(holder_node, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + installed = true; + /* * PGRAC: spec-6.12a ㉕ — the holder downgraded X->S and shipped * a DURABLE S grant. We are the master: register ourselves as * an S holder with a LOCAL transition apply (no ACK wire). The @@ -3620,17 +3874,45 @@ cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, int32 holder_node) * pcm_state stays N (Rule 8.A: never a durable copy the * master entry does not track). */ - if (slot->reply_header.status - == (uint8)GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE) { - if (cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_S, - cluster_node_id)) - durable_s = true; - else - cluster_lever_a_note_remote_ack_degraded(); + if (slot->reply_header.status + == (uint8)GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE) { + if (cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_S, + cluster_node_id)) + durable_s = true; + else + cluster_lever_a_note_remote_ack_degraded(); + } + } else { + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_checksum_fail_count, 1); } - } else { - pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_checksum_fail_count, 1); } + + if (installed || durable_s) + break; + /* P0-21 completion: holder-side conditional BufferContent refusal is + * HC105 transient. Retry without ever blocking the DATA worker. A new + * request id on the next iteration prevents a delayed old denial from + * winning the re-armed slot. */ + if (got_reply + && slot->reply_header.status == (uint8)GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER + && retry_attempt < max_retries) { + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_master_not_holder_count, 1); + /* A refusal is transient only while the complete holder identity + * remains authoritative. Drift exits through the outer + * GRANT_PENDING abort/rearm path; this helper never guesses the + * successor holder. */ + if (!cluster_pcm_lock_authority_matches(tag, expected)) { + *out_retry_denied = true; + break; + } + continue; + } + /* Preserve the stable-authority terminal fail-close below, but do + * not surface it for an old identity displaced during a timeout, + * checksum failure, or other non-success terminal. */ + if (!cluster_pcm_lock_authority_matches(tag, expected)) + *out_retry_denied = true; + break; } } PG_CATCH(); @@ -3642,6 +3924,8 @@ cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, int32 holder_node) gcs_block_release_slot(slot); + if (*out_retry_denied) + return false; if (durable_s) return true; /* spec-6.12a ㉕ — durable S; caller mirrors pcm_state = S */ if (installed) @@ -3654,6 +3938,7 @@ cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, int32 holder_node) "for tag spc=%u db=%u relNumber=%u block=%u", holder_node, tag.spcOid, tag.dbOid, (unsigned int)BufTagGetRelNumber(&tag), (unsigned int)tag.blockNum), + errdetail("attempts=%d last_status=%d", attempts, last_status), errhint("The X holder did not ship a current image in time; retry, or " "inspect dump_gcs.cf_xheld_read_ship_count."))); return true; /* unreachable */ @@ -4395,8 +4680,8 @@ cluster_gcs_block_undo_multi_verdict_fetch_and_wait(int32 origin_node, MultiXact * of cluster_gcs_local_master_read_image_and_wait. */ bool -cluster_gcs_local_master_x_transfer_and_wait(BufferDesc *buf, int32 holder_node, - bool clean_eligible) +cluster_gcs_local_master_x_transfer_and_wait(BufferDesc *buf, const PcmAuthoritySnapshot *expected, + bool clean_eligible, bool *out_retry_denied) { ClusterGcsBlockOutstandingSlot *slot; uint64 request_id = 0; @@ -4408,15 +4693,32 @@ cluster_gcs_local_master_x_transfer_and_wait(BufferDesc *buf, int32 holder_node, uint8 reply_status = (uint8)GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE; /* spec-5.2a D3 */ XLogRecPtr installed_page_lsn = InvalidXLogRecPtr; SCN installed_page_scn = InvalidScn; + int32 holder_node; + PcmXTransferCommitResult commit_result; - if (buf == NULL) - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("cluster_gcs_local_master_x_transfer_and_wait: NULL BufferDesc"))); + Assert(out_retry_denied != NULL); + if (buf == NULL || expected == NULL || out_retry_denied == NULL) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cluster_gcs_local_master_x_transfer_and_wait: invalid exact input"))); + *out_retry_denied = false; + holder_node = expected->x_holder_node; + if (expected->state != PCM_STATE_X || holder_node < 0 || holder_node >= 32 + || holder_node == cluster_node_id || expected->s_holders_bitmap != 0 + || expected->master_holder.node_id != (uint32)holder_node) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cluster_gcs_block: invalid remote-X authority for exact transfer"))); tag = buf->tag; + /* P0-26 first barrier: do not emit a request for an authority token + * already displaced by another queue winner/session. */ + if (!cluster_pcm_lock_authority_matches(tag, expected)) { + *out_retry_denied = true; + return false; + } cluster_gcs_block_dedup_register_backend_exit_hook(); slot = gcs_block_reserve_slot(tag, (uint8)PCM_TRANS_N_TO_X, cluster_node_id, &request_id); - PG_TRY(); { ClusterGcsBlockBackendBlock *blk = gcs_block_my_block(); @@ -4583,17 +4885,31 @@ cluster_gcs_local_master_x_transfer_and_wait(BufferDesc *buf, int32 holder_node, gcs_block_release_slot(slot); if (installed) { - /* The holder shipped its current image and released its X; record self - * as the new authoritative X holder (durable). No master round-trip: - * THIS node is the master. spec-2.41 D2 — also advance the detector's - * SCN watermark from the installed image's pd_block_scn (local-page - * source; bytes are slot->reply_block_data). */ - cluster_pcm_lock_master_take_x_after_transfer(tag, installed_page_lsn, installed_page_scn, - holder_node, request_id, fwd.epoch); + /* P0-26 final barrier: the slot/request id isolates late wire + * replies; this exact authority compare isolates a valid old reply + * from a newer queue handoff, holder generation, or session. */ + commit_result = cluster_pcm_lock_master_take_x_after_transfer( + tag, expected, installed_page_lsn, installed_page_scn, holder_node, + (uint32)MyProc->pgprocno, request_id, fwd.epoch); + if (commit_result == PCM_X_TRANSFER_COMMIT_STALE + || commit_result == PCM_X_TRANSFER_COMMIT_NOT_FOUND) { + *out_retry_denied = true; + return false; + } + if (commit_result != PCM_X_TRANSFER_COMMIT_OK) + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cluster_gcs_block: invalid exact X-transfer commit state"))); pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_x_granted_from_holder_count, 1); if (clean_eligible) pg_atomic_fetch_add_u64(&ClusterGcsBlock->clean_page_xfer_count, 1); - return true; /* durable X grant — bufmgr mirrors buf->pcm_state = X */ + return true; + } + /* A denial/timeout from a holder whose authority was concurrently + * replaced is not a client terminal. Return to bufmgr so the exact + * GRANT_PENDING token and request id are both replaced. */ + if (!cluster_pcm_lock_authority_matches(tag, expected)) { + *out_retry_denied = true; + return false; } if (read_image) @@ -4922,6 +5238,8 @@ gcs_block_send_reply(int32 dest_node, const GcsBlockRequestPayload *req, GcsBloc hdr.checksum = gcs_block_compute_checksum(block_data); if (GcsBlockRequestPayloadIsDirectLandArmed(req) && dest_node != cluster_node_id) { + if (!GcsBlockReplyStatusIsDirectLandSendable(status)) + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "direct-land-nonsendable"); if (status != GCS_BLOCK_REPLY_GRANTED || block_data == NULL) { char zero_page[GCS_BLOCK_DATA_SIZE]; @@ -4981,6 +5299,7 @@ gcs_block_deny_direct_armed_forward_request(const GcsBlockRequestPayload *req) * consume/abort the direct receive first and let the requester retry * without direct-land. */ + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "direct-land-forward-rearm"); gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER, InvalidXLogRecPtr, NULL); return true; @@ -5060,6 +5379,10 @@ gcs_block_resend_cached_reply(int32 dest_node, const GcsBlockDedupEntry *entry) if ((entry->request_flags & GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND) != 0 && dest_node != cluster_node_id) { + if (!GcsBlockReplyStatusIsDirectLandSendable((GcsBlockReplyStatus)hdr->status)) + gcs_block_log_master_not_holder_producer( + "cached-direct-land-nonsendable", entry->tag, entry->key.request_id, + entry->key.cluster_epoch, (int32)entry->key.origin_node_id, entry->transition_id); (void)gcs_block_try_send_direct_reply(dest_node, true, hdr, has_block_payload ? block_data : NULL, 0, NULL, NULL); pfree(buf); @@ -5095,20 +5418,37 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe GcsBlockPcmXImageResult image_result; PcmXLocalHolderProgress holder_after; PcmXLocalHolderProgress holder_before; + PcmXImageFetchRequestRefusal request_refusal; PcmXQueueResult progress_result; uint64 master_session; uint64 requester_session; uint64 current_epoch; + int32 decoded_backend = -1; int32 encoded_master; + int32 decoded_requester = -1; int32 tag_master; + bool diagnostic; + bool master_authenticated = false; + bool master_binding_valid = false; + bool request_exact = false; + bool reply_result; if (req == NULL || !cluster_pcm_x_image_id_decode(req->request_id, &encoded_master, NULL)) return false; /* From here onward the reserved namespace is always consumed here. */ if (env == NULL || worker_id < 0 || worker_id >= cluster_lms_workers) return true; + diagnostic = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); current_epoch = cluster_epoch_get_current(); tag_master = cluster_gcs_lookup_master(req->tag); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch source boundary: ingress"), + errdetail("worker=%d source=%u dest=%u requester=%d backend=%d " + "request_id=%llu epoch=%llu encoded_master=%d tag_master=%d", + worker_id, env->source_node_id, env->dest_node_id, req->sender_node, + req->requester_backend_id, (unsigned long long)req->request_id, + (unsigned long long)req->epoch, encoded_master, tag_master))); if (encoded_master != tag_master || req->epoch != current_epoch || !gcs_block_pcm_x_source_capable(req->sender_node) || !cluster_qvotec_in_quorum() || !cluster_membership_is_member(cluster_node_id) @@ -5117,16 +5457,62 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe || !gcs_block_pcm_x_authenticated_session(req->sender_node, current_epoch, &requester_session) || !gcs_block_pcm_x_revalidate_peer_binding(req->sender_node, current_epoch, - requester_session)) + requester_session)) return true; + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch source boundary: validate"), + errdetail("stage=auth requester_session=%llu current_epoch=%llu tag_master=%d", + (unsigned long long)requester_session, + (unsigned long long)current_epoch, tag_master))); progress_result = cluster_pcm_x_local_holder_progress_exact(&req->tag, &holder_before); - if (progress_result != PCM_X_QUEUE_OK - || !cluster_pcm_x_image_fetch_request_exact(env, req, &holder_before, cluster_node_id, - tag_master, current_epoch) - || !gcs_block_pcm_x_authenticated_session(tag_master, current_epoch, &master_session) - || master_session != holder_before.master_session_incarnation - || !gcs_block_pcm_x_revalidate_peer_binding(tag_master, current_epoch, master_session)) + if (progress_result == PCM_X_QUEUE_OK) + request_exact = cluster_pcm_x_image_fetch_request_exact_diagnosed( + env, req, &holder_before, cluster_node_id, tag_master, current_epoch, &request_refusal); + else + request_refusal = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_ARGUMENT; + (void)cluster_gcs_requester_id_decode(holder_before.ref.identity.request_id, + &decoded_requester, &decoded_backend, NULL); + if (request_exact) + master_authenticated + = gcs_block_pcm_x_authenticated_session(tag_master, current_epoch, &master_session); + if (master_authenticated) + master_binding_valid + = master_session == holder_before.master_session_incarnation + && gcs_block_pcm_x_revalidate_peer_binding(tag_master, current_epoch, master_session); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch source boundary: validate"), + errdetail("stage=binding progress_result=%d request_exact=%s refusal=%d " + "master_authenticated=%s master_binding_valid=%s master_session=%llu " + "holder_master_session=%llu holder_master=%d holder_requester=%d " + "holder_pending=%u holder_phase=%u holder_flags=%u " + "payload_len=%u expected_len=%zu request_backend=%d decoded_backend=%d " + "decoded_requester=%d transition=%u holder_epoch=%llu wait_seq=%llu " + "ticket=%llu queue_generation=%llu grant_generation=%llu " + "image_source=%u holder_request_id=%llu holder_image_id=%llu", + (int)progress_result, request_exact ? "true" : "false", + (int)request_refusal, + master_authenticated ? "true" : "false", + master_binding_valid ? "true" : "false", + (unsigned long long)(master_authenticated ? master_session : 0), + (unsigned long long)holder_before.master_session_incarnation, + holder_before.master_node, holder_before.ref.identity.node_id, + (unsigned)holder_before.pending_opcode, + (unsigned)holder_before.phase, (unsigned)holder_before.flags, + (unsigned)env->payload_length, sizeof(*req), req->requester_backend_id, + decoded_backend, decoded_requester, (unsigned)req->transition_id, + (unsigned long long)holder_before.ref.identity.cluster_epoch, + (unsigned long long)holder_before.ref.identity.wait_seq, + (unsigned long long)holder_before.ref.handle.ticket_id, + (unsigned long long)holder_before.ref.handle.queue_generation, + (unsigned long long)holder_before.ref.grant_generation, + holder_before.image.source_node, + (unsigned long long)holder_before.ref.identity.request_id, + (unsigned long long)holder_before.image.image_id))); + if (progress_result != PCM_X_QUEUE_OK || !request_exact || !master_authenticated + || !master_binding_valid) return true; memset(&key, 0, sizeof(key)); @@ -5138,11 +5524,23 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe binding.identity.ref = holder_before.ref; binding.identity.image = holder_before.image; binding.master_session = holder_before.master_session_incarnation; + binding.required_page_scn = holder_before.required_page_scn; memset(&cached, 0, sizeof(cached)); image_result = cluster_gcs_block_dedup_pcm_x_lookup(worker_id, &key, &req->tag, &binding, &cached); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch source boundary: lookup"), + errdetail("result=%d worker=%d requester=%u backend=%d request_id=%llu " + "ticket=%llu grant_generation=%llu image_id=%llu", + (int)image_result, worker_id, key.origin_node_id, + key.requester_backend_id, (unsigned long long)key.request_id, + (unsigned long long)binding.identity.ref.handle.ticket_id, + (unsigned long long)binding.identity.ref.grant_generation, + (unsigned long long)binding.identity.image.image_id))); if (image_result == GCS_BLOCK_PCM_X_IMAGE_NOT_READY) { /* Materialization/publication is still progressing on this same shard. */ + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "pcm-x-image-not-ready"); gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER, InvalidXLogRecPtr, NULL); return true; @@ -5163,7 +5561,14 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe tag_master, current_epoch)) return true; - gcs_block_resend_cached_reply(req->sender_node, &cached); + reply_result = gcs_block_resend_cached_reply(req->sender_node, &cached); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch source boundary: reply"), + errdetail("stage_result=%s requester=%d backend=%d request_id=%llu status=%u", + reply_result ? "true" : "false", req->sender_node, + req->requester_backend_id, (unsigned long long)req->request_id, + (unsigned)cached.status))); if (ClusterGcsBlock != NULL) { pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_reply_count, 1); pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_ship_bytes_total, GCS_BLOCK_DATA_SIZE); @@ -5195,6 +5600,7 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe */ static bool gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, + bool preprepared_image, GcsBlockReplyStatus *out_status, XLogRecPtr *out_page_lsn, const char **out_block_payload, uint32 *out_block_lkey, ClusterICSgeReleaseCallback *out_release_cb, void **out_release_arg, @@ -5202,11 +5608,18 @@ gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, { uint64 current_epoch; PcmLockMode state; + PcmGcsTransitionApplyResult apply_result; + ClusterBufmgrGcsCopyRefusal copy_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE; bool found; *out_status = GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE; - *out_page_lsn = InvalidXLogRecPtr; - *out_block_payload = NULL; + if (!preprepared_image) { + *out_page_lsn = InvalidXLogRecPtr; + *out_block_payload = NULL; + } else { + Assert(*out_page_lsn != InvalidXLogRecPtr); + Assert(*out_block_payload == block_buf); + } if (out_block_lkey != NULL) *out_block_lkey = 0; if (out_release_cb != NULL) @@ -5246,7 +5659,7 @@ gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, * - buffer present: D4 helper handles HC82 + HC89 then reply GRANTED */ state = cluster_pcm_lock_query(req->tag); - found = cluster_bufmgr_probe_block_for_gcs(req->tag); + found = preprepared_image || cluster_bufmgr_probe_block_for_gcs(req->tag); if (!found && state == PCM_LOCK_MODE_N) { SCN fallback_watermark_scn; @@ -5270,9 +5683,11 @@ gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, * image newer than storage) and must never be overwritten. */ fallback_watermark_scn = cluster_pcm_lock_pi_watermark_scn_query(req->tag); - if (!cluster_pcm_lock_apply_gcs_transition(req->tag, (PcmLockTransition)req->transition_id, - req->sender_node)) { - *out_status = GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE; + apply_result = cluster_pcm_lock_apply_gcs_transition_result( + req->tag, (PcmLockTransition)req->transition_id, req->sender_node); + if (apply_result != PCM_GCS_TRANSITION_APPLIED) { + *out_status = GcsBlockApplyRefusalStatus( + apply_result, (PcmLockTransition)req->transition_id); return true; } pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_storage_fallback_count, 1); @@ -5303,27 +5718,36 @@ gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, if (!found) { pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_master_not_holder_count, 1); + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "produce-no-resident-authority"); *out_status = GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; return true; } /* * D4 bufmgr helper performs HC82 XLogFlush(page_lsn) + content_lock dance - * + HC89 single-retry revalidation. Returns false if revalidation cannot - * stabilize after one retry → DENIED_MASTER_NOT_HOLDER fail-closed. + * + HC89 single-retry revalidation. Conditional BufferContent refusal is + * retried by the owning backend with a fresh reservation/request identity; + * structural and HC89 exhaustion remain DENIED_MASTER_NOT_HOLDER. */ - if (!gcs_block_get_ship_image(req->tag, req->sender_node, true, out_page_lsn, block_buf, - out_block_payload, out_block_lkey, out_release_cb, - out_release_arg, out_sf_dep_vec, out_sf_dep_valid)) { - *out_status = GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; + if (!preprepared_image + && !gcs_block_get_ship_image(req->tag, req->sender_node, true, out_page_lsn, block_buf, + out_block_payload, out_block_lkey, out_release_cb, + out_release_arg, out_sf_dep_vec, out_sf_dep_valid, ©_refusal)) { + char producer_reason[96]; + + snprintf(producer_reason, sizeof(producer_reason), "produce-copy-refused:%s", + cluster_bufmgr_gcs_copy_refusal_name(copy_refusal)); + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, producer_reason); + *out_status = GcsBlockMasterDirectCopyRefusalStatus(copy_refusal); return true; } if (out_sf_dep_valid == NULL || !*out_sf_dep_valid) pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_wal_flush_before_ship_count, 1); /* HC77: master-side is the single transition-apply owner. */ - if (!cluster_pcm_lock_apply_gcs_transition(req->tag, (PcmLockTransition)req->transition_id, - req->sender_node)) { + apply_result = cluster_pcm_lock_apply_gcs_transition_result( + req->tag, (PcmLockTransition)req->transition_id, req->sender_node); + if (apply_result != PCM_GCS_TRANSITION_APPLIED) { gcs_block_release_ship_image(out_release_cb != NULL ? *out_release_cb : NULL, out_release_arg != NULL ? *out_release_arg : NULL); *out_block_payload = NULL; @@ -5331,7 +5755,8 @@ gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, *out_release_cb = NULL; if (out_release_arg != NULL) *out_release_arg = NULL; - *out_status = GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE; + *out_status = GcsBlockApplyRefusalStatus( + apply_result, (PcmLockTransition)req->transition_id); return true; } @@ -5385,6 +5810,9 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo void *block_payload_release_arg = NULL; ClusterSfDepVec sf_dep_vec; bool sf_dep_valid = false; + bool scache_image_prepared = false; + ClusterBufmgrGcsDowngradeOutcome local_downgrade_outcome + = CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; bool queue_pending_x_before = false; uint8 request_flags = 0; @@ -5743,7 +6171,7 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo if (gcs_block_get_ship_image(req->tag, req->sender_node, false, &page_lsn, block_buf, &block_payload, &block_payload_lkey, &block_payload_release_cb, &block_payload_release_arg, - &sf_dep_vec, &sf_dep_valid)) { + &sf_dep_vec, &sf_dep_valid, NULL)) { /* * PGRAC: spec-5.2 §3.5 D11 — active-ITL hard boundary. Even as master+holder, * if WE still have an uncommitted ITL slot on this block @@ -5907,6 +6335,7 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo == GCS_CLEAN_XFER_THIRD_PARTY_DENY) { pg_atomic_fetch_add_u64(&ClusterGcsBlock->clean_page_xfer_third_party_denied_count, 1); pg_atomic_fetch_add_u64(&ClusterGcsBlock->clean_page_xfer_fail_closed_count, 1); + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "clean-third-party-master"); gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER, InvalidXLogRecPtr, NULL); return; @@ -6008,6 +6437,7 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo * precedent above). */ cluster_gcs_block_dedup_remove(dedup_worker_id, &key); + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "live-x-other-holder"); gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER, InvalidXLogRecPtr, NULL); return; @@ -6023,10 +6453,12 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo cluster_gcs_block_dedup_remove(dedup_worker_id, &key); gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_PENDING_X, InvalidXLogRecPtr, NULL); - } else + } else { + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "pending-x-reserve-failed"); gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER, InvalidXLogRecPtr, NULL); + } return; } @@ -6116,12 +6548,14 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo &fwd_hdr, NULL); return; } + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "x-forward-send-failed"); (void)cluster_pcm_lock_clear_pending_x_if(req->tag, req->sender_node); status = GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; page_lsn = InvalidXLogRecPtr; block_payload = NULL; goto build_and_send_reply; } + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "x-state-holder-unroutable"); (void)cluster_pcm_lock_clear_pending_x_if(req->tag, req->sender_node); status = GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; page_lsn = InvalidXLogRecPtr; @@ -6174,7 +6608,7 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo && gcs_block_get_ship_image( req->tag, req->sender_node, false, &page_lsn, block_buf, &block_payload, &block_payload_lkey, &block_payload_release_cb, &block_payload_release_arg, - &sf_dep_vec, &sf_dep_valid)) + &sf_dep_vec, &sf_dep_valid, NULL)) xvs_b2_captured = true; if (!xvs_b2_captured) { @@ -6432,11 +6866,19 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo * state raced / flush unavailable) keeps today's one-shot ship. */ if (cluster_read_scache) { - bool downgraded = cluster_bufmgr_downgrade_x_to_s_for_gcs(req->tag); - - cluster_lever_a_note_downgrade(downgraded); - if (downgraded) + local_downgrade_outcome + = cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image( + req->tag, &page_lsn, block_buf, NULL); + cluster_lever_a_note_downgrade( + local_downgrade_outcome == CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED); + if (local_downgrade_outcome + == CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY) + return; + if (local_downgrade_outcome == CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED) { + scache_image_prepared = true; + block_payload = block_buf; goto scache_downgraded_fall_through; + } } cluster_xp_begin(&xp_ship, CLXP_R_READIMAGE_SHIP); @@ -6444,7 +6886,7 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo if (gcs_block_get_ship_image(req->tag, req->sender_node, true, &page_lsn, block_buf, &block_payload, &block_payload_lkey, &block_payload_release_cb, &block_payload_release_arg, - &sf_dep_vec, &sf_dep_valid)) { + &sf_dep_vec, &sf_dep_valid, NULL)) { status = GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER; if (ClusterGcsBlock != NULL) pg_atomic_fetch_add_u64(&ClusterGcsBlock->cf_xheld_read_ship_count, 1); @@ -6598,7 +7040,8 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo * master state is now S with a resident clean buffer, so produce_reply * serves a durable GRANTED (image + requester N->S quick re-grant). */ scache_downgraded_fall_through: - (void)gcs_block_produce_reply(req, block_buf, &status, &page_lsn, &block_payload, + (void)gcs_block_produce_reply(req, block_buf, scache_image_prepared, &status, &page_lsn, + &block_payload, &block_payload_lkey, &block_payload_release_cb, &block_payload_release_arg, &sf_dep_vec, &sf_dep_valid); if (req->transition_id == PCM_TRANS_N_TO_X || req->transition_id == PCM_TRANS_S_TO_X_UPGRADE) @@ -6918,6 +7361,8 @@ build_and_send_reply: { = has_block_payload && block_payload_release_cb == gcs_block_release_live_sge; if (GcsBlockRequestPayloadIsDirectLandArmed(req)) { + if (!GcsBlockReplyStatusIsDirectLandSendable((GcsBlockReplyStatus)hdr->status)) + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "direct-land-nonsendable"); (void)gcs_block_try_send_direct_reply( req->sender_node, true, hdr, has_block_payload ? block_payload : NULL, has_block_payload ? block_payload_lkey : 0, block_payload_release_cb, @@ -7067,12 +7512,29 @@ gcs_block_direct_fail_slot(ClusterGcsBlockBackendBlock *blk, ClusterGcsBlockOuts const GcsBlockReplyHeader *hdr) { BufferDesc *target_buf; + BufferTag target_tag; + ClusterPcmOwnSnapshot own; + ClusterPcmOwnResult own_result = CLUSTER_PCM_OWN_INVALID; + uint64 request_id; + uint64 request_epoch; + int backend_idx; + int slot_idx; + int direct_state; + int buffer_id = -1; bool prepared; + bool reply_received; Assert(blk != NULL); Assert(slot != NULL); target_buf = slot->direct_target_buf; + target_tag = slot->tag; + request_id = slot->request_id; + request_epoch = slot->request_epoch; + backend_idx = (int)(blk - &gcs_block_backend_blocks[0]); + slot_idx = (int)(slot - &blk->slots[0]); + direct_state = (int)slot->direct_state; + reply_received = slot->reply_received; prepared = slot->direct_target_prepared; slot->direct_state = GCS_BLOCK_DIRECT_ABORTED; slot->direct_abort_reason = reason; @@ -7098,6 +7560,23 @@ gcs_block_direct_fail_slot(ClusterGcsBlockBackendBlock *blk, ClusterGcsBlockOuts ConditionVariableSignal(&slot->reply_cv); LWLockRelease(&blk->lock.lock); gcs_block_direct_finish_target(target_buf, prepared, false, InvalidXLogRecPtr); + if (prepared || authoritative_denial) { + memset(&own, 0, sizeof(own)); + own_result = cluster_bufmgr_pcm_own_snapshot_by_tag(&target_tag, &buffer_id, &own); + elog(LOG, + "cluster GCS block direct abort observation: backend=%d slot=%d request_id=%llu " + "epoch=%llu rel=%u fork=%d blk=%u reason=%d authoritative_denial=%d " + "reply_received_before=%d direct_state_before=%d prepared=%d target_cleanup_done=%d " + "own_result=%d buffer=%d own_state=%u own_generation=%llu own_token=%llu " + "own_flags=0x%x", + backend_idx, slot_idx, (unsigned long long)request_id, + (unsigned long long)request_epoch, target_tag.relNumber, + (int)target_tag.forkNum, target_tag.blockNum, (int)reason, + authoritative_denial ? 1 : 0, reply_received ? 1 : 0, direct_state, + prepared ? 1 : 0, prepared ? 1 : 0, (int)own_result, buffer_id, + own.pcm_state, (unsigned long long)own.generation, + (unsigned long long)own.reservation_token, own.flags); + } } void @@ -7531,6 +8010,7 @@ gcs_block_forward_reply_immediate_deny(const GcsBlockForwardPayload *fwd) char *deny_buf = (char *)palloc0(deny_total); GcsBlockReplyHeader *deny_hdr = (GcsBlockReplyHeader *)deny_buf; + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_FORWARD(fwd, "holder-immediate-deny"); deny_hdr->request_id = fwd->request_id; deny_hdr->epoch = cluster_epoch_get_current(); deny_hdr->sender_node = cluster_node_id; @@ -7557,6 +8037,10 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo void *block_payload_release_arg = NULL; XLogRecPtr page_lsn = InvalidXLogRecPtr; bool holder_ship_ok; + ClusterBufmgrGcsCopyRefusal copy_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE; + bool holder_evicted_injected = false; + ClusterBufmgrGcsDowngradeOutcome remote_downgrade_outcome + = CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; bool remote_downgraded = false; /* spec-6.12a ㉕ — holder accepted the * master's downgrade request */ ClusterSfDepVec sf_dep_vec; @@ -7702,6 +8186,13 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo return; } + /* Decide the holder-eviction fault before any irreversible downgrade. + * A forced DENIED must never first publish X->S and then discard the only + * image prepared for the requester. */ + CLUSTER_INJECTION_POINT("cluster-gcs-block-evict-holder-before-ship"); + holder_evicted_injected + = cluster_injection_should_skip("cluster-gcs-block-evict-holder-before-ship"); + /* * PGRAC: spec-6.12a ㉕ — remote-holder downgrade. The master asked us * (the X holder) to TRY the quiescent X->S self-downgrade before @@ -7718,23 +8209,33 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo && GcsBlockForwardPayloadIsDowngradeRequest(fwd) && fwd->transition_id == PCM_TRANS_N_TO_S) { CLUSTER_INJECTION_POINT("cluster-gcs-block-remote-downgrade"); - if (!cluster_injection_should_skip("cluster-gcs-block-remote-downgrade")) - remote_downgraded - = cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(fwd->tag, fwd->master_node); + if (!holder_evicted_injected + && !cluster_injection_should_skip("cluster-gcs-block-remote-downgrade")) + remote_downgrade_outcome + = cluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image( + fwd->tag, fwd->master_node, &page_lsn, block_buf, ©_refusal); + remote_downgraded + = remote_downgrade_outcome == CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED; cluster_lever_a_note_remote_downgrade(remote_downgraded); } + if (remote_downgrade_outcome == CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY) + return; - /* spec-2.35 D15 — fault injection. SKIP simulates evict race: - * holder pretends buffer is not cached so we exercise the HC105 - * DENIED_MASTER_NOT_HOLDER + sender retransmit budget path. */ - CLUSTER_INJECTION_POINT("cluster-gcs-block-evict-holder-before-ship"); - if (cluster_injection_should_skip("cluster-gcs-block-evict-holder-before-ship")) + /* spec-2.35 D15 — SKIP simulates the evict race. */ + if (holder_evicted_injected) { + copy_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INJECTED_EVICT; holder_ship_ok = false; - else + } else if (remote_downgraded) { + /* The exact downgrade prepared these bytes before notify+commit while + * holding the same content EXCLUSIVE lock. A second conditional copy + * would reopen the grant->ship race P0-32 closes. */ + holder_ship_ok = remote_downgraded; + block_payload = block_buf; + } else holder_ship_ok = gcs_block_get_ship_image( fwd->tag, fwd->original_requester_node, true, &page_lsn, block_buf, &block_payload, &block_payload_lkey, &block_payload_release_cb, &block_payload_release_arg, &sf_dep_vec, - &sf_dep_valid); + &sf_dep_valid, ©_refusal); /* Build reply (header + 8KB block or zero pad) and direct-ship to * the original requester. HC109 stores fwd->master_node in the @@ -7983,11 +8484,14 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo if (dres == CLUSTER_BUFMGR_GCS_DROP_PINNED || dres == CLUSTER_BUFMGR_GCS_DROP_STALE) { - if (dres == CLUSTER_BUFMGR_GCS_DROP_STALE) + if (dres == CLUSTER_BUFMGR_GCS_DROP_STALE) { pg_atomic_fetch_add_u64(&ClusterGcsBlock->xfer_stale_deny_count, 1); - else + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_FORWARD(fwd, "holder-drop-stale"); + } else { pg_atomic_fetch_add_u64(&ClusterGcsBlock->drop_pinned_deny_count, 1); + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_FORWARD(fwd, "holder-drop-pinned"); + } /* undo the from-holder ship count taken with the * grant status above — this reply is a deny now */ pg_atomic_fetch_sub_u64(&ClusterGcsBlock->block_from_holder_ship_count, @@ -8018,9 +8522,20 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo } } else { /* HC105 evict race */ + const char *copy_refusal_name = cluster_bufmgr_gcs_copy_refusal_name(copy_refusal); + hdr->checksum = gcs_block_compute_checksum(buf + header_len); hdr->status = (uint8)GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_forward_holder_evicted_count, 1); + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_FORWARD(fwd, "holder-copy-refused"); + ereport(LOG, + (errmsg("cluster_gcs_block: holder ship image refused"), + errdetail("reason=%s request_id=" UINT64_FORMAT + " requester=%d master=%d tag spc=%u db=%u relNumber=%u block=%u", + copy_refusal_name, fwd->request_id, fwd->original_requester_node, + fwd->master_node, fwd->tag.spcOid, fwd->tag.dbOid, + (unsigned int)BufTagGetRelNumber(&fwd->tag), + (unsigned int)fwd->tag.blockNum))); } send_sf_dep = sf_peer_v2 && sf_dep_valid && block_payload != NULL @@ -8047,6 +8562,8 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo } if (GcsBlockForwardPayloadIsDirectLandArmed(fwd)) { + if (!GcsBlockReplyStatusIsDirectLandSendable((GcsBlockReplyStatus)hdr->status)) + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_FORWARD(fwd, "holder-direct-land-nonsendable"); (void)gcs_block_try_send_direct_reply(fwd->original_requester_node, true, hdr, holder_ship_ok ? block_payload : NULL, holder_ship_ok ? block_payload_lkey : 0, @@ -8216,6 +8733,8 @@ typedef struct GcsBlockPcmXPreflightEvidence { uint32 own_flags; uint64 live_generation; uint64 base_own_generation; + uint64 reservation_token; + uint64 writer_activation_token; } GcsBlockPcmXPreflightEvidence; static GcsBlockPcmXPreflightEvidence gcs_block_pcm_x_requester_preflight_evidence; @@ -9121,6 +9640,9 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim evidence->own_flags = reservation_base.flags; evidence->live_generation = reservation_base.generation; evidence->base_own_generation = progress.identity.base_own_generation; + evidence->reservation_token = reservation_base.reservation_token; + evidence->writer_activation_token + = reservation_base.writer_activation_token; retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) @@ -9130,6 +9652,22 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim own_result = cluster_bufmgr_pcm_own_begin_x_reservation( buf, &reservation_base, &reservation_token); result = gcs_block_pcm_x_fetch_own_result(own_result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { + GcsBlockPcmXPreflightEvidence *evidence + = &gcs_block_pcm_x_requester_preflight_evidence; + + evidence->valid = true; + evidence->rebase_wire_active + = gcs_block_pcm_x_rebase_wire_active(&request_runtime); + evidence->pcm_state = reservation_base.pcm_state; + evidence->own_flags = reservation_base.flags; + evidence->live_generation = reservation_base.generation; + evidence->base_own_generation + = progress.identity.base_own_generation; + evidence->reservation_token = reservation_base.reservation_token; + evidence->writer_activation_token + = reservation_base.writer_activation_token; + } } cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) @@ -9289,10 +9827,15 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim LOG, (errmsg("PCM-X requester reservation-preflight evidence"), errdetail( - "live_gen=%llu base_gen=%llu pcm_state=%u own_flags=%u rebase_active=%d", + "live_gen=%llu base_gen=%llu reservation_token=%llu " + "writer_activation_token=%llu pcm_state=%u own_flags=%u rebase_active=%d", (unsigned long long)gcs_block_pcm_x_requester_preflight_evidence.live_generation, (unsigned long long) gcs_block_pcm_x_requester_preflight_evidence.base_own_generation, + (unsigned long long) + gcs_block_pcm_x_requester_preflight_evidence.reservation_token, + (unsigned long long) + gcs_block_pcm_x_requester_preflight_evidence.writer_activation_token, (unsigned int)gcs_block_pcm_x_requester_preflight_evidence.pcm_state, (unsigned int)gcs_block_pcm_x_requester_preflight_evidence.own_flags, gcs_block_pcm_x_requester_preflight_evidence.rebase_wire_active ? 1 : 0))); @@ -9579,8 +10122,7 @@ cluster_gcs_block_pcm_x_formation_tick(void) return; fail_closed: - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + cluster_pcm_x_runtime_fail_closed(); } @@ -9606,6 +10148,31 @@ cluster_gcs_pcm_x_stage_frame(uint8 msg_type, int32 dest_node_id, const void *pa payload_len); } +/* V2 type-49 is valid only on the exact DATA connection whose verified HELLO + * advertised the source-floor family. Route it to the same tag shard as the + * V1 frame, but retain the capability/generation fence in the LMS ring slot + * until immediately before transport admission. */ +static bool +gcs_block_pcm_x_stage_frame_cap_bound(uint8 msg_type, int32 dest_node_id, + const void *payload, uint16 payload_len, + uint32 required_capability, + uint32 connection_generation) +{ + int worker; + + if (msg_type < PGRAC_IC_MSG_PCM_X_ENQUEUE || msg_type > PGRAC_IC_MSG_PCM_X_RETIRE_ACK + || dest_node_id < 0 || dest_node_id >= PCM_X_PROTOCOL_NODE_LIMIT || payload == NULL + || payload_len == 0 || required_capability == 0 || cluster_lms_workers <= 0) + return false; + worker = cluster_gcs_block_payload_shard(msg_type, payload, payload_len, + cluster_lms_workers); + if (worker < 0 || worker >= cluster_lms_workers) + return false; + return cluster_lms_outbound_enqueue_cap_bound( + worker, msg_type, (uint32)dest_node_id, payload, payload_len, required_capability, + connection_generation); +} + /* RETIRE deliberately carries a cluster-wide ticket watermark rather than a * BufferTag. The LMON terminal driver must therefore supply the ticket's tag @@ -9677,8 +10244,7 @@ gcs_block_pcm_x_master_drive_fail_closed(PcmXQueueResult result) if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED || result == PCM_X_QUEUE_BAD_STATE || result == PCM_X_QUEUE_INVALID || result == PCM_X_QUEUE_NO_CAPACITY) - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + cluster_pcm_x_runtime_fail_closed(); } @@ -9894,18 +10460,43 @@ gcs_block_pcm_x_stage_frame_callback(uint8 msg_type, int32 dest_node_id, const v } +static uint64 +gcs_block_pcm_x_invalidate_busy_retry_delay_ms(const PcmXMasterDriveSnapshot *snapshot) +{ + uint64 delay_ms; + + /* reliable.retry_count belongs to the already-armed REVOKE leg. The + * transfer driver increments it while it is still draining earlier S + * holders, so it can be large before the first INVALIDATE BUSY arrives. + * It is not a BUSY-attempt counter and must not stretch that first retry + * to the 25s saturation ceiling (which phase-locks a denied reader's + * GRANT_PENDING retries against every INVALIDATE). Keep one bounded + * scheduling interval; repeated BUSY replies install fresh exact + * deadlines on the same ticket. */ + (void)snapshot; + delay_ms = (uint64)Max(Max(cluster_gcs_block_starvation_backoff_ms, + cluster_lmon_main_loop_interval), + 1); + return delay_ms; +} + + static PcmXQueueResult gcs_block_pcm_x_master_drive_transfer(const PcmXMasterDriveSnapshot *snapshot) { + ClusterGcsPcmXAuthSample auth_sample; PcmXMasterDriveSnapshot retried; PcmXRevokePayload revoke; + PcmXRevokePayloadV2 revoke_v2; PcmXQueueResult result; uint64 now_ms; uint64 retry_delay_ms; uint64 source_session = 0; + uint32 source_connection_generation = 0; uint32 source_bit; uint32 unacked; int32 source; + bool source_floor_capable = false; if (snapshot == NULL || snapshot->ticket_state != PCM_XT_ACTIVE_TRANSFER) return PCM_X_QUEUE_INVALID; @@ -9922,8 +10513,14 @@ gcs_block_pcm_x_master_drive_transfer(const PcmXMasterDriveSnapshot *snapshot) } if (!cluster_gcs_pcm_x_transfer_pre_handoff_phase(snapshot->pending_opcode)) return PCM_X_QUEUE_NOT_READY; - if (!cluster_pcm_lock_queue_pending_x_exact(snapshot->ref.identity.tag, - snapshot->ref.identity.node_id, + if (snapshot->pending_opcode == PGRAC_IC_MSG_PCM_X_REVOKE + && snapshot->retry_deadline_ms != 0) { + now_ms = (uint64)(GetCurrentTimestamp() / 1000); + if (now_ms < snapshot->retry_deadline_ms) + return PCM_X_QUEUE_NOT_READY; + } + if (!cluster_pcm_lock_queue_pending_x_exact(snapshot->ref.identity.tag, + snapshot->ref.identity.node_id, snapshot->ref.handle.ticket_id)) return PCM_X_QUEUE_BAD_STATE; if (snapshot->pending_opcode == PGRAC_IC_MSG_PCM_X_REVOKE) @@ -9943,10 +10540,16 @@ gcs_block_pcm_x_master_drive_transfer(const PcmXMasterDriveSnapshot *snapshot) if ((snapshot->pending_s_holders_bitmap & source_bit) == 0 || (snapshot->acked_s_holders_bitmap & source_bit) == 0) return PCM_X_QUEUE_CORRUPT; - if (!gcs_block_pcm_x_authenticated_session(source, snapshot->ref.identity.cluster_epoch, - &source_session) + memset(&auth_sample, 0, sizeof(auth_sample)); + if ((source == cluster_node_id + ? !gcs_block_pcm_x_authenticated_session(source, + snapshot->ref.identity.cluster_epoch, + &source_session) + : gcs_block_pcm_x_authenticated_session_result( + source, snapshot->ref.identity.cluster_epoch, &source_session, &auth_sample) + != PCM_X_SESSION_AUTH_OK) || !gcs_block_pcm_x_revalidate_peer_binding(source, snapshot->ref.identity.cluster_epoch, - source_session)) + source_session)) return PCM_X_QUEUE_NOT_READY; result = cluster_pcm_x_master_revoke_arm_exact(&snapshot->ref, source, source_session, &revoke); cluster_pcm_x_stats_note_queue_result(result); @@ -9955,6 +10558,37 @@ gcs_block_pcm_x_master_drive_transfer(const PcmXMasterDriveSnapshot *snapshot) unacked = snapshot->pending_s_holders_bitmap & ~snapshot->acked_s_holders_bitmap; if (unacked != 0) return gcs_block_pcm_x_stage_queue_invalidations(snapshot, unacked); + if (source == cluster_node_id) { + memset(&revoke_v2, 0, sizeof(revoke_v2)); + revoke_v2.v1 = revoke; + revoke_v2.required_page_scn + = (uint64)cluster_pcm_lock_pi_watermark_scn_query(snapshot->ref.identity.tag); + return cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_REVOKE, source, &revoke_v2, + sizeof(revoke_v2)) + ? PCM_X_QUEUE_OK + : PCM_X_QUEUE_BUSY; + } + /* Resample CONVERT + optional SOURCE_FLOOR under one capability-store + * lock after the authority pass. Any reconnect since authentication is + * retryable: the already-armed reliable leg remains intact. */ + if (!cluster_sf_peer_pcm_x_source_floor_sample( + source, &source_floor_capable, &source_connection_generation) + || source_connection_generation != auth_sample.connection_generation_before + || source_connection_generation != auth_sample.connection_generation_after) + return PCM_X_QUEUE_NOT_READY; + if (source_floor_capable) { + memset(&revoke_v2, 0, sizeof(revoke_v2)); + revoke_v2.v1 = revoke; + revoke_v2.required_page_scn + = (uint64)cluster_pcm_lock_pi_watermark_scn_query(snapshot->ref.identity.tag); + return gcs_block_pcm_x_stage_frame_cap_bound( + PGRAC_IC_MSG_PCM_X_REVOKE, source, &revoke_v2, sizeof(revoke_v2), + PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 + | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1, + source_connection_generation) + ? PCM_X_QUEUE_OK + : PCM_X_QUEUE_BUSY; + } return cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_REVOKE, source, &revoke, sizeof(revoke)) ? PCM_X_QUEUE_OK : PCM_X_QUEUE_BUSY; @@ -10130,29 +10764,72 @@ gcs_block_pcm_x_master_drive_cancel_requested(const PcmXMasterDriveSnapshot *sna * that repeats consecutively with the same shape is a stalled drive, and a * frozen wedge must name its refusing arm. Progress resets the streak. */ static void -gcs_block_pcm_x_master_drive_note(const char *stage, PcmXQueueResult result, uint64 ticket_id) +gcs_block_pcm_x_master_drive_note(const char *stage, PcmXQueueResult result, const BufferTag *tag, + uint64 cluster_epoch, const PcmXMasterDriveSnapshot *snapshot) { static const char *last_stage = NULL; static uint64 last_ticket = 0; + static uint64 last_request = 0; + static uint64 last_epoch = 0; + static BufferTag last_tag; + static bool last_tag_valid = false; static int last_result = 0; static bool logged = false; + uint64 ticket_id = snapshot != NULL ? snapshot->ref.handle.ticket_id : 0; + uint64 request_id = snapshot != NULL ? snapshot->ref.identity.request_id : 0; + bool same_tag + = tag == NULL ? !last_tag_valid : last_tag_valid && BufferTagsEqual(tag, &last_tag); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { last_stage = NULL; + last_tag_valid = false; logged = false; return; } - if (stage == last_stage && (int)result == last_result && ticket_id == last_ticket) { + if (stage == last_stage && (int)result == last_result && ticket_id == last_ticket + && request_id == last_request && cluster_epoch == last_epoch && same_tag) { if (!logged) { logged = true; - ereport(LOG, (errmsg("PCM-X master drive is repeating %s (result %d, ticket %llu)", - stage, (int)result, (unsigned long long)ticket_id))); + if (tag != NULL) + ereport(LOG, + (errmsg("PCM-X master drive is repeating %s", stage), + errdetail( + "result=%d epoch=%llu tag=%u/%u/%u/%d/%u " + "requester=%d procno=%u xid=%u request_id=%llu wait_seq=%llu " + "ticket=%llu queue_generation=%llu grant_generation=%llu " + "image_id=%llu source_generation=%llu source_node=%u", + (int)result, (unsigned long long)cluster_epoch, tag->spcOid, + tag->dbOid, tag->relNumber, (int)tag->forkNum, tag->blockNum, + snapshot != NULL ? snapshot->ref.identity.node_id : -1, + snapshot != NULL ? snapshot->ref.identity.procno : 0, + snapshot != NULL ? snapshot->ref.identity.xid : 0, + (unsigned long long)request_id, + (unsigned long long)(snapshot != NULL ? snapshot->ref.identity.wait_seq + : 0), + (unsigned long long)ticket_id, + (unsigned long long)(snapshot != NULL + ? snapshot->ref.handle.queue_generation + : 0), + (unsigned long long)(snapshot != NULL ? snapshot->ref.grant_generation + : 0), + (unsigned long long)(snapshot != NULL ? snapshot->image.image_id : 0), + (unsigned long long)(snapshot != NULL + ? snapshot->image.source_own_generation + : 0), + snapshot != NULL ? snapshot->image.source_node : 0))); } return; } last_stage = stage; last_result = (int)result; last_ticket = ticket_id; + last_request = request_id; + last_epoch = cluster_epoch; + if (tag != NULL) { + last_tag = *tag; + last_tag_valid = true; + } else + last_tag_valid = false; logged = false; } @@ -10246,19 +10923,25 @@ gcs_block_pcm_x_master_drive_tag(const BufferTag *tag, uint64 cluster_epoch) result = cluster_pcm_x_master_promote_head_exact(tag, cluster_epoch, &active); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_BUSY) { + PcmXMasterDriveSnapshot observed; + PcmXMasterDriveSnapshot *observed_ptr = NULL; + + if (cluster_pcm_x_master_drive_snapshot_exact(tag, cluster_epoch, &observed) + == PCM_X_QUEUE_OK) + observed_ptr = &observed; /* No promotable head means any earlier per-ticket anomaly for this * tag has resolved (cancelled / retired); settle its streaks. */ if (result == PCM_X_QUEUE_NOT_FOUND) cluster_gcs_pcm_x_drive_anomaly_settle(gcs_block_pcm_x_drive_anomaly_table, GCS_BLOCK_PCM_X_DRIVE_ANOMALY_SLOTS, tag); - gcs_block_pcm_x_master_drive_note("promote", result, 0); + gcs_block_pcm_x_master_drive_note("promote", result, tag, cluster_epoch, observed_ptr); gcs_block_pcm_x_master_drive_fail_closed(result); return; } result = cluster_pcm_x_master_drive_snapshot_exact(tag, cluster_epoch, &snapshot); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK) { - gcs_block_pcm_x_master_drive_note("snapshot", result, 0); + gcs_block_pcm_x_master_drive_note("snapshot", result, tag, cluster_epoch, NULL); gcs_block_pcm_x_master_drive_fail_closed(result); return; } @@ -10285,7 +10968,7 @@ gcs_block_pcm_x_master_drive_tag(const BufferTag *tag, uint64 cluster_epoch) result = PCM_X_QUEUE_CORRUPT; } cluster_pcm_x_stats_note_queue_result(result); - gcs_block_pcm_x_master_drive_note(drive_stage, result, snapshot.ref.handle.ticket_id); + gcs_block_pcm_x_master_drive_note(drive_stage, result, tag, cluster_epoch, &snapshot); /* Only definite drive progress settles the tag; indeterminate results * (NOT_READY / BUSY / STALE) must not reset a live streak, or a real * wedge interleaved with transients would never fuse. */ @@ -10753,8 +11436,7 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en commit->nblockers); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK) { - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + cluster_pcm_x_runtime_fail_closed(); goto blocker_commit_done; } for (i = 0; i < commit->nblockers; i++) @@ -10765,8 +11447,7 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en &waiter, blockers, (int)commit->nblockers, waiter.request_id); if (graph_generation == 0) { cluster_lmd_pcm_convert_wfg_note_replace_fail(); - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + cluster_pcm_x_runtime_fail_closed(); goto blocker_commit_done; } cluster_lmd_pcm_convert_wfg_note_replace(); @@ -10777,8 +11458,7 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en /* The ticket set has already been published. Removing by waiter here * could delete a concurrently newer graph generation; retain evidence * and close the runtime for recovery instead. */ - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + cluster_pcm_x_runtime_fail_closed(); goto blocker_commit_done; } @@ -10790,8 +11470,7 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en * this ACK. */ if (!cluster_gcs_pcm_x_blocker_ack_build(&commit->ref, commit->set_generation, &ack)) { Assert(false); - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + cluster_pcm_x_runtime_fail_closed(); goto blocker_commit_done; } if (!cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK, source_node, &ack, @@ -10809,8 +11488,7 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en * the ACK above was merely replayed after a legal phase advance. The * classifier keeps BAD_STATE/STALE/NOT_READY/BUSY benign in that replay * case while structural CORRUPT still halts the runtime. */ - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + cluster_pcm_x_runtime_fail_closed(); goto blocker_commit_done; } gcs_block_pcm_x_master_drive_tag(&commit->ref.identity.tag, commit->ref.identity.cluster_epoch); @@ -11365,6 +12043,13 @@ gcs_block_pcm_x_local_retire_apply_and_wake(const PcmXRetirePayload *request, wake_batch.count = 0; result = cluster_pcm_x_local_retire_up_to_collect_exact( request, authenticated_master_node, authenticated_master_session, &wake_batch); + if ((result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + && !cluster_gcs_block_dedup_pcm_x_retire_up_to( + request->cluster_epoch, authenticated_master_node, authenticated_master_session, + request->retire_through_ticket_id)) { + cluster_pcm_x_runtime_fail_closed(); + result = PCM_X_QUEUE_CORRUPT; + } if (result == PCM_X_QUEUE_OK) { for (i = 0; i < wake_batch.count && i < wake_batch.capacity; i++) (void)gcs_block_pcm_x_wake_requester_exact(&wake_batch.items[i], all_proc_count); @@ -11439,7 +12124,8 @@ gcs_block_pcm_x_image_key(const PcmXTicketRef *ref, uint64 image_id, GcsBlockDed static void gcs_block_pcm_x_reserved_binding(const PcmXRevokePayload *revoke, uint64 source_generation, - uint64 master_session, GcsBlockPcmXImageBinding *binding_out) + uint64 master_session, uint64 required_page_scn, + GcsBlockPcmXImageBinding *binding_out) { memset(binding_out, 0, sizeof(*binding_out)); binding_out->identity.ref = revoke->ref; @@ -11447,6 +12133,7 @@ gcs_block_pcm_x_reserved_binding(const PcmXRevokePayload *revoke, uint64 source_ binding_out->identity.image.source_own_generation = source_generation; binding_out->identity.image.source_node = (uint32)cluster_node_id; binding_out->master_session = master_session; + binding_out->required_page_scn = required_page_scn; } @@ -11464,7 +12151,8 @@ static bool gcs_block_pcm_x_release_image_exact(int worker_id, const GcsBlockPcmXImageWork *work, const GcsBlockPcmXImageBinding *binding) { - return cluster_gcs_block_dedup_pcm_x_release_exact(worker_id, &work->key, &work->tag, binding) + return cluster_gcs_block_dedup_pcm_x_release_exact(worker_id, &work->key, &work->tag, binding, + -1) == GCS_BLOCK_PCM_X_IMAGE_RELEASED; } @@ -11499,26 +12187,49 @@ gcs_block_pcm_x_abort_image_before_finish(int worker_id, const GcsBlockPcmXImage } +static void gcs_block_pcm_x_revoke_refusal_note_exact(const char *site, int own_result, + const ClusterPcmOwnSnapshot *snap, + const PcmXTicketRef *ref, uint64 image_id, + uint64 source_generation); +static void gcs_block_pcm_x_image_ready_arm_refusal_note_work( + int result, const GcsBlockPcmXImageWork *work, PcmXLocalImageReadyRefusal refusal); + + static void gcs_block_pcm_x_stage_ready_work(int worker_id, const GcsBlockPcmXImageWork *work) { PcmXGrantPayload image_ready; PcmXGrantPayload replay; + PcmXLocalImageReadyRefusal refusal; PcmXQueueResult result; GcsBlockPcmXImageResult mark_result; GcsBlockPcmXImageResult rollback_result; int32 master_node; + bool diagnostic; + bool stage_result; if (!cluster_pcm_x_image_id_decode(work->binding.identity.image.image_id, &master_node, NULL)) { cluster_pcm_x_runtime_fail_closed(); return; } + diagnostic = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); memset(&image_ready, 0, sizeof(image_ready)); image_ready.ref = work->binding.identity.ref; image_ready.image = work->binding.identity.image; - result = cluster_pcm_x_local_holder_image_ready_arm_exact( - &image_ready, master_node, work->binding.master_session, &replay); + result = cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( + &image_ready, master_node, work->binding.master_session, &replay, &refusal); cluster_pcm_x_stats_note_queue_result(result); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X IMAGE_READY stage boundary: arm"), + errdetail("result=%d refusal=%d master=%d request_id=%llu ticket=%llu " + "grant_generation=%llu image_id=%llu", + (int)result, (int)refusal, master_node, + (unsigned long long)image_ready.ref.identity.request_id, + (unsigned long long)image_ready.ref.handle.ticket_id, + (unsigned long long)image_ready.ref.grant_generation, + (unsigned long long)image_ready.image.image_id))); + gcs_block_pcm_x_image_ready_arm_refusal_note_work((int)result, work, refusal); if (result == PCM_X_QUEUE_BUSY || result == PCM_X_QUEUE_NOT_READY) return; if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { @@ -11527,25 +12238,293 @@ gcs_block_pcm_x_stage_ready_work(int worker_id, const GcsBlockPcmXImageWork *wor } mark_result = cluster_gcs_block_dedup_pcm_x_mark_staged_exact(worker_id, &work->key, &work->tag, &work->binding); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X IMAGE_READY stage boundary: mark"), + errdetail("mark_result=%d worker=%d request_id=%llu ticket=%llu image_id=%llu", + (int)mark_result, worker_id, + (unsigned long long)image_ready.ref.identity.request_id, + (unsigned long long)image_ready.ref.handle.ticket_id, + (unsigned long long)image_ready.image.image_id))); if (mark_result == GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) return; if (mark_result != GCS_BLOCK_PCM_X_IMAGE_STAGED) { cluster_pcm_x_runtime_fail_closed(); return; } - if (cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_IMAGE_READY, master_node, &replay, - sizeof(replay))) + stage_result = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_IMAGE_READY, master_node, + &replay, sizeof(replay)); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X IMAGE_READY stage boundary: DATA ring"), + errdetail("stage_result=%s worker=%d master=%d request_id=%llu ticket=%llu " + "image_id=%llu", + stage_result ? "true" : "false", worker_id, master_node, + (unsigned long long)image_ready.ref.identity.request_id, + (unsigned long long)image_ready.ref.handle.ticket_id, + (unsigned long long)image_ready.image.image_id))); + if (stage_result) return; rollback_result = cluster_gcs_block_dedup_pcm_x_unmark_staged_exact(worker_id, &work->key, - &work->tag, &work->binding); + &work->tag, &work->binding); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X IMAGE_READY stage boundary: rollback"), + errdetail("rollback_result=%d worker=%d request_id=%llu ticket=%llu " + "image_id=%llu", + (int)rollback_result, worker_id, + (unsigned long long)image_ready.ref.identity.request_id, + (unsigned long long)image_ready.ref.handle.ticket_id, + (unsigned long long)image_ready.image.image_id))); if (rollback_result != GCS_BLOCK_PCM_X_IMAGE_REARMED && rollback_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) cluster_pcm_x_runtime_fail_closed(); } -static void gcs_block_pcm_x_revoke_refusal_note(const char *site, int own_result, - const ClusterPcmOwnSnapshot *snap); +static void gcs_block_pcm_x_revoke_refusal_note_exact_diagnosed( + const char *site, int own_result, const ClusterPcmOwnSnapshot *snap, + const PcmXTicketRef *ref, uint64 image_id, uint64 source_generation, + const ClusterPcmOwnFinishRefusal *finish_refusal); +static void +gcs_block_pcm_x_revoke_refusal_note_work(const char *site, int own_result, + const ClusterPcmOwnSnapshot *snap, + const GcsBlockPcmXImageWork *work) +{ + gcs_block_pcm_x_revoke_refusal_note_exact( + site, own_result, snap, work != NULL ? &work->binding.identity.ref : NULL, + work != NULL ? work->binding.identity.image.image_id : 0, + work != NULL ? work->binding.identity.image.source_own_generation : 0); +} + +static void +gcs_block_pcm_x_revoke_refusal_note_finish_work( + int own_result, const ClusterPcmOwnSnapshot *snap, const GcsBlockPcmXImageWork *work, + const ClusterPcmOwnFinishRefusal *finish_refusal) +{ + const char *site = "materialized-finish-other"; + + if (finish_refusal != NULL) { + if (finish_refusal->reason == CLUSTER_PCM_OWN_FINISH_REFUSAL_VM_FSM_PINNED) + site = "materialized-finish-vm-fsm-pinned"; + else if (finish_refusal->reason == CLUSTER_PCM_OWN_FINISH_REFUSAL_IO_IN_PROGRESS) + site = "materialized-finish-io-in-progress"; + else if (finish_refusal->reason == CLUSTER_PCM_OWN_FINISH_REFUSAL_LIVE_FLAGS) + site = "materialized-finish-live-flags"; + } + gcs_block_pcm_x_revoke_refusal_note_exact_diagnosed( + site, own_result, snap, work != NULL ? &work->binding.identity.ref : NULL, + work != NULL ? work->binding.identity.image.image_id : 0, + work != NULL ? work->binding.identity.image.source_own_generation : 0, finish_refusal); +} + +static void +gcs_block_pcm_x_revoke_refusal_note_source_prepare_work( + int own_result, const ClusterPcmOwnSnapshot *snap, const GcsBlockPcmXImageWork *work, + ClusterPcmOwnSourcePrepareRefusal refusal) +{ + const char *site = "materialize-begin-s-other"; + + if (refusal == CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_BEGIN_REVOKE) + site = "materialize-begin-s-revoke"; + else if (refusal == CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_CONTENT_LOCK) + site = "materialize-begin-s-content-lock"; + else if (refusal == CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_DIRTY_FLUSHED) + site = "materialize-begin-s-dirty-flushed"; + else if (refusal == CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_DIRTY_RACED) + site = "materialize-begin-s-dirty-raced"; + else if (refusal == CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_IO_IN_PROGRESS) + site = "materialize-begin-s-io-in-progress"; + gcs_block_pcm_x_revoke_refusal_note_work(site, own_result, snap, work); +} + + +static void +gcs_block_pcm_x_image_ready_arm_refusal_note_work(int result, + const GcsBlockPcmXImageWork *work, + PcmXLocalImageReadyRefusal refusal) +{ + const char *site = "image-ready-arm-other"; + + if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_NONE) { + gcs_block_pcm_x_revoke_refusal_note_exact(NULL, 0, NULL, NULL, 0, 0); + return; + } + if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_INVALID) + site = "image-ready-arm-invalid"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_RUNTIME) + site = "image-ready-arm-runtime"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_DIRECTORY) + site = "image-ready-arm-directory"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_TAG_SLOT) + site = "image-ready-arm-tag-slot"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_LOCAL_GATE) + site = "image-ready-arm-local-gate"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_RUNTIME_RECHECK) + site = "image-ready-arm-runtime-recheck"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_IDENTITY) + site = "image-ready-arm-identity"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_ACTIVE_HOLDER) + site = "image-ready-arm-active-holder"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_RELIABLE_LEG) + site = "image-ready-arm-reliable-leg"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_BAD_PHASE) + site = "image-ready-arm-bad-phase"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_COUNTER) + site = "image-ready-arm-counter"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_GATE_RELEASE) + site = "image-ready-arm-gate-release"; + gcs_block_pcm_x_revoke_refusal_note_work(site, result, NULL, work); +} + +/* FlushBuffer may ERROR after immutable bytes have been staged but before the + * ownership commit. bufmgr first releases its content lock and raw pin. The + * A-record and exact REVOKING token are now recovery evidence across an + * irreversible boundary: validate them, trip the runtime fuse, and never run + * the pre-materialization rollback helper from this catch. The DATA worker + * absorbs the ERROR after preserving it: an aux-process rethrow would turn the + * local I/O failure into a postmaster-wide child crash. */ +static ClusterPcmOwnResult +gcs_block_pcm_x_finish_revoke_retain(int worker_id, const GcsBlockPcmXImageWork *work, + const GcsBlockPcmXImageBinding *binding, BufferDesc *buf, + const ClusterPcmOwnSnapshot *revoking, XLogRecPtr page_lsn, + ClusterPcmOwnSnapshot *retained, + ClusterPcmOwnFinishRefusal *finish_refusal) +{ + MemoryContext error_context = CurrentMemoryContext; + volatile ClusterPcmOwnResult result = CLUSTER_PCM_OWN_INVALID; + + PG_TRY(); + { + result = cluster_bufmgr_pcm_own_finish_revoke_retain( + buf, revoking, page_lsn, retained, finish_refusal); + } + PG_CATCH(); + { + ErrorData *original_error; + GcsBlockPcmXImageResult preserve_result; + + MemoryContextSwitchTo(error_context); + original_error = CopyErrorData(); + FlushErrorState(); + preserve_result = cluster_gcs_block_dedup_pcm_x_preserve_finish_error_exact( + worker_id, &work->key, &work->tag, binding, revoking->reservation_token, + revoking->pcm_state); + if (preserve_result != GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING) + gcs_block_pcm_x_revoke_refusal_note_work("finish-error-evidence", (int)preserve_result, + revoking, work); + ereport(LOG, + (errmsg_internal("PCM-X finish-error evidence exact"), + errdetail("preserve_result=%d retained=%s worker=%d tag=%u/%u/%u/%d/%u " + "requester=%u backend=%d request_id=%llu ticket=%llu " + "queue_generation=%llu grant_generation=%llu image_id=%llu " + "reservation_token=%llu source_state=%u", + (int)preserve_result, + preserve_result == GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING ? "true" + : "false", + worker_id, work->tag.spcOid, work->tag.dbOid, work->tag.relNumber, + (int)work->tag.forkNum, work->tag.blockNum, + work->key.origin_node_id, work->key.requester_backend_id, + (unsigned long long)work->binding.identity.ref.identity.request_id, + (unsigned long long)work->binding.identity.ref.handle.ticket_id, + (unsigned long long)work->binding.identity.ref.handle.queue_generation, + (unsigned long long)work->binding.identity.ref.grant_generation, + (unsigned long long)work->binding.identity.image.image_id, + (unsigned long long)revoking->reservation_token, + (unsigned)revoking->pcm_state))); + cluster_pcm_x_runtime_fail_closed(); + ereport(LOG, (errmsg_internal("cluster PCM-X retained-image finish FlushBuffer failed; " + "preserved immutable evidence and blocked recovery: %s", + original_error->message != NULL ? original_error->message + : "(no message)"))); + FreeErrorData(original_error); + result = CLUSTER_PCM_OWN_CORRUPT; + } + PG_END_TRY(); + return (ClusterPcmOwnResult)result; +} + + +/* Retry only the reversible lock-acquisition part after an immutable A-record + * exists. The by-value work token carries the original generation, revoke + * token, source state, tag, ticket and image identity, so a descriptor ABA or + * a different transfer can never inherit the staged bytes. */ +static void +gcs_block_pcm_x_finish_materialized_work(int worker_id, const GcsBlockPcmXImageWork *work) +{ + ClusterPcmOwnSnapshot current; + ClusterPcmOwnSnapshot retained; + ClusterPcmOwnFinishRefusal finish_refusal; + ClusterPcmOwnResult own_result; + ClusterPcmXRevokeFinishMode finish_mode; + GcsBlockPcmXImageResult image_result; + GcsBlockPcmXImageWork ready_work; + BufferDesc *buf; + bool descriptor_retained; + int buffer_id; + + if (work->reservation_token == 0 + || (work->source_pcm_state != (uint8)PCM_STATE_S + && work->source_pcm_state != (uint8)PCM_STATE_X)) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + own_result = cluster_bufmgr_pcm_own_snapshot_by_tag(&work->tag, &buffer_id, ¤t); + gcs_block_pcm_x_note_own_result(own_result); + if (own_result == CLUSTER_PCM_OWN_NOT_READY) { + gcs_block_pcm_x_revoke_refusal_note_work("materialized-snapshot", (int)own_result, ¤t, + work); + return; + } + if (own_result != CLUSTER_PCM_OWN_OK || !BufferTagsEqual(¤t.tag, &work->tag) + || current.generation != work->binding.identity.image.source_own_generation + || current.reservation_token != work->reservation_token + || current.flags != PCM_OWN_FLAG_REVOKING || current.pcm_state != work->source_pcm_state) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + + buf = GetBufferDescriptor(buffer_id); + own_result = gcs_block_pcm_x_finish_revoke_retain( + worker_id, work, &work->binding, buf, ¤t, + (XLogRecPtr)work->binding.identity.image.page_lsn, &retained, &finish_refusal); + gcs_block_pcm_x_note_own_result(own_result); + if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY) { + gcs_block_pcm_x_revoke_refusal_note_finish_work((int)own_result, ¤t, work, + &finish_refusal); + return; + } + if (own_result != CLUSTER_PCM_OWN_OK) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + + finish_mode = cluster_pcm_x_revoke_finish_mode(¤t.tag, 0); + descriptor_retained = finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_RETAIN; + if (current.generation == UINT64_MAX || retained.generation != current.generation + 1 + || retained.reservation_token != current.reservation_token + || (descriptor_retained ? retained.flags != PCM_OWN_FLAG_REVOKING : retained.flags != 0) + || retained.pcm_state != (uint8)PCM_STATE_N + || (finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_RETAIN + && finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_DROP) + || !BufferTagsEqual(&retained.tag, ¤t.tag)) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + image_result = cluster_gcs_block_dedup_pcm_x_publish_ready_exact(worker_id, &work->key, + &work->tag, &work->binding); + if (image_result != GCS_BLOCK_PCM_X_IMAGE_STORED + && image_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + ready_work = *work; + ready_work.reservation_token = 0; + ready_work.source_pcm_state = 0; + ready_work.entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE; + gcs_block_pcm_x_revoke_refusal_note_exact(NULL, 0, NULL, NULL, 0, 0); + gcs_block_pcm_x_stage_ready_work(worker_id, &ready_work); +} static void gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImageWork *work) @@ -11554,12 +12533,16 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage ClusterPcmOwnSnapshot current; ClusterPcmOwnSnapshot retained; ClusterPcmOwnSnapshot revoking; + ClusterPcmOwnFinishRefusal finish_refusal; + ClusterPcmOwnSourcePrepareRefusal source_prepare_refusal + = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_NONE; GcsBlockPcmXImageBinding ready_binding; GcsBlockPcmXImageResult image_result; GcsBlockReplyHeader reply_header; PcmXLocalHolderSnapshot holder_snapshot; PcmXQueueResult holder_result; ClusterPcmOwnResult own_result; + volatile ClusterPcmOwnResult n_prepare_result = CLUSTER_PCM_OWN_INVALID; ClusterPcmXRevokeFinishMode finish_mode; PageHeaderData page_header; BufferDesc *buf; @@ -11568,19 +12551,20 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage bool source_is_n; bool source_is_s; bool source_is_x; + volatile bool copy_ok = false; bool descriptor_retained; bool self_source_handoff; int buffer_id; holder_result = cluster_pcm_x_local_holder_snapshot(&work->tag, NULL, 0, &holder_snapshot); if (holder_result == PCM_X_QUEUE_NO_CAPACITY && holder_snapshot.holder_count > 0) { - gcs_block_pcm_x_revoke_refusal_note("materialize-holder-capacity", (int)holder_result, - NULL); + gcs_block_pcm_x_revoke_refusal_note_work("materialize-holder-capacity", (int)holder_result, + NULL, work); return; } if (holder_result == PCM_X_QUEUE_NOT_READY || holder_result == PCM_X_QUEUE_BUSY) { - gcs_block_pcm_x_revoke_refusal_note("materialize-holder-snapshot", (int)holder_result, - NULL); + gcs_block_pcm_x_revoke_refusal_note_work("materialize-holder-snapshot", (int)holder_result, + NULL, work); return; } if (holder_result != PCM_X_QUEUE_OK || holder_snapshot.holder_count != 0) { @@ -11591,7 +12575,8 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage own_result = cluster_bufmgr_pcm_own_snapshot_by_tag(&work->tag, &buffer_id, ¤t); gcs_block_pcm_x_note_own_result(own_result); if (own_result == CLUSTER_PCM_OWN_NOT_READY) { - gcs_block_pcm_x_revoke_refusal_note("materialize-snapshot", (int)own_result, ¤t); + gcs_block_pcm_x_revoke_refusal_note_work("materialize-snapshot", (int)own_result, ¤t, + work); return; } source_is_n = current.pcm_state == (uint8)PCM_STATE_N; @@ -11607,16 +12592,47 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage return; } buf = GetBufferDescriptor(buffer_id); - if (source_is_n) - own_result = cluster_bufmgr_pcm_own_prepare_n_source_image( - buf, ¤t, &revoking, block_data, &page_lsn, &page_scn); - else if (source_is_s) - own_result = cluster_bufmgr_pcm_own_begin_s_revoke(buf, ¤t, &revoking); + if (source_is_n) { + PG_TRY(); + { + n_prepare_result = cluster_bufmgr_pcm_own_prepare_n_source_image( + buf, ¤t, &revoking, block_data, &page_lsn, &page_scn); + } + PG_CATCH(); + { + if (!gcs_block_pcm_x_release_image_exact(worker_id, work, &work->binding)) + cluster_pcm_x_runtime_fail_closed(); + PG_RE_THROW(); + } + PG_END_TRY(); + own_result = (ClusterPcmOwnResult)n_prepare_result; + } else if (source_is_s) { + PG_TRY(); + { + n_prepare_result = cluster_bufmgr_pcm_own_prepare_s_source_image( + buf, ¤t, (SCN)work->binding.required_page_scn, &revoking, block_data, + &page_lsn, &page_scn, &source_prepare_refusal); + } + PG_CATCH(); + { + if (!gcs_block_pcm_x_release_image_exact(worker_id, work, &work->binding)) + cluster_pcm_x_runtime_fail_closed(); + PG_RE_THROW(); + } + PG_END_TRY(); + own_result = (ClusterPcmOwnResult)n_prepare_result; + } else own_result = cluster_bufmgr_pcm_own_begin_x_revoke(buf, ¤t, &revoking); gcs_block_pcm_x_note_own_result(own_result); - if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY) { - gcs_block_pcm_x_revoke_refusal_note("materialize-begin", (int)own_result, ¤t); + if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY + || own_result == CLUSTER_PCM_OWN_STALE) { + if (source_is_s) + gcs_block_pcm_x_revoke_refusal_note_source_prepare_work( + (int)own_result, ¤t, work, source_prepare_refusal); + else + gcs_block_pcm_x_revoke_refusal_note_work("materialize-begin", (int)own_result, + ¤t, work); return; } if (own_result != CLUSTER_PCM_OWN_OK) { @@ -11625,16 +12641,30 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage return; } - if (!source_is_n && !cluster_bufmgr_copy_block_for_gcs(work->tag, &page_lsn, block_data)) { - gcs_block_pcm_x_revoke_refusal_note("materialize-copy", (int)CLUSTER_PCM_OWN_BUSY, - &revoking); + if (source_is_x) { + PG_TRY(); + { + copy_ok = cluster_bufmgr_copy_block_for_gcs(work->tag, &page_lsn, block_data, NULL); + } + PG_CATCH(); + { + gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &work->binding, buf, + &revoking, true); + PG_RE_THROW(); + } + PG_END_TRY(); + } + if (source_is_x && !copy_ok) { + gcs_block_pcm_x_revoke_refusal_note_work("materialize-copy", (int)CLUSTER_PCM_OWN_BUSY, + &revoking, work); gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &work->binding, buf, &revoking, true); return; } memcpy(&page_header, block_data, sizeof(page_header)); if ((uint64)page_lsn != (uint64)PageXLogRecPtrGet(page_header.pd_lsn) - || (source_is_n && page_scn != (uint64)page_header.pd_block_scn)) { + || ((source_is_n || source_is_s) + && page_scn != (uint64)page_header.pd_block_scn)) { gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &work->binding, buf, &revoking, true); return; @@ -11656,7 +12686,8 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage reply_header.status = (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER; GcsBlockReplyHeaderSetForwardingMasterNode(&reply_header, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); image_result = cluster_gcs_block_dedup_pcm_x_materialize( - worker_id, &work->key, &work->tag, &ready_binding, &reply_header, block_data); + worker_id, &work->key, &work->tag, &ready_binding, revoking.reservation_token, + revoking.pcm_state, &reply_header, block_data); if (image_result != GCS_BLOCK_PCM_X_IMAGE_STORED && image_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) { gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &work->binding, buf, &revoking, @@ -11682,14 +12713,18 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage return; } } else { - own_result - = cluster_bufmgr_pcm_own_finish_revoke_retain(buf, &revoking, page_lsn, &retained); + own_result = gcs_block_pcm_x_finish_revoke_retain(worker_id, work, &ready_binding, buf, + &revoking, page_lsn, &retained, &finish_refusal); gcs_block_pcm_x_note_own_result(own_result); + if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY) { + gcs_block_pcm_x_revoke_refusal_note_work("materialize-finish", (int)own_result, + &revoking, work); + return; + } if (own_result != CLUSTER_PCM_OWN_OK) { - gcs_block_pcm_x_revoke_refusal_note("materialize-finish", (int)own_result, - &revoking); - gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &ready_binding, buf, - &revoking, true); + /* Immutable bytes exist: do not erase the A-record or exact revoke + * evidence on a failed irreversible-boundary check. */ + cluster_pcm_x_runtime_fail_closed(); return; } finish_mode = cluster_pcm_x_revoke_finish_mode(&revoking.tag, 0); @@ -11727,7 +12762,7 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage ready_work.binding = ready_binding; ready_work.entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE; - gcs_block_pcm_x_revoke_refusal_note(NULL, 0, NULL); + gcs_block_pcm_x_revoke_refusal_note_exact(NULL, 0, NULL, NULL, 0, 0); gcs_block_pcm_x_stage_ready_work(worker_id, &ready_work); } } @@ -11771,7 +12806,8 @@ cluster_gcs_block_pcm_x_image_pump_tick(int worker_id) result = cluster_gcs_block_dedup_pcm_x_next_work(worker_id, &work); if (result == GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND || result == GCS_BLOCK_PCM_X_IMAGE_FULL) return; - if (result != GCS_BLOCK_PCM_X_IMAGE_RESERVED && result != GCS_BLOCK_PCM_X_IMAGE_REPLAY) { + if (result != GCS_BLOCK_PCM_X_IMAGE_RESERVED && result != GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING + && result != GCS_BLOCK_PCM_X_IMAGE_REPLAY) { cluster_pcm_x_runtime_fail_closed(); return; } @@ -11781,6 +12817,8 @@ cluster_gcs_block_pcm_x_image_pump_tick(int worker_id) } if (result == GCS_BLOCK_PCM_X_IMAGE_RESERVED) gcs_block_pcm_x_materialize_reserved_work(worker_id, &work); + else if (result == GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING) + gcs_block_pcm_x_finish_materialized_work(worker_id, &work); else gcs_block_pcm_x_stage_ready_work(worker_id, &work); } @@ -11794,36 +12832,114 @@ cluster_gcs_block_pcm_x_image_pump_tick(int worker_id) * every refusal arm here is silent and the master keeps re-sending forever. * A NULL site resets the streak (the revoke made progress). */ static void -gcs_block_pcm_x_revoke_refusal_note(const char *site, int own_result, - const ClusterPcmOwnSnapshot *snap) +gcs_block_pcm_x_revoke_refusal_note_exact(const char *site, int own_result, + const ClusterPcmOwnSnapshot *snap, + const PcmXTicketRef *ref, uint64 image_id, + uint64 source_generation) +{ + gcs_block_pcm_x_revoke_refusal_note_exact_diagnosed(site, own_result, snap, ref, image_id, + source_generation, NULL); +} + +static void +gcs_block_pcm_x_revoke_refusal_note_exact_diagnosed( + const char *site, int own_result, const ClusterPcmOwnSnapshot *snap, + const PcmXTicketRef *ref, uint64 image_id, uint64 source_generation, + const ClusterPcmOwnFinishRefusal *finish_refusal) { static const char *last_site = NULL; static uint32 last_flags = 0; + static uint64 last_request = 0; + static uint64 last_ticket = 0; + static uint64 last_image = 0; + static uint64 last_source_generation = 0; + static BufferTag last_tag; + static bool last_tag_valid = false; static int last_result = 0; + static ClusterPcmOwnFinishRefusalReason last_finish_refusal + = CLUSTER_PCM_OWN_FINISH_REFUSAL_NONE; + static uint32 last_shared_refcount = 0; + static uint32 last_live_flags = 0; + static uint64 last_live_token = 0; + static bool last_bm_io_in_progress = false; static bool logged = false; uint32 flags = snap != NULL ? snap->flags : 0; + ClusterPcmOwnFinishRefusalReason finish_reason + = finish_refusal != NULL ? finish_refusal->reason : CLUSTER_PCM_OWN_FINISH_REFUSAL_NONE; + uint32 shared_refcount = finish_refusal != NULL ? finish_refusal->shared_refcount : 0; + uint32 live_flags = finish_refusal != NULL ? finish_refusal->live_flags : 0; + uint64 live_token = finish_refusal != NULL ? finish_refusal->live_token : 0; + bool bm_io_in_progress = finish_refusal != NULL && finish_refusal->bm_io_in_progress; + const BufferTag *tag = ref != NULL ? &ref->identity.tag : snap != NULL ? &snap->tag : NULL; + uint64 request_id = ref != NULL ? ref->identity.request_id : 0; + uint64 ticket_id = ref != NULL ? ref->handle.ticket_id : 0; + bool same_tag + = tag == NULL ? !last_tag_valid : last_tag_valid && BufferTagsEqual(tag, &last_tag); if (site == NULL) { last_site = NULL; + last_tag_valid = false; logged = false; return; } - if (site == last_site && own_result == last_result && flags == last_flags) { + if (site == last_site && own_result == last_result && flags == last_flags + && request_id == last_request && ticket_id == last_ticket && image_id == last_image + && source_generation == last_source_generation && finish_reason == last_finish_refusal + && shared_refcount == last_shared_refcount && live_flags == last_live_flags + && live_token == last_live_token && bm_io_in_progress == last_bm_io_in_progress + && same_tag) { if (!logged) { logged = true; - ereport(LOG, + if (tag != NULL) + ereport( + LOG, (errmsg("PCM-X source revoke is repeating a refusal at %s", site), - errdetail("result=%d pcm_state=%u own_flags=%u generation=%llu token=%llu", - own_result, (unsigned int)(snap != NULL ? snap->pcm_state : 0), - (unsigned int)flags, + errdetail("result=%d epoch=%llu tag=%u/%u/%u/%d/%u " + "requester=%d procno=%u xid=%u request_id=%llu wait_seq=%llu " + "ticket=%llu queue_generation=%llu grant_generation=%llu " + "image_id=%llu source_generation=%llu pcm_state=%u " + "own_generation=%llu token=%llu own_flags=%u " + "finish_refusal=%u shared_refcount=%u " + "bm_io_in_progress=%s live_flags=%u live_token=%llu", + own_result, + (unsigned long long)(ref != NULL ? ref->identity.cluster_epoch : 0), + tag->spcOid, tag->dbOid, tag->relNumber, (int)tag->forkNum, + tag->blockNum, ref != NULL ? ref->identity.node_id : -1, + ref != NULL ? ref->identity.procno : 0, + ref != NULL ? ref->identity.xid : 0, (unsigned long long)request_id, + (unsigned long long)(ref != NULL ? ref->identity.wait_seq : 0), + (unsigned long long)ticket_id, + (unsigned long long)(ref != NULL ? ref->handle.queue_generation : 0), + (unsigned long long)(ref != NULL ? ref->grant_generation : 0), + (unsigned long long)image_id, (unsigned long long)source_generation, + (unsigned int)(snap != NULL ? snap->pcm_state : 0), (unsigned long long)(snap != NULL ? snap->generation : 0), - (unsigned long long)(snap != NULL ? snap->reservation_token : 0)))); + (unsigned long long)(snap != NULL ? snap->reservation_token : 0), + (unsigned int)flags, (unsigned int)finish_reason, + (unsigned int)shared_refcount, + bm_io_in_progress ? "true" : "false", + (unsigned int)live_flags, + (unsigned long long)live_token))); } return; } last_site = site; last_result = own_result; last_flags = flags; + last_request = request_id; + last_ticket = ticket_id; + last_image = image_id; + last_source_generation = source_generation; + last_finish_refusal = finish_reason; + last_shared_refcount = shared_refcount; + last_live_flags = live_flags; + last_live_token = live_token; + last_bm_io_in_progress = bm_io_in_progress; + if (tag != NULL) { + last_tag = *tag; + last_tag_valid = true; + } else + last_tag_valid = false; logged = false; } @@ -11831,6 +12947,7 @@ gcs_block_pcm_x_revoke_refusal_note(const char *site, int own_result, static void cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const void *payload) { + PcmXRevokePayload revoke_frame; const PcmXRevokePayload *revoke; ClusterPcmOwnSnapshot own_snapshot; GcsBlockDedupKey image_key; @@ -11840,6 +12957,7 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi PcmXQueueResult progress_result; PcmXQueueResult result; uint64 current_epoch; + uint64 required_page_scn = 0; uint64 source_session; uint64 source_generation = 0; int buffer_id; @@ -11849,18 +12967,31 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi bool have_source_generation = false; bool new_reservation = false; - if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXRevokePayload) + if (env == NULL || payload == NULL + || (env->payload_length != sizeof(PcmXRevokePayload) + && env->payload_length != sizeof(PcmXRevokePayloadV2)) || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) return; source_node = (int32)env->source_node_id; - revoke = (const PcmXRevokePayload *)payload; + memset(&revoke_frame, 0, sizeof(revoke_frame)); + memcpy(&revoke_frame, payload, sizeof(revoke_frame)); + revoke = &revoke_frame; + if (env->payload_length == sizeof(PcmXRevokePayloadV2)) { + const PcmXRevokePayloadV2 *revoke_v2 = (const PcmXRevokePayloadV2 *)payload; + + if (source_node != cluster_node_id + && !cluster_sf_peer_supports_pcm_x_source_floor(source_node)) + return; + required_page_scn = revoke_v2->required_page_scn; + } current_epoch = cluster_epoch_get_current(); tag_master = cluster_gcs_lookup_master(revoke->ref.identity.tag); if (!cluster_gcs_pcm_x_revoke_ingress_valid(revoke, env->payload_length, source_node, current_epoch, tag_master, cluster_node_id) || !gcs_block_pcm_x_transfer_ingress_authorized(&revoke->ref.identity.tag, source_node, current_epoch, &source_session)) { - gcs_block_pcm_x_revoke_refusal_note("ingress-auth", 0, NULL); + gcs_block_pcm_x_revoke_refusal_note_exact("ingress-auth", 0, NULL, &revoke->ref, + revoke->image_id, 0); return; } if (!gcs_block_pcm_x_handler_tag_shard_exact(&revoke->ref.identity.tag)) @@ -11881,8 +13012,9 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi if (!gcs_block_pcm_x_ticket_ref_equal(&holder_progress.ref, &revoke->ref) || holder_progress.image.image_id != revoke->image_id) { cluster_pcm_x_stats_note_queue_result(PCM_X_QUEUE_STALE); - gcs_block_pcm_x_revoke_refusal_note("holder-ledger-stale", (int)PCM_X_QUEUE_STALE, - NULL); + gcs_block_pcm_x_revoke_refusal_note_exact("holder-ledger-stale", (int)PCM_X_QUEUE_STALE, + NULL, &revoke->ref, revoke->image_id, + holder_progress.image.source_own_generation); return; } if ((holder_progress.flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0) { @@ -11908,7 +13040,8 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi } } else if (progress_result != PCM_X_QUEUE_NOT_FOUND) { cluster_pcm_x_stats_note_queue_result(progress_result); - gcs_block_pcm_x_revoke_refusal_note("holder-progress", (int)progress_result, NULL); + gcs_block_pcm_x_revoke_refusal_note_exact("holder-progress", (int)progress_result, NULL, + &revoke->ref, revoke->image_id, 0); return; } @@ -11926,17 +13059,22 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi source_is_x = own_snapshot.pcm_state == (uint8)PCM_STATE_X; if (own_result != CLUSTER_PCM_OWN_OK || (!source_is_n && !source_is_s && !source_is_x) || own_snapshot.flags != 0) { - gcs_block_pcm_x_revoke_refusal_note("ingress-snapshot", (int)own_result, &own_snapshot); + gcs_block_pcm_x_revoke_refusal_note_exact("ingress-snapshot", (int)own_result, + &own_snapshot, &revoke->ref, revoke->image_id, + own_snapshot.generation); return; } source_generation = own_snapshot.generation; } - gcs_block_pcm_x_reserved_binding(revoke, source_generation, source_session, &reserved_binding); + gcs_block_pcm_x_reserved_binding(revoke, source_generation, source_session, required_page_scn, + &reserved_binding); image_result = cluster_gcs_block_dedup_pcm_x_reserve( worker_id, &image_key, &revoke->ref.identity.tag, &reserved_binding); if (image_result != GCS_BLOCK_PCM_X_IMAGE_RESERVED && image_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) { - gcs_block_pcm_x_revoke_refusal_note("image-reserve", (int)image_result, NULL); + gcs_block_pcm_x_revoke_refusal_note_exact("image-reserve", (int)image_result, NULL, + &revoke->ref, revoke->image_id, + source_generation); return; } new_reservation = image_result == GCS_BLOCK_PCM_X_IMAGE_RESERVED; @@ -11950,17 +13088,19 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi return; } } - result = cluster_pcm_x_local_holder_revoke_apply_exact(revoke, source_node, source_session); + result = cluster_pcm_x_local_holder_revoke_apply_floor_exact( + revoke, required_page_scn, source_node, source_session); cluster_pcm_x_stats_note_queue_result(result); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { /* Ingress only installs/re-arms holder work. It does not prove the * materializer advanced, so only a READY image resets the refusal streak. */ gcs_block_pcm_x_wake_registered_holders(&revoke->ref.identity.tag); } else { - gcs_block_pcm_x_revoke_refusal_note("apply", (int)result, NULL); + gcs_block_pcm_x_revoke_refusal_note_exact("apply", (int)result, NULL, &revoke->ref, + revoke->image_id, source_generation); if (new_reservation && cluster_gcs_block_dedup_pcm_x_release_exact( - worker_id, &image_key, &revoke->ref.identity.tag, &reserved_binding) + worker_id, &image_key, &revoke->ref.identity.tag, &reserved_binding, -1) != GCS_BLOCK_PCM_X_IMAGE_RELEASED) cluster_pcm_x_runtime_fail_closed(); } @@ -11972,11 +13112,17 @@ cluster_gcs_handle_pcm_x_image_ready_envelope(const ClusterICEnvelope *env, cons { const PcmXGrantPayload *image_ready; PcmXGrantPayload prepare; + GcsLostWriteVerdict floor_verdict; PcmXQueueResult result; + SCN required_page_scn; uint64 current_epoch; uint64 source_session; int32 source_node; int32 tag_master; + bool authorized; + bool diagnostic; + bool prepare_stage_result = false; + bool wire_valid; if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXGrantPayload) || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) @@ -11985,18 +13131,73 @@ cluster_gcs_handle_pcm_x_image_ready_envelope(const ClusterICEnvelope *env, cons image_ready = (const PcmXGrantPayload *)payload; current_epoch = cluster_epoch_get_current(); tag_master = cluster_gcs_lookup_master(image_ready->ref.identity.tag); - if (!cluster_gcs_pcm_x_image_ready_ingress_valid(image_ready, env->payload_length, source_node, - current_epoch, tag_master, cluster_node_id) - || !gcs_block_pcm_x_transfer_ingress_authorized(&image_ready->ref.identity.tag, source_node, - current_epoch, &source_session)) + diagnostic = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); + wire_valid = cluster_gcs_pcm_x_image_ready_ingress_valid( + image_ready, env->payload_length, source_node, current_epoch, tag_master, cluster_node_id); + authorized + = wire_valid + && gcs_block_pcm_x_transfer_ingress_authorized( + &image_ready->ref.identity.tag, source_node, current_epoch, &source_session); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X IMAGE_READY master boundary: ingress"), + errdetail("wire_valid=%s authorized=%s source=%d local=%d tag_master=%d " + "epoch=%llu source_session=%llu request_id=%llu ticket=%llu " + "grant_generation=%llu image_source=%d image_id=%llu", + wire_valid ? "true" : "false", authorized ? "true" : "false", + source_node, cluster_node_id, tag_master, + (unsigned long long)current_epoch, + (unsigned long long)(authorized ? source_session : 0), + (unsigned long long)image_ready->ref.identity.request_id, + (unsigned long long)image_ready->ref.handle.ticket_id, + (unsigned long long)image_ready->ref.grant_generation, + image_ready->image.source_node, + (unsigned long long)image_ready->image.image_id))); + if (!wire_valid || !authorized) + return; + required_page_scn + = cluster_pcm_lock_pi_watermark_scn_query(image_ready->ref.identity.tag); + floor_verdict + = gcs_block_lost_write_verdict(required_page_scn, (SCN)image_ready->image.page_scn); + if (floor_verdict == GCS_LOST_WRITE_FAIL_STALE + || floor_verdict == GCS_LOST_WRITE_FAIL_ANOMALY) { + ereport(LOG, + (errmsg_internal("PCM-X IMAGE_READY source stale before PREPARE: " + "source=%d request=%llu ticket=%llu image=%llu " + "image_scn=%llu required_scn=%llu verdict=%d", + source_node, + (unsigned long long)image_ready->ref.identity.request_id, + (unsigned long long)image_ready->ref.handle.ticket_id, + (unsigned long long)image_ready->image.image_id, + (unsigned long long)image_ready->image.page_scn, + (unsigned long long)required_page_scn, + (int)floor_verdict))); + cluster_pcm_x_runtime_fail_closed(); return; + } result = cluster_pcm_x_master_image_ready_exact(image_ready, source_node, source_session, &prepare); cluster_pcm_x_stats_note_queue_result(result); - if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) - (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, - prepare.ref.identity.node_id, &prepare, - sizeof(prepare)); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + gcs_block_pcm_x_revoke_refusal_note_exact(NULL, 0, NULL, NULL, 0, 0); + prepare_stage_result + = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, + prepare.ref.identity.node_id, &prepare, sizeof(prepare)); + } else + gcs_block_pcm_x_revoke_refusal_note_exact( + "image-ready-master-consume", (int)result, NULL, &image_ready->ref, + image_ready->image.image_id, image_ready->image.source_own_generation); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X IMAGE_READY master boundary: consume"), + errdetail("result=%d prepare_stage_result=%s requester=%d source=%d " + "request_id=%llu ticket=%llu grant_generation=%llu image_id=%llu", + (int)result, prepare_stage_result ? "true" : "false", + image_ready->ref.identity.node_id, source_node, + (unsigned long long)image_ready->ref.identity.request_id, + (unsigned long long)image_ready->ref.handle.ticket_id, + (unsigned long long)image_ready->ref.grant_generation, + (unsigned long long)image_ready->image.image_id))); } @@ -12005,11 +13206,15 @@ cluster_gcs_handle_pcm_x_prepare_grant_envelope(const ClusterICEnvelope *env, co { const PcmXGrantPayload *prepare; PcmXLocalHandle leader; + PcmXQueueResult lookup_result; PcmXQueueResult result; uint64 current_epoch; uint64 source_session; int32 source_node; int32 tag_master; + bool authorized; + bool diagnostic; + bool wire_valid; if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXGrantPayload) || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) @@ -12018,16 +13223,55 @@ cluster_gcs_handle_pcm_x_prepare_grant_envelope(const ClusterICEnvelope *env, co prepare = (const PcmXGrantPayload *)payload; current_epoch = cluster_epoch_get_current(); tag_master = cluster_gcs_lookup_master(prepare->ref.identity.tag); - if (!cluster_gcs_pcm_x_prepare_grant_ingress_valid(prepare, env->payload_length, source_node, - current_epoch, tag_master, cluster_node_id) - || !gcs_block_pcm_x_transfer_ingress_authorized(&prepare->ref.identity.tag, source_node, - current_epoch, &source_session)) + diagnostic = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); + wire_valid = cluster_gcs_pcm_x_prepare_grant_ingress_valid( + prepare, env->payload_length, source_node, current_epoch, tag_master, cluster_node_id); + authorized + = wire_valid + && gcs_block_pcm_x_transfer_ingress_authorized( + &prepare->ref.identity.tag, source_node, current_epoch, &source_session); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X PREPARE_GRANT requester boundary: ingress"), + errdetail("wire_valid=%s authorized=%s source=%d local=%d tag_master=%d " + "epoch=%llu source_session=%llu request_id=%llu ticket=%llu " + "grant_generation=%llu image_id=%llu", + wire_valid ? "true" : "false", authorized ? "true" : "false", + source_node, cluster_node_id, tag_master, + (unsigned long long)current_epoch, + (unsigned long long)(authorized ? source_session : 0), + (unsigned long long)prepare->ref.identity.request_id, + (unsigned long long)prepare->ref.handle.ticket_id, + (unsigned long long)prepare->ref.grant_generation, + (unsigned long long)prepare->image.image_id))); + if (!wire_valid || !authorized) return; - result = cluster_pcm_x_local_lookup_exact(&prepare->ref.identity, &leader); - if (result != PCM_X_QUEUE_OK) + lookup_result = cluster_pcm_x_local_lookup_exact(&prepare->ref.identity, &leader); + if (lookup_result != PCM_X_QUEUE_OK) { + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X PREPARE_GRANT requester boundary: apply"), + errdetail("lookup_result=%d apply_result=-1 source=%d request_id=%llu " + "ticket=%llu grant_generation=%llu image_id=%llu", + (int)lookup_result, source_node, + (unsigned long long)prepare->ref.identity.request_id, + (unsigned long long)prepare->ref.handle.ticket_id, + (unsigned long long)prepare->ref.grant_generation, + (unsigned long long)prepare->image.image_id))); return; + } result = cluster_pcm_x_local_prepare_grant_exact(&leader, prepare, source_node, source_session); cluster_pcm_x_stats_note_queue_result(result); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X PREPARE_GRANT requester boundary: apply"), + errdetail("lookup_result=%d apply_result=%d source=%d request_id=%llu " + "ticket=%llu grant_generation=%llu image_id=%llu", + (int)lookup_result, (int)result, source_node, + (unsigned long long)prepare->ref.identity.request_id, + (unsigned long long)prepare->ref.handle.ticket_id, + (unsigned long long)prepare->ref.grant_generation, + (unsigned long long)prepare->image.image_id))); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) gcs_block_pcm_x_wake_requester(&prepare->ref.identity); } @@ -12125,18 +13369,27 @@ cluster_gcs_handle_pcm_x_final_ack_envelope(const ClusterICEnvelope *env, const PcmXGrdHandoffToken handoff; PcmXGrdHandoffResult handoff_result; PcmXMasterFinalAckToken token; + ClusterPcmWmProv wm_prov; + SCN watermark_scn; + bool wm_have; PcmXPhasePayload final_commit; PcmXQueueResult result; uint64 current_epoch; uint64 source_session; int32 source_node; int32 tag_master; + const char *fail_stage = "entry"; + bool authority_valid = false; if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXFinalAckPayload) || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) return; source_node = (int32)env->source_node_id; final_ack = (const PcmXFinalAckPayload *)payload; + memset(&authority, 0, sizeof(authority)); + memset(&handoff, 0, sizeof(handoff)); + memset(&token, 0, sizeof(token)); + handoff_result = PCM_X_GRD_HANDOFF_INVALID; current_epoch = cluster_epoch_get_current(); tag_master = cluster_gcs_lookup_master(final_ack->ref.identity.tag); if (!cluster_gcs_pcm_x_final_ack_ingress_valid(final_ack, env->payload_length, source_node, @@ -12148,33 +13401,74 @@ cluster_gcs_handle_pcm_x_final_ack_envelope(const ClusterICEnvelope *env, const &token); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { - gcs_block_pcm_x_master_drive_fail_closed(result); - return; + fail_stage = "prepare"; + goto final_ack_fail_closed; } - if (!cluster_pcm_lock_authority_snapshot(final_ack->ref.identity.tag, &authority) - || !cluster_gcs_pcm_x_grd_handoff_token_build(&token, &authority, &handoff)) { - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); - return; + authority_valid + = cluster_pcm_lock_authority_snapshot(final_ack->ref.identity.tag, &authority); + if (!authority_valid) { + fail_stage = "authority-snapshot"; + goto final_ack_fail_closed; + } + if (!cluster_gcs_pcm_x_grd_handoff_token_build(&token, &authority, &handoff)) { + fail_stage = "handoff-token"; + goto final_ack_fail_closed; } handoff_result = cluster_pcm_lock_queue_handoff_x_exact(&handoff); if (handoff_result != PCM_X_GRD_HANDOFF_OK && handoff_result != PCM_X_GRD_HANDOFF_DUPLICATE) { - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); - return; + fail_stage = "grd-handoff"; + goto final_ack_fail_closed; } result = cluster_pcm_x_master_final_ack_finalize_exact(&token, &final_commit); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { /* GRD already committed; retaining the engine/image evidence and * closing the runtime is the only 8.A-safe outcome. */ - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); - return; + fail_stage = "finalize"; + goto final_ack_fail_closed; } (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK, final_commit.ref.identity.node_id, &final_commit, sizeof(final_commit)); + return; + +final_ack_fail_closed: + watermark_scn + = cluster_pcm_lock_pi_watermark_scn_query(final_ack->ref.identity.tag); + wm_have + = cluster_pcm_lock_pi_watermark_prov_query(final_ack->ref.identity.tag, &wm_prov); + ereport( + LOG, + (errmsg("PCM-X FINAL_ACK fail-closed at %s", fail_stage), + errdetail( + "result=%d handoff_result=%d authority_valid=%d " + "tag=%u/%u/%u/%d/%u requester=%d procno=%u request_id=%llu " + "ticket=%llu grant=%llu image=%llu committed_generation=%llu " + "authority_state=%u x_holder=%d s_holders=0x%x " + "master_holder=%u pending_x_node=%d pending_x_value=%llu source=%u " + "image_page_scn=%llu watermark_scn=%llu " + "wm_src=%s wm_have=%d wm_table_full=%d wm_sender=%d " + "wm_request_id=%llu wm_epoch=%llu wm_old_scn=%llu wm_new_scn=%llu " + "wm_matches_current=%d", + (int)result, (int)handoff_result, authority_valid ? 1 : 0, + final_ack->ref.identity.tag.spcOid, final_ack->ref.identity.tag.dbOid, + final_ack->ref.identity.tag.relNumber, (int)final_ack->ref.identity.tag.forkNum, + final_ack->ref.identity.tag.blockNum, final_ack->ref.identity.node_id, + final_ack->ref.identity.procno, (unsigned long long)final_ack->ref.identity.request_id, + (unsigned long long)final_ack->ref.handle.ticket_id, + (unsigned long long)final_ack->ref.grant_generation, + (unsigned long long)final_ack->image_id, + (unsigned long long)final_ack->committed_own_generation, (unsigned int)authority.state, + authority.x_holder_node, authority.s_holders_bitmap, authority.master_holder.node_id, + authority.pending_x_requester_node, (unsigned long long)authority.pending_x_since_lsn, + token.image.source_node, (unsigned long long)handoff.page_scn, + (unsigned long long)watermark_scn, + cluster_pcm_wm_src_text(wm_prov.source), wm_have ? 1 : 0, + wm_prov.table_full ? 1 : 0, wm_prov.sender_node, + (unsigned long long)wm_prov.request_id, (unsigned long long)wm_prov.epoch, + (unsigned long long)wm_prov.old_scn, (unsigned long long)wm_prov.new_scn, + wm_have && wm_prov.new_scn == watermark_scn ? 1 : 0))); + cluster_pcm_x_runtime_fail_closed(); } @@ -12295,12 +13589,14 @@ gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, { GcsBlockPcmXImageBinding binding; GcsBlockDedupKey key; + GcsBlockPcmXImageResult image_result; PcmXLocalHolderProgress progress; PcmXLocalDrainCertificate certificate; PcmXQueueResult progress_result; PcmXQueueResult result; ClusterPcmOwnSelfHandoffSample handoff_sample; ClusterPcmOwnResult handoff_result; + ClusterPcmOwnResult own_result; ClusterPcmXRevokeFinishMode finish_mode; uint64 holder_image_id = 0; bool certificate_exact; @@ -12318,7 +13614,7 @@ gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, result = cluster_pcm_x_local_drain_poll_certificate_exact( poll, authenticated_master_node, authenticated_master_session, &certificate); - if (result != PCM_X_QUEUE_OK) + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) return result; /* DRAIN mutates the holder leg. Capture the image after that transition so @@ -12340,6 +13636,7 @@ gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, binding.identity.ref = progress.ref; binding.identity.image = progress.image; binding.master_session = authenticated_master_session; + binding.required_page_scn = progress.required_page_scn; holder_image = true; } else if (progress_result != PCM_X_QUEUE_OK && progress_result != PCM_X_QUEUE_NOT_FOUND) { /* Local DRAIN is already durable; losing its exact image evidence cannot @@ -12352,7 +13649,15 @@ gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, return PCM_X_QUEUE_CORRUPT; } if (!holder_image) - return PCM_X_QUEUE_OK; + return result; + image_result = cluster_gcs_block_dedup_pcm_x_drain_status_exact( + worker_id, &key, &poll->ref.identity.tag, &binding); + if (image_result == GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) + return result; + if (image_result != GCS_BLOCK_PCM_X_IMAGE_NOT_READY) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } self_source_handoff = binding.identity.image.source_node == (uint32)cluster_node_id && poll->ref.identity.node_id == cluster_node_id; if (self_source_handoff) { @@ -12390,9 +13695,9 @@ gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, return PCM_X_QUEUE_CORRUPT; } if (!certificate_exact) { - /* Preserve the immutable record: without the exact certificate the - * release cannot be proven, but a mismatched ledger is not node - * corruption. The master's next poll replays as DUPLICATE. */ + /* Local DRAIN is already durable, and a duplicate poll cannot fabricate + * this certificate. Preserve the immutable record and stop the runtime + * for formation recovery instead of retrying into a false ACK. */ ereport(LOG, (errmsg("PCM-X self-source DRAIN certificate is not exact"), errdetail("valid=%d cert_committed=%llu cert_image=%llu cert_source_gen=%llu " @@ -12409,34 +13714,43 @@ gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, (unsigned int)handoff_sample.own_flags, (unsigned int)handoff_sample.pcm_state, handoff_sample.buffer_found ? 1 : 0))); - return PCM_X_QUEUE_NOT_READY; + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; } } - if (cluster_gcs_block_dedup_pcm_x_release_exact(worker_id, &key, &poll->ref.identity.tag, - &binding) - != GCS_BLOCK_PCM_X_IMAGE_RELEASED) { + if (!self_source_handoff) { + finish_mode = cluster_pcm_x_revoke_finish_mode(&poll->ref.identity.tag, 0); + if (finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_RETAIN) { + /* The A-record remains the retry authority until the conditional + * descriptor release succeeds. A busy BufferContent lock is normal + * DATA-worker contention, not corruption and not a reason to erase + * immutable evidence. */ + own_result = cluster_bufmgr_pcm_own_release_retained_image( + &poll->ref.identity.tag, binding.identity.image.source_own_generation); + gcs_block_pcm_x_note_own_result(own_result); + if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY) + return PCM_X_QUEUE_NOT_READY; + if (own_result != CLUSTER_PCM_OWN_OK) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + } else if (finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_DROP) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + } + image_result = cluster_gcs_block_dedup_pcm_x_release_exact( + worker_id, &key, &poll->ref.identity.tag, &binding, authenticated_master_node); + if (image_result != GCS_BLOCK_PCM_X_IMAGE_RELEASED + && image_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) { cluster_pcm_x_runtime_fail_closed(); return PCM_X_QUEUE_CORRUPT; } if (self_source_handoff) { if (ClusterGcsBlock != NULL) pg_atomic_fetch_add_u64(&ClusterGcsBlock->pcm_x_self_handoff_drain_count, 1); - return PCM_X_QUEUE_OK; - } - finish_mode = cluster_pcm_x_revoke_finish_mode(&poll->ref.identity.tag, 0); - if (finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_DROP) - return PCM_X_QUEUE_OK; - if (finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_RETAIN) { - cluster_pcm_x_runtime_fail_closed(); - return PCM_X_QUEUE_CORRUPT; - } - if (cluster_bufmgr_pcm_own_release_retained_image(&poll->ref.identity.tag, - binding.identity.image.source_own_generation) - != CLUSTER_PCM_OWN_OK) { - cluster_pcm_x_runtime_fail_closed(); - return PCM_X_QUEUE_CORRUPT; } - return PCM_X_QUEUE_OK; + return result; } @@ -13660,8 +14974,11 @@ gcs_block_pi_discard_master_apply(BufferTag tag, SCN written_scn) void cluster_gcs_block_pi_discard_drain(void) { + bool data_plane; + if (ClusterGcsBlock == NULL || !cluster_past_image) return; + data_plane = cluster_gcs_block_family_on_data_plane(); for (;;) { BufferTag tag; @@ -13677,10 +14994,39 @@ cluster_gcs_block_pi_discard_drain(void) slot = ClusterGcsBlock->pi_note_drain_seq % CLUSTER_GCS_PI_NOTE_RING_SIZE; tag = ClusterGcsBlock->pi_note_ring[slot].tag; page_scn = ClusterGcsBlock->pi_note_ring[slot].page_scn; - ClusterGcsBlock->pi_note_drain_seq++; + if (!data_plane) + ClusterGcsBlock->pi_note_drain_seq++; SpinLockRelease(&ClusterGcsBlock->pi_note_lock); master_node = cluster_gcs_lookup_master(tag); + if (data_plane) { + GcsBlockInvalidateAckPayload note; + int worker; + + if (master_node < 0 || master_node >= 32) + break; + memset(¬e, 0, sizeof(note)); + note.request_id = 0; /* unsolicited: diverted before slot logic */ + note.epoch = cluster_epoch_get_current(); + note.tag = tag; + note.sender_node = cluster_node_id; + note.ack_status = GCS_BLOCK_INVALIDATE_ACK_STATUS_PI_DURABLE_NOTE; + GcsBlockInvalidateAckPayloadSetPageScn(¬e, page_scn); + note.checksum = gcs_block_compute_invalidate_ack_checksum(¬e); + worker = cluster_lms_shard_for_tag(&tag, cluster_lms_workers); + if (!cluster_lms_outbound_enqueue(worker, PGRAC_IC_MSG_GCS_BLOCK_INVALIDATE_ACK, + (uint32)master_node, ¬e, sizeof(note))) + break; + + /* The staged ring owns a byte copy now. Advancing afterwards gives + * at-least-once delivery across a crash between these two operations; + * a duplicate status-3 apply is idempotent. Never take the outbound + * LWLock while holding this spinlock. */ + SpinLockAcquire(&ClusterGcsBlock->pi_note_lock); + ClusterGcsBlock->pi_note_drain_seq++; + SpinLockRelease(&ClusterGcsBlock->pi_note_lock); + continue; + } if (master_node == cluster_node_id) { gcs_block_pi_discard_master_apply(tag, page_scn); } else if (master_node >= 0 && master_node < 32) { @@ -13890,6 +15236,243 @@ cluster_gcs_block_local_x_upgrade(BufferTag tag) * holder would). * ============================================================ */ +/* + * Break the mirror-N GRANT_PENDING side of the pin/INVALIDATE cycle without + * touching another backend's ownership token. A backend reserves its GCS + * slot only after it has installed the descriptor's single GRANT_PENDING + * reservation, so at most one live N->S slot can match this tag. The current + * authenticated master INVALIDATE is sufficient negative authority for that + * exact attempt: publish a local DENIED_PENDING_X and let the owning backend + * perform the established token-exact abort/rearm sequence. + * + * Direct-land attempts are deliberately excluded while their target is live; + * their lane has its own LMON abort protocol. A later INVALIDATE retry can + * wake the slot after that protocol reaches ABORTED. First-reply-wins remains + * intact because reply_received is tested and set under the backend block's + * exclusive lock. + */ +static bool +gcs_block_wake_local_pending_s_request(const GcsBlockInvalidatePayload *inv) +{ + int backend_idx; + + if (inv == NULL || gcs_block_backend_blocks == NULL || ClusterGcsBlock == NULL) + return false; + + for (backend_idx = 0; backend_idx < MaxBackends; backend_idx++) { + ClusterGcsBlockBackendBlock *blk = &gcs_block_backend_blocks[backend_idx]; + int slot_idx; + + LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); + for (slot_idx = 0; slot_idx < MAX_OUTSTANDING_BLOCK_REQUESTS_PER_BACKEND; + slot_idx++) { + ClusterGcsBlockOutstandingSlot *slot = &blk->slots[slot_idx]; + GcsBlockReplyHeader *hdr; + + if (!GcsBlockLocalPendingSDenialMatches( + slot->in_use, slot->reply_received, slot->stale, slot->transition_id, + &slot->tag, slot->request_epoch, slot->expected_master_node, + slot->direct_state, slot->direct_target_prepared, &inv->tag, inv->epoch, + inv->master_node)) + continue; + + hdr = &slot->reply_header; + memset(hdr, 0, sizeof(*hdr)); + hdr->request_id = slot->request_id; + hdr->epoch = inv->epoch; + hdr->sender_node = inv->master_node; + hdr->requester_backend_id = backend_idx + 1; + hdr->transition_id = slot->transition_id; + hdr->status = (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X; + GcsBlockReplyHeaderSetForwardingMasterNode( + hdr, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + memset(slot->reply_block_data, 0, sizeof(slot->reply_block_data)); + hdr->checksum = gcs_block_compute_checksum(slot->reply_block_data); + slot->reply_sf_dep_valid = false; + slot->reply_sf_flags = 0; + cluster_sf_dep_vec_reset(&slot->reply_sf_dep_vec); + slot->reply_undo_trailer_valid = false; + slot->reply_undo_tt_generation = 0; + slot->reply_undo_authority_scn = 0; + slot->reply_received = true; + ConditionVariableSignal(&slot->reply_cv); + pg_atomic_fetch_add_u64(&ClusterGcsBlock->starvation_denied_pending_x_count, 1); + LWLockRelease(&blk->lock.lock); + return true; + } + LWLockRelease(&blk->lock.lock); + } + return false; +} + +/* Observation-only rejection bits for the closest local N->S request slot. + * They must never feed protocol control flow; the authority predicate remains + * GcsBlockLocalPendingSDenialMatches(). */ +#define GCS_BLOCK_PENDING_OBS_NO_SLOT UINT32_C(0x001) +#define GCS_BLOCK_PENDING_OBS_REPLY UINT32_C(0x002) +#define GCS_BLOCK_PENDING_OBS_STALE UINT32_C(0x004) +#define GCS_BLOCK_PENDING_OBS_TRANSITION UINT32_C(0x008) +#define GCS_BLOCK_PENDING_OBS_TAG UINT32_C(0x010) +#define GCS_BLOCK_PENDING_OBS_EPOCH UINT32_C(0x020) +#define GCS_BLOCK_PENDING_OBS_MASTER UINT32_C(0x040) +#define GCS_BLOCK_PENDING_OBS_DIRECT_STATE UINT32_C(0x080) +#define GCS_BLOCK_PENDING_OBS_DIRECT_PREPARED UINT32_C(0x100) + +typedef struct GcsBlockGrantPendingObservation +{ + bool valid; + uint64 invalidate_request_id; + uint64 invalidate_epoch; + BufferTag tag; + int32 invalidate_master; + int own_result; + int buffer_id; + uint8 own_state; + uint64 own_generation; + uint64 own_token; + uint32 own_flags; + bool woke_local; + int slot_backend; + int slot_index; + uint64 slot_request_id; + uint64 slot_epoch; + int32 slot_master; + uint8 slot_transition; + bool slot_reply_received; + bool slot_stale; + int slot_direct_state; + bool slot_direct_prepared; + uint32 reject_mask; +} GcsBlockGrantPendingObservation; + +/* LMON is single-threaded. A tiny process-local cache suppresses identical + * BUSY retries while retaining the first sighting and every identity/state + * change for the exact INVALIDATE. */ +static void +gcs_block_observe_grant_pending_invalidate(const GcsBlockInvalidatePayload *inv, + bool woke_local) +{ + static GcsBlockGrantPendingObservation cache[8]; + static uint32 next_cache_slot = 0; + GcsBlockGrantPendingObservation obs; + ClusterPcmOwnSnapshot own; + int backend_idx; + int candidate_priority = 0; + int cache_idx = -1; + int i; + + if (inv == NULL) + return; + memset(&obs, 0, sizeof(obs)); + memset(&own, 0, sizeof(own)); + obs.valid = true; + obs.invalidate_request_id = inv->request_id; + obs.invalidate_epoch = inv->epoch; + obs.tag = inv->tag; + obs.invalidate_master = inv->master_node; + obs.buffer_id = -1; + obs.slot_backend = -1; + obs.slot_index = -1; + obs.slot_master = -1; + obs.woke_local = woke_local; + obs.own_result + = (int)cluster_bufmgr_pcm_own_snapshot_by_tag(&inv->tag, &obs.buffer_id, &own); + obs.own_state = own.pcm_state; + obs.own_generation = own.generation; + obs.own_token = own.reservation_token; + obs.own_flags = own.flags; + + if (gcs_block_backend_blocks != NULL) { + for (backend_idx = 0; backend_idx < MaxBackends; backend_idx++) { + ClusterGcsBlockBackendBlock *blk = &gcs_block_backend_blocks[backend_idx]; + int slot_idx; + + LWLockAcquire(&blk->lock.lock, LW_SHARED); + for (slot_idx = 0; slot_idx < MAX_OUTSTANDING_BLOCK_REQUESTS_PER_BACKEND; + slot_idx++) { + ClusterGcsBlockOutstandingSlot *slot = &blk->slots[slot_idx]; + int priority; + + if (!slot->in_use) + continue; + priority = BufferTagsEqual(&slot->tag, &inv->tag) ? 2 : 1; + if (priority <= candidate_priority) + continue; + candidate_priority = priority; + obs.slot_backend = backend_idx; + obs.slot_index = slot_idx; + obs.slot_request_id = slot->request_id; + obs.slot_epoch = slot->request_epoch; + obs.slot_master = slot->expected_master_node; + obs.slot_transition = slot->transition_id; + obs.slot_reply_received = slot->reply_received; + obs.slot_stale = slot->stale; + obs.slot_direct_state = (int)slot->direct_state; + obs.slot_direct_prepared = slot->direct_target_prepared; + obs.reject_mask = 0; + if (slot->reply_received) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_REPLY; + if (slot->stale) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_STALE; + if (slot->transition_id != (uint8)PCM_TRANS_N_TO_S) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_TRANSITION; + if (!BufferTagsEqual(&slot->tag, &inv->tag)) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_TAG; + if (slot->request_epoch != inv->epoch) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_EPOCH; + if (slot->expected_master_node != inv->master_node) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_MASTER; + if (slot->direct_state != GCS_BLOCK_DIRECT_UNARMED + && slot->direct_state != GCS_BLOCK_DIRECT_ABORTED) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_DIRECT_STATE; + if (slot->direct_target_prepared) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_DIRECT_PREPARED; + } + LWLockRelease(&blk->lock.lock); + if (candidate_priority == 2) + break; + } + } + if (candidate_priority == 0) + obs.reject_mask = GCS_BLOCK_PENDING_OBS_NO_SLOT; + + for (i = 0; i < lengthof(cache); i++) { + if (cache[i].valid + && cache[i].invalidate_request_id == obs.invalidate_request_id + && cache[i].invalidate_epoch == obs.invalidate_epoch + && cache[i].invalidate_master == obs.invalidate_master + && BufferTagsEqual(&cache[i].tag, &obs.tag)) { + cache_idx = i; + break; + } + } + if (cache_idx >= 0 && memcmp(&cache[cache_idx], &obs, sizeof(obs)) == 0) + return; + if (cache_idx < 0) { + cache_idx = (int)(next_cache_slot % lengthof(cache)); + next_cache_slot++; + } + cache[cache_idx] = obs; + + elog(LOG, + "cluster PCM grant-pending invalidate observation: invalidate_request_id=%llu " + "epoch=%llu master=%d x_requester=%u rel=%u fork=%d blk=%u own_result=%d " + "buffer=%d own_state=%u own_generation=%llu own_token=%llu own_flags=0x%x " + "woke_local=%d slot_backend=%d slot_index=%d slot_request_id=%llu " + "slot_epoch=%llu slot_master=%d slot_transition=%u reply_received=%d stale=%d " + "direct_state=%d direct_prepared=%d reject_mask=0x%x", + (unsigned long long)obs.invalidate_request_id, + (unsigned long long)obs.invalidate_epoch, obs.invalidate_master, + (unsigned int)inv->invalidating_for_x_node, inv->tag.relNumber, + (int)inv->tag.forkNum, inv->tag.blockNum, obs.own_result, obs.buffer_id, + obs.own_state, (unsigned long long)obs.own_generation, + (unsigned long long)obs.own_token, obs.own_flags, obs.woke_local ? 1 : 0, + obs.slot_backend, obs.slot_index, (unsigned long long)obs.slot_request_id, + (unsigned long long)obs.slot_epoch, obs.slot_master, obs.slot_transition, + obs.slot_reply_received ? 1 : 0, obs.slot_stale ? 1 : 0, + obs.slot_direct_state, obs.slot_direct_prepared ? 1 : 0, obs.reject_mask); +} + /* * gcs_block_invalidate_execute — apply one INVALIDATE directive + ACK. * @@ -13901,7 +15484,7 @@ static bool gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) { GcsBlockInvalidateAckPayload ack; - ClusterPcmOwnResult own_result; + ClusterPcmOwnResult own_result = CLUSTER_PCM_OWN_INVALID; PcmXQueueResult queue_result; PcmLockMode pre_state; XLogRecPtr page_lsn = InvalidXLogRecPtr; @@ -13910,6 +15493,7 @@ gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) uint64 master_session = 0; uint8 ack_status = 0; /* OK */ bool kept_pi = false; /* spec-6.12h D-h2 — drop converted to a PI */ + bool woke_local = false; uint64 current_epoch = cluster_epoch_get_current(); if (inv->epoch != current_epoch) { @@ -13936,12 +15520,17 @@ gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) * LockBuffer finalize then sets pcm_state=X; acking already_invalidated * here would let the master clear this node's holder bit and re-grant X * elsewhere, stranding the just-finalized stale X (double X holder, - * Rule 8.A). Return false so the caller PARKS the directive (the round-5 - * A2 park lot); the LMS-loop retry re-runs once the grant finalized - * (PENDING cleared), and then sees the real X/S and invalidates it. + * Rule 8.A). A BUSY-capable master gets a negative ACK and retries; + * legacy peers use the round-5 park lot. Either route re-runs only after + * the in-flight grant has finalized or its owner has aborted PENDING. */ if (cluster_bufmgr_block_grant_pending(inv->tag)) { cluster_pcm_note_invalidate_parked_grant_pending(); + /* The master-side denial may be queued behind this same-tag BUSY + * loop. Deliver its exact local equivalent now so the reservation + * owner clears GRANT_PENDING before the next INVALIDATE retry. */ + woke_local = gcs_block_wake_local_pending_s_request(inv); + gcs_block_observe_grant_pending_invalidate(inv, woke_local); /* * Ruling ② — a BUSY-capable master gets the negative ACK RIGHT * NOW instead of a silent park: it aborts the round (clears @@ -13951,7 +15540,8 @@ gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) * very pending_x). Nothing local changed; terminal for THIS * directive. An old master falls back to the round-5 park. */ - if (cluster_sf_peer_supports_gcs_inval_busy(inv->master_node)) { + if (inv->master_node == cluster_node_id + || cluster_sf_peer_supports_gcs_inval_busy(inv->master_node)) { pg_atomic_fetch_add_u64(&ClusterGcsBlock->invalidate_busy_sent_count, 1); ack_status = (uint8)GCS_BLOCK_INVALIDATE_ACK_STATUS_RETRYABLE_BUSY; goto send_ack; @@ -14043,6 +15633,22 @@ gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) } /* FALLTHROUGH */ case CLUSTER_BUFMGR_GCS_DROP_STALE: + if (pre_state == PCM_LOCK_MODE_S) { + static bool pcm_x_queue_invalidate_refusal_logged = false; + + if (!pcm_x_queue_invalidate_refusal_logged) { + pcm_x_queue_invalidate_refusal_logged = true; + ereport(LOG, + (errmsg_internal("PCM-X queue INVALIDATE passive-S release refused"), + errdetail_internal("epoch=%llu tag=%u/%u/%u/%d/%u requester=%u " + "master=%d request_id=%llu queue_result=%d own_result=%d", + (unsigned long long)inv->epoch, inv->tag.spcOid, + inv->tag.dbOid, inv->tag.relNumber, (int)inv->tag.forkNum, + inv->tag.blockNum, (unsigned int)inv->invalidating_for_x_node, + inv->master_node, (unsigned long long)inv->request_id, + (int)queue_result, (int)own_result))); + } + } /* GCS serve-stall round-5 (A2): nothing changed, no ACK — the * caller parks the directive and the LMS loop retries it. STALE is * unreachable here (the invalidate wrapper passes no expected_lsn, so @@ -14054,7 +15660,8 @@ gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) * is often itself waiting behind the master's pending_x, so parking * only burns the master's timeout). Nothing local changed. */ - if (cluster_sf_peer_supports_gcs_inval_busy(inv->master_node)) { + if (inv->master_node == cluster_node_id + || cluster_sf_peer_supports_gcs_inval_busy(inv->master_node)) { pg_atomic_fetch_add_u64(&ClusterGcsBlock->invalidate_busy_sent_count, 1); ack_status = (uint8)GCS_BLOCK_INVALIDATE_ACK_STATUS_RETRYABLE_BUSY; goto send_ack; @@ -14209,12 +15816,11 @@ cluster_gcs_block_invalidate_park_tick(void) * window point instead. Same force-behavior inject pattern as * cluster-gcs-block-duplicate-grant-reply / -stale-ship. * - * With GRANT_PENDING staged the handler parks (returns false, bumps - * pcm.invalidate_parked_grant_pending_count) BEFORE any wire send, so the - * synthetic request_id/master_node never reach the ACK path. Without the - * park fix it would have acked already_invalidated (the W3 defect) — the - * ACK then goes to a stale request_id slot and is rejected (HC100), so - * even the defect arm cannot corrupt master state from this shim. + * With GRANT_PENDING staged the handler refuses a positive ACK and bumps + * pcm.invalidate_parked_grant_pending_count. Modern/self masters receive + * RETRYABLE_BUSY; legacy peers park. The synthetic request id cannot match + * a real invalidate slot, so even this deterministic test arm cannot mutate + * master holder state. * * Caller (bufmgr LockBuffer) holds the buffer's content lock; the handler's * park path takes only the mapping partition (SHARED) + header spinlock — @@ -14438,9 +16044,11 @@ cluster_gcs_handle_block_invalidate_ack_envelope(const ClusterICEnvelope *env, c * Off-epoch notes are dropped (fail-safe: the PI merely lingers). */ if (ack->ack_status == GCS_BLOCK_INVALIDATE_ACK_STATUS_PI_DURABLE_NOTE) { - if (ack->epoch == cluster_epoch_get_current()) + if (ack->epoch == cluster_epoch_get_current()) { + pg_atomic_fetch_add_u64(&ClusterGcsBlock->pi_durable_note_apply_count, 1); gcs_block_pi_discard_master_apply(ack->tag, GcsBlockInvalidateAckPayloadGetPageScn(ack)); + } return; } if (ack->ack_status == GCS_BLOCK_INVALIDATE_ACK_STATUS_PI_KEPT_NOTE) { @@ -14456,6 +16064,20 @@ cluster_gcs_handle_block_invalidate_ack_envelope(const ClusterICEnvelope *env, c cluster_pcm_x_stats_note_queue_result(queue_result); if (queue_result == PCM_X_QUEUE_OK) queue_positive = true; + else if (queue_result == PCM_X_QUEUE_BUSY) { + uint64 now_ms = (uint64)(GetCurrentTimestamp() / 1000); + uint64 retry_delay_ms + = gcs_block_pcm_x_invalidate_busy_retry_delay_ms(&queue_snapshot); + + queue_result = cluster_pcm_x_master_invalidate_busy_backoff_exact( + &queue_snapshot, ack->sender_node, now_ms, retry_delay_ms, &queue_updated); + cluster_pcm_x_stats_note_queue_result(queue_result); + if (queue_result == PCM_X_QUEUE_OK || queue_result == PCM_X_QUEUE_DUPLICATE) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->invalidate_busy_received_count, 1); + else + gcs_block_pcm_x_master_drive_fail_closed(queue_result); + return; + } else if (queue_result == PCM_X_QUEUE_BAD_STATE || queue_result == PCM_X_QUEUE_CORRUPT || queue_result == PCM_X_QUEUE_COUNTER_EXHAUSTED || queue_result == PCM_X_QUEUE_INVALID) @@ -15199,6 +16821,12 @@ cluster_gcs_get_pi_watermark_retire_count(void) return ClusterGcsBlock ? pg_atomic_read_u64(&ClusterGcsBlock->pi_watermark_retire_count) : 0; } +uint64 +cluster_gcs_get_pi_durable_note_apply_count(void) +{ + return ClusterGcsBlock ? pg_atomic_read_u64(&ClusterGcsBlock->pi_durable_note_apply_count) : 0; +} + uint64 cluster_gcs_get_lost_write_detected_count(void) { diff --git a/src/backend/cluster/cluster_gcs_block_dedup.c b/src/backend/cluster/cluster_gcs_block_dedup.c index 8bcfa89b10..513c078d27 100644 --- a/src/backend/cluster/cluster_gcs_block_dedup.c +++ b/src/backend/cluster/cluster_gcs_block_dedup.c @@ -397,6 +397,31 @@ dedup_pcm_x_note_failclosed(ClusterGcsBlockDedupShard *shard) pg_atomic_fetch_add_u64(&shard->pcm_x_failclosed_count, 1); } +/* The generic smart-fusion metadata cell is eight bytes and PCM-X entry + * kinds never use it. Keep the exact revoke token there without changing + * the fixed shared-memory entry layout. */ +static uint64 +dedup_pcm_x_reservation_token_get(const GcsBlockDedupEntry *entry) +{ + uint64 token; + + memcpy(&token, &entry->has_sf_dep, sizeof(token)); + return token; +} + +static void +dedup_pcm_x_reservation_token_set(GcsBlockDedupEntry *entry, uint64 token) +{ + memcpy(&entry->has_sf_dep, &token, sizeof(token)); +} + +static bool +dedup_pcm_x_source_state_valid(uint8 pcm_state) +{ + return pcm_state == (uint8)PCM_STATE_N || pcm_state == (uint8)PCM_STATE_S + || pcm_state == (uint8)PCM_STATE_X; +} + static uint32 dedup_pcm_x_block_checksum(const char *block_data) { @@ -452,6 +477,25 @@ dedup_pcm_x_binding_valid(const GcsBlockDedupKey *key, const BufferTag *tag, return true; } +/* Generic entries retain the established signed TTL meaning of + * pinned_lifetime_us. PCM-X entries are excluded from generic GC, so that + * same fixed 8-byte cell carries their exact required source SCN without + * changing the 8472-byte entry ABI. */ +static uint64 +dedup_pcm_x_required_page_scn_get(const GcsBlockDedupEntry *entry) +{ + return entry != NULL ? (uint64)entry->pinned_lifetime_us : 0; +} + +static void +dedup_pcm_x_binding_from_entry(const GcsBlockDedupEntry *entry, + GcsBlockPcmXImageBinding *binding) +{ + binding->identity = entry->payload_meta.pcm_x_identity; + binding->master_session = entry->pcm_x_master_session; + binding->required_page_scn = dedup_pcm_x_required_page_scn_get(entry); +} + static bool dedup_pcm_x_reservation_equal(const GcsBlockDedupEntry *entry, const GcsBlockPcmXImageBinding *binding) @@ -460,6 +504,7 @@ dedup_pcm_x_reservation_equal(const GcsBlockDedupEntry *entry, const GcsBlockPcmXImageIdentity *incoming = &binding->identity; return entry->pcm_x_master_session == binding->master_session + && dedup_pcm_x_required_page_scn_get(entry) == binding->required_page_scn && memcmp(&stored->ref, &incoming->ref, sizeof(stored->ref)) == 0 && stored->image.image_id == incoming->image.image_id && stored->image.source_own_generation == incoming->image.source_own_generation @@ -504,8 +549,7 @@ dedup_pcm_x_entry_payload_valid(const GcsBlockDedupKey *key, const BufferTag *ta { GcsBlockPcmXImageBinding binding; - binding.identity = entry->payload_meta.pcm_x_identity; - binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &binding); return entry->transition_id == (uint8)PCM_TRANS_N_TO_S && entry->status == (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER && dedup_pcm_x_ready_payload_valid(key, tag, &binding, &entry->reply_header, @@ -522,6 +566,17 @@ dedup_pcm_x_entry_ready_valid(const GcsBlockDedupKey *key, const BufferTag *tag, } +static bool +dedup_pcm_x_entry_drained_valid(const GcsBlockDedupKey *key, const BufferTag *tag, + const GcsBlockDedupEntry *entry) +{ + return entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED && entry->request_flags > 0 + && entry->request_flags <= (uint8)PCM_X_PROTOCOL_NODE_LIMIT + && dedup_pcm_x_reservation_token_get(entry) == 0 + && dedup_pcm_x_entry_payload_valid(key, tag, entry); +} + + /* ============================================================ * Public API. * ============================================================ */ @@ -918,13 +973,21 @@ cluster_gcs_block_dedup_pcm_x_reserve(int worker_id, const GcsBlockDedupKey *key LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); if (found) { + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED + && !dedup_pcm_x_entry_drained_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } if ((entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED || entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE - || entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) + || entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED + || entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) && entry->transition_id == (uint8)PCM_TRANS_N_TO_S && memcmp(&entry->tag, tag, sizeof(*tag)) == 0 && dedup_pcm_x_reservation_equal(entry, reserved_binding)) { - dedup_pcm_x_work_pending[worker_id] = true; + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) + dedup_pcm_x_work_pending[worker_id] = true; LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_DUPLICATE; } @@ -950,6 +1013,7 @@ cluster_gcs_block_dedup_pcm_x_reserve(int worker_id, const GcsBlockDedupKey *key entry->transition_id = (uint8)PCM_TRANS_N_TO_S; entry->entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED; entry->pcm_x_master_session = reserved_binding->master_session; + entry->pinned_lifetime_us = (int64)reserved_binding->required_page_scn; entry->payload_meta.pcm_x_identity = reserved_binding->identity; entry->registered_at_ts = GetCurrentTimestamp(); pg_atomic_fetch_add_u32(&shard->entry_count, 1); @@ -962,6 +1026,7 @@ GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_materialize(int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, const GcsBlockPcmXImageBinding *ready_binding, + uint64 reservation_token, uint8 source_pcm_state, const GcsBlockReplyHeader *reply_header, const char *block_data) { @@ -973,7 +1038,8 @@ cluster_gcs_block_dedup_pcm_x_materialize(int worker_id, const GcsBlockDedupKey shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); if (shard == NULL) return GCS_BLOCK_PCM_X_IMAGE_FULL; - if (!dedup_pcm_x_ready_payload_valid(key, tag, ready_binding, reply_header, block_data)) { + if (reservation_token == 0 || !dedup_pcm_x_source_state_valid(source_pcm_state) + || !dedup_pcm_x_ready_payload_valid(key, tag, ready_binding, reply_header, block_data)) { dedup_pcm_x_note_failclosed(shard); return GCS_BLOCK_PCM_X_IMAGE_INVALID; } @@ -989,12 +1055,14 @@ cluster_gcs_block_dedup_pcm_x_materialize(int worker_id, const GcsBlockDedupKey || entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) { GcsBlockPcmXImageBinding stored_binding; - stored_binding.identity = entry->payload_meta.pcm_x_identity; - stored_binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &stored_binding); if (entry->transition_id == (uint8)PCM_TRANS_N_TO_S && memcmp(&entry->tag, tag, sizeof(*tag)) == 0 && GcsBlockPcmXImageBindingEqual(&stored_binding, ready_binding) && dedup_pcm_x_entry_payload_valid(key, tag, entry) + && (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + || (dedup_pcm_x_reservation_token_get(entry) == reservation_token + && entry->request_flags == source_pcm_state)) && memcmp(&entry->reply_header, reply_header, sizeof(*reply_header)) == 0 && memcmp(entry->block_data, block_data, GCS_BLOCK_DATA_SIZE) == 0) { LWLockRelease(&shard->lock.lock); @@ -1019,17 +1087,20 @@ cluster_gcs_block_dedup_pcm_x_materialize(int worker_id, const GcsBlockDedupKey entry->reply_header = *reply_header; entry->status = (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER; + entry->request_flags = source_pcm_state; + dedup_pcm_x_reservation_token_set(entry, reservation_token); entry->payload_meta.pcm_x_identity = ready_binding->identity; memcpy(entry->block_data, block_data, GCS_BLOCK_DATA_SIZE); entry->completed_at_ts = GetCurrentTimestamp(); - stored_binding.identity = entry->payload_meta.pcm_x_identity; - stored_binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &stored_binding); if (!dedup_pcm_x_ready_payload_valid(key, tag, &stored_binding, &entry->reply_header, entry->block_data)) { memset(&entry->reply_header, 0, sizeof(entry->reply_header)); memset(entry->block_data, 0, GCS_BLOCK_DATA_SIZE); entry->payload_meta.pcm_x_identity = reserved_identity; entry->status = 0; + entry->request_flags = 0; + dedup_pcm_x_reservation_token_set(entry, 0); entry->completed_at_ts = 0; dedup_pcm_x_note_failclosed(shard); LWLockRelease(&shard->lock.lock); @@ -1037,6 +1108,7 @@ cluster_gcs_block_dedup_pcm_x_materialize(int worker_id, const GcsBlockDedupKey } } entry->entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED; + dedup_pcm_x_work_pending[worker_id] = true; pg_atomic_fetch_add_u64(&shard->pcm_x_stage_count, 1); LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_STORED; @@ -1069,8 +1141,7 @@ cluster_gcs_block_dedup_pcm_x_publish_ready_exact(int worker_id, const GcsBlockD LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; } - stored_binding.identity = entry->payload_meta.pcm_x_identity; - stored_binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &stored_binding); if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 || !GcsBlockPcmXImageBindingEqual(&stored_binding, ready_binding) @@ -1090,6 +1161,8 @@ cluster_gcs_block_dedup_pcm_x_publish_ready_exact(int worker_id, const GcsBlockD } entry->entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE; + entry->request_flags = 0; + dedup_pcm_x_reservation_token_set(entry, 0); dedup_pcm_x_work_pending[worker_id] = true; LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_STORED; @@ -1128,6 +1201,19 @@ cluster_gcs_block_dedup_pcm_x_lookup(int worker_id, const GcsBlockDedupKey *key, LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) { + dedup_pcm_x_binding_from_entry(entry, &stored_binding); + if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 + || !GcsBlockPcmXImageBindingEqual(&stored_binding, expected_binding) + || !dedup_pcm_x_entry_drained_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; + } if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 || (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED @@ -1137,8 +1223,7 @@ cluster_gcs_block_dedup_pcm_x_lookup(int worker_id, const GcsBlockDedupKey *key, LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_STALE; } - stored_binding.identity = entry->payload_meta.pcm_x_identity; - stored_binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &stored_binding); if (!GcsBlockPcmXImageBindingEqual(&stored_binding, expected_binding)) { dedup_pcm_x_note_failclosed(shard); LWLockRelease(&shard->lock.lock); @@ -1171,10 +1256,59 @@ cluster_gcs_block_dedup_pcm_x_lookup(int worker_id, const GcsBlockDedupKey *key, return GCS_BLOCK_PCM_X_IMAGE_REPLAY; } +GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_drain_status_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *binding) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + GcsBlockPcmXImageBinding stored_binding; + bool found = false; + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PCM_X_IMAGE_FULL; + if (!dedup_pcm_x_binding_valid(key, tag, binding, false)) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + + LWLockAcquire(&shard->lock.lock, LW_SHARED); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (!found) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; + } + dedup_pcm_x_binding_from_entry(entry, &stored_binding); + if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 + || !GcsBlockPcmXImageBindingEqual(&stored_binding, binding) + || (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) + || (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + ? !dedup_pcm_x_entry_ready_valid(key, tag, entry) + : !dedup_pcm_x_entry_drained_valid(key, tag, entry))) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) { + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_DUPLICATE; + } + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_READY; +} + + GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_release_exact(int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, - const GcsBlockPcmXImageBinding *binding) + const GcsBlockPcmXImageBinding *binding, + int32 drained_master_node) { ClusterGcsBlockDedupShard *shard; HTAB *htab = NULL; @@ -1193,6 +1327,10 @@ cluster_gcs_block_dedup_pcm_x_release_exact(int worker_id, const GcsBlockDedupKe dedup_pcm_x_note_failclosed(shard); return GCS_BLOCK_PCM_X_IMAGE_INVALID; } + if (drained_master_node < -1 || drained_master_node >= PCM_X_PROTOCOL_NODE_LIMIT) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); @@ -1201,21 +1339,62 @@ cluster_gcs_block_dedup_pcm_x_release_exact(int worker_id, const GcsBlockDedupKe LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; } - stored_binding.identity = entry->payload_meta.pcm_x_identity; - stored_binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &stored_binding); if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 || (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE - && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) || !GcsBlockPcmXImageBindingEqual(&stored_binding, binding)) { dedup_pcm_x_note_failclosed(shard); LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_STALE; } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) { + if (!dedup_pcm_x_entry_drained_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + if (drained_master_node < 0 || entry->request_flags != (uint8)(drained_master_node + 1)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_DUPLICATE; + } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE) { + if (drained_master_node < 0) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + if (!dedup_pcm_x_entry_ready_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + entry->entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED; + entry->request_flags = (uint8)(drained_master_node + 1); + entry->done_at_ts = 0; + pg_atomic_fetch_add_u64(&shard->pcm_x_release_count, 1); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_RELEASED; + } + if (drained_master_node != -1) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } (void)hash_search(htab, key, HASH_REMOVE, &found); - Assert(found); + if (!found) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } pg_atomic_fetch_sub_u32(&shard->entry_count, 1); pg_atomic_fetch_add_u64(&shard->pcm_x_release_count, 1); LWLockRelease(&shard->lock.lock); @@ -1223,13 +1402,111 @@ cluster_gcs_block_dedup_pcm_x_release_exact(int worker_id, const GcsBlockDedupKe } +bool +cluster_gcs_block_dedup_pcm_x_retire_up_to(uint64 cluster_epoch, int32 authenticated_master_node, + uint64 authenticated_master_session, + uint64 retire_through_ticket_id) +{ + int s; + + if (authenticated_master_node < 0 || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_master_session == 0 || retire_through_ticket_id == 0 + || cluster_gcs_block_dedup_shards == NULL) + return false; + + for (s = 0; s < cluster_gcs_block_dedup_n_shards; s++) { + ClusterGcsBlockDedupShard *shard = &cluster_gcs_block_dedup_shards[s]; + HTAB *htab = cluster_gcs_block_dedup_htabs[s]; + HASH_SEQ_STATUS scan; + GcsBlockDedupEntry *entry; + int removed = 0; + + if (htab == NULL) + continue; + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + hash_seq_init(&scan, htab); + while ((entry = (GcsBlockDedupEntry *)hash_seq_search(&scan)) != NULL) { + const PcmXTicketRef *ref; + + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) + continue; + if (!dedup_pcm_x_entry_drained_valid(&entry->key, &entry->tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + hash_seq_term(&scan); + LWLockRelease(&shard->lock.lock); + return false; + } + ref = &entry->payload_meta.pcm_x_identity.ref; + if (ref->identity.cluster_epoch == cluster_epoch + && entry->request_flags == (uint8)(authenticated_master_node + 1) + && entry->pcm_x_master_session == authenticated_master_session + && ref->handle.ticket_id <= retire_through_ticket_id) { + (void)hash_search(htab, &entry->key, HASH_REMOVE, NULL); + removed++; + } + } + if (removed > 0) + pg_atomic_fetch_sub_u32(&shard->entry_count, (uint32)removed); + LWLockRelease(&shard->lock.lock); + } + return true; +} + + +GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_preserve_finish_error_exact( + int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, + const GcsBlockPcmXImageBinding *binding, uint64 reservation_token, uint8 source_pcm_state) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + GcsBlockPcmXImageBinding stored_binding; + bool found = false; + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PCM_X_IMAGE_FULL; + if (reservation_token == 0 || !dedup_pcm_x_source_state_valid(source_pcm_state) + || !dedup_pcm_x_binding_valid(key, tag, binding, false)) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + + LWLockAcquire(&shard->lock.lock, LW_SHARED); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (!found) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; + } + dedup_pcm_x_binding_from_entry(entry, &stored_binding); + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED + || entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 + || !GcsBlockPcmXImageBindingEqual(&stored_binding, binding) + || !dedup_pcm_x_entry_payload_valid(key, tag, entry) + || dedup_pcm_x_reservation_token_get(entry) != reservation_token + || entry->request_flags != source_pcm_state) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING; +} + + static void dedup_pcm_x_copy_work(const GcsBlockDedupEntry *entry, GcsBlockPcmXImageWork *work) { memset(work, 0, sizeof(*work)); work->key = entry->key; - work->binding.identity = entry->payload_meta.pcm_x_identity; - work->binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &work->binding); + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) { + work->reservation_token = dedup_pcm_x_reservation_token_get(entry); + work->source_pcm_state = entry->request_flags; + } work->tag = entry->tag; work->entry_kind = entry->entry_kind; } @@ -1268,11 +1545,11 @@ dedup_pcm_x_consider_work(const GcsBlockDedupEntry *entry, const GcsBlockDedupKe } -/* Return at most one immutable work token per LMS tick. RESERVED wins the - * first mixed-class tick, then mixed-class selections alternate so neither - * fresh byte materialization nor READY replay can starve. Within each class - * a process-local exact-key cursor prevents one pinned holder from - * monopolizing every tick. */ +/* Return at most one immutable work token per LMS tick. Admission work + * includes fresh RESERVED entries and commit-only retries for immutable + * MATERIALIZED_UNCOMMITTED entries; the common exact-key cursor prevents a + * contended commit from monopolizing the worker. Admission wins the first + * mixed-class tick, then selections alternate with READY replay. */ GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_next_work(int worker_id, GcsBlockPcmXImageWork *work_out) { @@ -1307,24 +1584,29 @@ cluster_gcs_block_dedup_pcm_x_next_work(int worker_id, GcsBlockPcmXImageWork *wo hash_seq_init(&scan, htab); while ((entry = (GcsBlockDedupEntry *)hash_seq_search(&scan)) != NULL) { if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED - && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE) + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) continue; /* A READY entry admitted to the outbound ring sleeps until an exact * type-49 retransmit clears this marker. */ if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE && entry->done_at_ts != 0) continue; - binding.identity = entry->payload_meta.pcm_x_identity; - binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &binding); if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S || (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED && !dedup_pcm_x_binding_valid(&entry->key, &entry->tag, &binding, true)) + || (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED + && (!dedup_pcm_x_entry_payload_valid(&entry->key, &entry->tag, entry) + || dedup_pcm_x_reservation_token_get(entry) == 0 + || !dedup_pcm_x_source_state_valid(entry->request_flags))) || (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE && !dedup_pcm_x_entry_ready_valid(&entry->key, &entry->tag, entry))) { dedup_pcm_x_note_failclosed(shard); result = GCS_BLOCK_PCM_X_IMAGE_INVALID; break; } - if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED) { + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED + || entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) { dedup_pcm_x_consider_work(entry, &dedup_pcm_x_work_cursor[worker_id], dedup_pcm_x_work_cursor_valid[worker_id], &reserved_after, &have_reserved_after, &reserved_wrap, &have_reserved_wrap); @@ -1344,7 +1626,9 @@ cluster_gcs_block_dedup_pcm_x_next_work(int worker_id, GcsBlockPcmXImageWork *wo if (have_reserved && !choose_ready) { *work_out = have_reserved_after ? reserved_after : reserved_wrap; - result = GCS_BLOCK_PCM_X_IMAGE_RESERVED; + result = work_out->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED + ? GCS_BLOCK_PCM_X_IMAGE_RESERVED + : GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING; } else if (have_ready) { *work_out = have_ready_after ? ready_after : ready_wrap; result = GCS_BLOCK_PCM_X_IMAGE_REPLAY; @@ -1390,8 +1674,7 @@ cluster_gcs_block_dedup_pcm_x_mark_staged_exact(int worker_id, const GcsBlockDed LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; } - stored_binding.identity = entry->payload_meta.pcm_x_identity; - stored_binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &stored_binding); if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE || entry->transition_id != (uint8)PCM_TRANS_N_TO_S || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 @@ -1440,8 +1723,7 @@ cluster_gcs_block_dedup_pcm_x_unmark_staged_exact(int worker_id, const GcsBlockD LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; } - stored_binding.identity = entry->payload_meta.pcm_x_identity; - stored_binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &stored_binding); if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE || entry->transition_id != (uint8)PCM_TRANS_N_TO_S || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 @@ -1490,7 +1772,8 @@ cluster_gcs_block_dedup_pcm_x_rearm_exact(int worker_id, const GcsBlockDedupKey return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; } if ((entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED - && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE) + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) || entry->transition_id != (uint8)PCM_TRANS_N_TO_S || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 || !dedup_pcm_x_reservation_equal(entry, reserved_binding)) { @@ -1503,6 +1786,15 @@ cluster_gcs_block_dedup_pcm_x_rearm_exact(int worker_id, const GcsBlockDedupKey LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_NOT_READY; } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) { + if (!dedup_pcm_x_entry_drained_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_READY; + } if (!dedup_pcm_x_entry_ready_valid(key, tag, entry)) { dedup_pcm_x_note_failclosed(shard); LWLockRelease(&shard->lock.lock); diff --git a/src/backend/cluster/cluster_gcs_block_shard.c b/src/backend/cluster/cluster_gcs_block_shard.c index 75c374ba22..fc56439f53 100644 --- a/src/backend/cluster/cluster_gcs_block_shard.c +++ b/src/backend/cluster/cluster_gcs_block_shard.c @@ -142,7 +142,12 @@ cluster_gcs_block_payload_shard(uint8 msg_type, const void *payload, uint16 payl pcm_x_expected_len = sizeof(PcmXBlockerChunkPayload); break; case PGRAC_IC_MSG_PCM_X_REVOKE: - pcm_x_expected_len = sizeof(PcmXRevokePayload); + /* Source-floor V2 appends one SCN to the byte-identical V1 prefix. */ + if (payload_len != sizeof(PcmXRevokePayload) + && payload_len != sizeof(PcmXRevokePayloadV2)) + return -1; + memcpy(&pcm_x_tag, payload, sizeof(pcm_x_tag)); + tag = &pcm_x_tag; break; case PGRAC_IC_MSG_PCM_X_IMAGE_READY: case PGRAC_IC_MSG_PCM_X_PREPARE_GRANT: diff --git a/src/backend/cluster/cluster_ic.c b/src/backend/cluster/cluster_ic.c index 068b948336..86a3dc6e91 100644 --- a/src/backend/cluster/cluster_ic.c +++ b/src/backend/cluster/cluster_ic.c @@ -666,6 +666,9 @@ cluster_ic_build_hello(uint8 out_buf[PGRAC_IC_HELLO_BYTES], uint16 hello_version /* A' rebase: V2 INSTALL_READY understanding, same unconditional protocol * discipline; activation is formation-wide (see cluster_ic.h). */ capabilities |= PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1; + /* P0-20: V2 REVOKE source-floor carrier. Senders still gate the extended + * frame on the selected peer's current verified connection. */ + capabilities |= PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1; if (capabilities != 0) ic_le_write_uint32(out_buf + PGRAC_IC_HELLO_CAPABILITIES_OFFSET, capabilities); diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index 8e50dd4b79..febcc74332 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -606,6 +606,17 @@ static ClusterInjectPoint cluster_injection_points[] = { */ { .name = "cluster-pcm-writer-cached-x-stall" }, { .name = "cluster-pcm-restore-aba-window" }, + /* + * PCM-X retained-image finish FlushBuffer boundary (t/400). + * + * The finish path first proves its exact REVOKING lifecycle while holding + * caller pin + content EXCLUSIVE. An armed point makes the unchanged page + * dirty so this otherwise race-dependent FlushBuffer leg is deterministic. + * The point itself fires only inside that exact FlushBuffer call, just + * before smgrwrite; SKIP_N raises the fail-closed test ERROR there, while + * SLEEP:0 supplies a success-path observation without changing behavior. + */ + { .name = "cluster-pcm-x-retain-flush-error" }, /* * cluster-pcm-grant-finalize-window (W3): * Fires in LockBuffer AFTER a real X acquire installed the grant and @@ -1272,6 +1283,18 @@ cluster_injection_should_skip(const char *name) } +/* Read one point's armed state without dispatching it or consuming SKIP_N. */ +bool +cluster_injection_is_armed(const char *name) +{ + ClusterInjectPoint *p; + + cluster_injection_initialise(); + p = cluster_injection_lookup(name); + return p != NULL && pg_atomic_read_u32(&p->armed_type) != (uint32)CLUSTER_FAULT_NONE; +} + + /* * cluster_cr_injection_armed -- spec-3.9 D7 CR-specific armed-state peek. * diff --git a/src/backend/cluster/cluster_lms.c b/src/backend/cluster/cluster_lms.c index bebf5a5566..d9d0bdf080 100644 --- a/src/backend/cluster/cluster_lms.c +++ b/src/backend/cluster/cluster_lms.c @@ -63,6 +63,7 @@ #include "cluster/cluster_cssd.h" #include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ #include "cluster/cluster_gcs_block.h" /* PGRAC: spec-7.2 D4 plane probe + pi drain */ +#include "cluster/cluster_inject.h" #include "cluster/cluster_write_fence.h" /* PGRAC: spec-7.2 D5 fence linkage */ #include "cluster/cluster_ges.h" #include "cluster/cluster_ges_dedup.h" @@ -211,6 +212,7 @@ cluster_lms_shmem_init(void) pg_atomic_init_u64(&cluster_lms_state->worker_inline_serve_count[w], 0); pg_atomic_init_u64(&cluster_lms_state->worker_outbound_not_admitted_count[w], 0); pg_atomic_init_u64(&cluster_lms_state->worker_outbound_requeue_drop_count[w], 0); + pg_atomic_init_u64(&cluster_lms_state->worker_outbound_cap_guard_drop_count[w], 0); for (b = 0; b < CLUSTER_LMS_SERVE_HIST_BUCKETS; b++) pg_atomic_init_u64(&cluster_lms_state->worker_serve_hist[w][b], 0); } @@ -699,6 +701,47 @@ lms_sigusr1_handler(SIGNAL_ARGS) SetLatch(MyLatch); } +/* Report when this DATA worker has applied the process-local finish-Flush + * injection state. The transition log is the reload acknowledgement used by + * t/400; without it, a fixed sleep can race a worker that has not processed + * SIGHUP yet and turn a fault-injection leg into a false negative. */ +static void +lms_note_pcm_x_finish_flush_injection_reload(int worker_id) +{ + static bool was_armed = false; + bool armed; + + armed = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); + if (!armed && !was_armed) + return; + ereport(LOG, + (errmsg_internal("cluster_lms: DATA worker=%d applied PCM-X finish Flush " + "injection config: pid=%d armed=%s value=\"%s\"", + worker_id, (int)MyProcPid, armed ? "true" : "false", + cluster_injection_points != NULL ? cluster_injection_points : ""))); + was_armed = armed; +} + + +void +cluster_lms_note_pcm_x_image_ready_boundary( + uint8 msg_type, const char *boundary, int result, int runtime_state, bool fence_enforcing, + bool fence_allowed, uint32 dest_node_id, uint64 request_id, uint64 ticket_id, uint64 grant_generation, + uint64 image_id) +{ + if (boundary == NULL + || !cluster_injection_is_armed("cluster-pcm-x-retain-flush-error")) + return; + ereport(LOG, + (errmsg_internal("PCM-X IMAGE_READY transport boundary: %s", boundary), + errdetail("msg_type=%u result=%d runtime=%d fence_enforcing=%s fence_allowed=%s dest=%u " + "request_id=%llu ticket=%llu grant_generation=%llu image_id=%llu", + (unsigned)msg_type, result, runtime_state, fence_enforcing ? "true" : "false", + fence_allowed ? "true" : "false", dest_node_id, + (unsigned long long)request_id, (unsigned long long)ticket_id, + (unsigned long long)grant_generation, (unsigned long long)image_id))); +} + void LmsMain(void) { @@ -785,6 +828,7 @@ LmsMain(void) if (ConfigReloadPending) { ConfigReloadPending = false; ProcessConfigFile(PGC_SIGHUP); + lms_note_pcm_x_finish_flush_injection_reload(0); /* PGRAC: spec-7.3 D8 — re-apply cluster.lms_nice on reload. */ cluster_lms_apply_nice(); } @@ -956,6 +1000,7 @@ LmsWorkerMain(int worker_id) if (ConfigReloadPending) { ConfigReloadPending = false; ProcessConfigFile(PGC_SIGHUP); + lms_note_pcm_x_finish_flush_injection_reload(worker_id); /* PGRAC: spec-7.3 D8 — re-apply cluster.lms_nice on reload. */ cluster_lms_apply_nice(); } @@ -1123,6 +1168,14 @@ cluster_lms_obs_note_outbound_requeue_drop(int worker_id) pg_atomic_fetch_add_u64(&cluster_lms_state->worker_outbound_requeue_drop_count[worker_id], 1); } +void +cluster_lms_obs_note_outbound_cap_guard_drop(int worker_id) +{ + if (cluster_lms_state == NULL || worker_id < 0 || worker_id >= CLUSTER_LMS_MAX_WORKERS) + return; + pg_atomic_fetch_add_u64(&cluster_lms_state->worker_outbound_cap_guard_drop_count[worker_id], 1); +} + uint64 cluster_lms_obs_get_outbound_not_admitted(int worker_id) { @@ -1139,6 +1192,14 @@ cluster_lms_obs_get_outbound_requeue_drop(int worker_id) : lms_obs_read(cluster_lms_state->worker_outbound_requeue_drop_count, worker_id); } +uint64 +cluster_lms_obs_get_outbound_cap_guard_drop(int worker_id) +{ + return cluster_lms_state == NULL + ? 0 + : lms_obs_read(cluster_lms_state->worker_outbound_cap_guard_drop_count, worker_id); +} + uint64 cluster_lms_obs_get_dispatch_count(int worker_id) { diff --git a/src/backend/cluster/cluster_lms_outbound.c b/src/backend/cluster/cluster_lms_outbound.c index a46ba15071..1b1186cf9c 100644 --- a/src/backend/cluster/cluster_lms_outbound.c +++ b/src/backend/cluster/cluster_lms_outbound.c @@ -43,6 +43,7 @@ #include "cluster/cluster_ic_router.h" #include "cluster/cluster_lms.h" #include "cluster/cluster_shmem.h" +#include "cluster/cluster_sf_dep.h" #include "cluster/cluster_write_fence.h" #include "miscadmin.h" #include "storage/lwlock.h" @@ -65,9 +66,14 @@ typedef struct ClusterLmsOutboundSlot { uint8 msg_type; uint8 kind; uint16 payload_len; + uint32 required_capability; + uint32 connection_generation; uint8 payload[PGRAC_LMS_OUTBOUND_PAYLOAD_MAX]; } ClusterLmsOutboundSlot; +StaticAssertDecl(sizeof(ClusterLmsOutboundSlot) == 144, + "LMS outbound slot capability guard layout changed"); + typedef struct ClusterLmsZeroBlockReplyWire { GcsBlockReplyHeader header; char block_data[GCS_BLOCK_DATA_SIZE]; @@ -150,6 +156,31 @@ lms_outbound_pcm_x_grant_held(uint8 msg_type) return cluster_write_fence_enforcing() && !cluster_write_fence_allowed(); } + +static void +lms_outbound_pcm_x_image_ready_note(const ClusterLmsOutboundSlot *slot, const char *boundary, + int result) +{ + const PcmXGrantPayload *ready; + PcmXRuntimeSnapshot runtime; + bool fence_enforcing; + bool fence_allowed; + + if (slot == NULL || boundary == NULL + || (slot->msg_type != PGRAC_IC_MSG_PCM_X_IMAGE_READY + && slot->msg_type != PGRAC_IC_MSG_PCM_X_PREPARE_GRANT) + || slot->payload_len != sizeof(PcmXGrantPayload)) + return; + ready = (const PcmXGrantPayload *)slot->payload; + runtime = cluster_pcm_x_runtime_snapshot(); + fence_enforcing = cluster_write_fence_enforcing(); + fence_allowed = !fence_enforcing || cluster_write_fence_allowed(); + cluster_lms_note_pcm_x_image_ready_boundary( + slot->msg_type, boundary, result, (int)runtime.state, fence_enforcing, fence_allowed, + slot->dest_node_id, ready->ref.identity.request_id, ready->ref.handle.ticket_id, + ready->ref.grant_generation, ready->image.image_id); +} + void cluster_lms_outbound_shmem_register(void) { @@ -171,9 +202,10 @@ cluster_lms_outbound_request_lwlocks(void) * retry machinery). Publish-before-signal: the slot is visible * before the LMS wakeup fires. */ -bool -cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, - const void *payload, uint16 payload_len) +static bool +lms_outbound_enqueue_internal(int worker_id, uint8 msg_type, uint32 dest_node_id, + const void *payload, uint16 payload_len, + uint32 required_capability, uint32 connection_generation) { ClusterLmsOutboundState *ring; LWLock *lock; @@ -199,6 +231,8 @@ cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, slot->msg_type = msg_type; slot->kind = (uint8)CLUSTER_LMS_OUTBOUND_FRAME; slot->payload_len = payload_len; + slot->required_capability = required_capability; + slot->connection_generation = connection_generation; if (payload_len > 0) memcpy(slot->payload, payload, payload_len); ring->head = (ring->head + 1) % PGRAC_LMS_OUTBOUND_CAPACITY; @@ -209,6 +243,30 @@ cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, return true; } +bool +cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, + const void *payload, uint16 payload_len) +{ + return lms_outbound_enqueue_internal(worker_id, msg_type, dest_node_id, payload, payload_len, + 0, 0); +} + +/* Stage a wire-version-sensitive frame for one exact HELLO-authenticated + * connection. The reliable protocol leg remains outside this ring, so a + * drain-side guard failure may consume this stale copy and let the periodic + * producer reconstruct the correct wire version. */ +bool +cluster_lms_outbound_enqueue_cap_bound(int worker_id, uint8 msg_type, uint32 dest_node_id, + const void *payload, uint16 payload_len, + uint32 required_capability, + uint32 connection_generation) +{ + if (required_capability == 0 || dest_node_id >= CLUSTER_MAX_NODES) + return false; + return lms_outbound_enqueue_internal(worker_id, msg_type, dest_node_id, payload, payload_len, + required_capability, connection_generation); +} + /* * Stage a header-only GCS denial from a CONTROL-plane producer. The DATA * owner expands the ABI-mandated zero block immediately before transport @@ -245,6 +303,8 @@ cluster_lms_outbound_enqueue_zero_block_reply(int worker_id, uint32 dest_node_id slot->kind = (uint8)(direct_land ? CLUSTER_LMS_OUTBOUND_DIRECT_ZERO_BLOCK_REPLY : CLUSTER_LMS_OUTBOUND_ZERO_BLOCK_REPLY); slot->payload_len = sizeof(*header); + slot->required_capability = 0; + slot->connection_generation = 0; memcpy(slot->payload, header, sizeof(*header)); ring->head = (ring->head + 1) % PGRAC_LMS_OUTBOUND_CAPACITY; ring->count++; @@ -325,6 +385,18 @@ cluster_lms_outbound_drain_send(int worker_id) if (!got) break; scanned++; + /* Wire-version-sensitive slots are valid only for the exact + * connection generation whose HELLO advertised the required bit. + * A drift consumes this stale ring copy without transport admission; + * no protocol ACK is generated, so the armed reliable leg retries. */ + if (slot.required_capability != 0 + && !cluster_sf_peer_capability_generation_matches( + (int32)slot.dest_node_id, slot.required_capability, + slot.connection_generation)) { + lms_outbound_pcm_x_image_ready_note(&slot, "capability-guard", -1); + cluster_lms_obs_note_outbound_cap_guard_drop(worker_id); + continue; + } send_payload = slot.payload_len > 0 ? slot.payload : NULL; send_payload_len = slot.payload_len; if (slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_ZERO_BLOCK_REPLY @@ -352,6 +424,7 @@ cluster_lms_outbound_drain_send(int worker_id) /* A peer that refused a frame this batch keeps its later frames * queued BEHIND the refused one (per-peer order). */ if (slot.dest_node_id < CLUSTER_MAX_NODES && peer_blocked[slot.dest_node_id]) { + lms_outbound_pcm_x_image_ready_note(&slot, "peer-blocked", -2); Assert(n_retained < (int)lengthof(retained)); retained[n_retained++] = slot; continue; @@ -361,6 +434,7 @@ cluster_lms_outbound_drain_send(int worker_id) * transport admission. Retaining also blocks later same-peer frames, * preserving the DATA FIFO while unrelated peers keep flowing. */ if (lms_outbound_pcm_x_grant_held(slot.msg_type)) { + lms_outbound_pcm_x_image_ready_note(&slot, "grant-held", -3); if (slot.dest_node_id < CLUSTER_MAX_NODES) peer_blocked[slot.dest_node_id] = true; Assert(n_retained < (int)lengthof(retained)); @@ -402,6 +476,7 @@ cluster_lms_outbound_drain_send(int worker_id) send_payload, send_payload_len); handle_send_result: + lms_outbound_pcm_x_image_ready_note(&slot, "send-result", (int)rc); if (slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_ZERO_BLOCK_REPLY || slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_DIRECT_ZERO_BLOCK_REPLY) cluster_gcs_block_note_send_outcome(GCS_BLOCK_SEND_FAMILY_REPLY, rc); diff --git a/src/backend/cluster/cluster_pcm_lock.c b/src/backend/cluster/cluster_pcm_lock.c index 3c00a15af2..8f25bb7827 100644 --- a/src/backend/cluster/cluster_pcm_lock.c +++ b/src/backend/cluster/cluster_pcm_lock.c @@ -611,6 +611,29 @@ cluster_pcm_lock_authority_snapshot(BufferTag tag, PcmAuthoritySnapshot *out) return found && entry != NULL; } +bool +cluster_pcm_lock_authority_matches(BufferTag tag, const PcmAuthoritySnapshot *expected) +{ + struct GrdEntry *entry; + PcmAuthoritySnapshot current; + bool found; + bool matches = false; + + if (expected == NULL || ClusterPcm == NULL || cluster_pcm_htab == NULL) + return false; + + LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); + entry = (struct GrdEntry *)hash_search(cluster_pcm_htab, &tag, HASH_FIND, &found); + if (found && entry != NULL) { + LWLockAcquire(&entry->entry_lock.lock, LW_SHARED); + pcm_authority_snapshot_locked(entry, ¤t); + matches = pcm_authority_snapshot_equal(¤t, expected); + LWLockRelease(&entry->entry_lock.lock); + } + LWLockRelease(&ClusterPcm->htab_lock.lock); + return matches; +} + /* * Atomically install the queue winner as the sole X holder, but only if the * authority still exactly matches the snapshot captured when the transfer @@ -852,6 +875,8 @@ cluster_pcm_lock_clear_queue_pending_x_exact(BufferTag tag, int32 requester_node cleared = true; } LWLockRelease(&entry->entry_lock.lock); + if (cleared) + ConditionVariableBroadcast(&entry->wait_cv); } LWLockRelease(&ClusterPcm->htab_lock.lock); return cleared; @@ -862,6 +887,7 @@ cluster_pcm_lock_clear_pending_x(BufferTag tag) { struct GrdEntry *entry; bool found; + bool cleared = false; if (cluster_pcm_htab == NULL) return; @@ -870,11 +896,15 @@ cluster_pcm_lock_clear_pending_x(BufferTag tag) entry = (struct GrdEntry *)hash_search(cluster_pcm_htab, &tag, HASH_FIND, &found); if (found && entry != NULL) { LWLockAcquire(&entry->entry_lock.lock, LW_EXCLUSIVE); - if ((entry->pending_x_since_lsn & PCM_PENDING_X_QUEUE_KIND) == 0) { + if (entry->pending_x_requester_node != -1 + && (entry->pending_x_since_lsn & PCM_PENDING_X_QUEUE_KIND) == 0) { entry->pending_x_requester_node = -1; entry->pending_x_since_lsn = 0; + cleared = true; } LWLockRelease(&entry->entry_lock.lock); + if (cleared) + ConditionVariableBroadcast(&entry->wait_cv); } LWLockRelease(&ClusterPcm->htab_lock.lock); } @@ -911,6 +941,8 @@ cluster_pcm_lock_clear_pending_x_if(BufferTag tag, int32 expected_requester) cleared = true; } LWLockRelease(&entry->entry_lock.lock); + if (cleared) + ConditionVariableBroadcast(&entry->wait_cv); } LWLockRelease(&ClusterPcm->htab_lock.lock); return cleared; @@ -1072,6 +1104,8 @@ cluster_pcm_lock_clear_pending_x_for_node(int32 dead_node) LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); hash_seq_init(&scan, cluster_pcm_htab); while ((entry = (struct GrdEntry *)hash_seq_search(&scan)) != NULL) { + bool cleared_entry = false; + /* Fast SHARED read first — most entries will not match. */ if (entry->pending_x_requester_node != dead_node) continue; @@ -1084,8 +1118,11 @@ cluster_pcm_lock_clear_pending_x_for_node(int32 dead_node) entry->pending_x_requester_node = -1; entry->pending_x_since_lsn = 0; cleared++; + cleared_entry = true; } LWLockRelease(&entry->entry_lock.lock); + if (cleared_entry) + ConditionVariableBroadcast(&entry->wait_cv); } LWLockRelease(&ClusterPcm->htab_lock.lock); @@ -1410,67 +1447,77 @@ cluster_gcs_block_master_rebuild_from_redeclare(BufferTag tag, uint8 held_mode, /* - * PGRAC: spec-5.2 D11 — local-master record self as the new X holder after a - * writer-transfer (revoke). - * - * THIS node is the GCS master for the block. A remote node held it in X; we - * (the master + the writing requester) forwarded an X-transfer request, the - * holder shipped its current image AND released its own X (invalidating its - * local copy so it can never flush a stale page — Rule 8.A no-stale-flush), - * and we just installed the shipped bytes under content_lock EXCLUSIVE. Now - * make the authoritative master GRD entry reflect the new ownership: X held - * by self, no S holders, PI watermark advanced to the shipped page_lsn. + * P0-26: exact local-master X-transfer commit. * - * The old holder released BEFORE this point (its forward handler dropped its - * copy before replying X_GRANTED_FROM_HOLDER), so there is never a window with - * two X holders; there is at most a brief no-holder window, which is safe. - * Mirrors the X branch of cluster_gcs_block_master_rebuild_from_redeclare. + * The remote holder drops its X before replying, but the local master keeps + * the sampled remote-X authority until this barrier. Requiring the complete + * entry-lock snapshot prevents a late reply from overwriting a newer queue + * winner, holder session, or generation. STALE performs no mutation. */ -void -cluster_pcm_lock_master_take_x_after_transfer(BufferTag tag, XLogRecPtr page_lsn, SCN page_scn, - int32 holder_node, uint64 request_id, uint64 epoch) +PcmXTransferCommitResult +cluster_pcm_lock_master_take_x_after_transfer(BufferTag tag, const PcmAuthoritySnapshot *expected, + XLogRecPtr page_lsn, SCN page_scn, int32 holder_node, + uint32 requester_procno, uint64 request_id, + uint64 epoch) { struct GrdEntry *entry; + PcmAuthoritySnapshot current; + ClusterGrdHolderId requester; + bool found; - if (cluster_pcm_htab == NULL) - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("PCM lock manager disabled (cluster.pcm_grd_max_entries=0)"))); + if (expected == NULL || holder_node < 0 || holder_node >= 32 || request_id == 0) + return PCM_X_TRANSFER_COMMIT_BAD_STATE; + if (ClusterPcm == NULL || cluster_pcm_htab == NULL) + return PCM_X_TRANSFER_COMMIT_NOT_FOUND; - entry = pcm_get_or_create_entry(tag); - if (entry == NULL) - ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("cluster_pcm_lock_master_take_x_after_transfer: PCM GRD HTAB " - "FULL (cap=%d)", - pcm_grd_effective))); + requester.node_id = (uint32)cluster_node_id; + requester.procno = requester_procno; + requester.cluster_epoch = epoch; + requester.request_id = request_id; + + LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); + entry = (struct GrdEntry *)hash_search(cluster_pcm_htab, &tag, HASH_FIND, &found); + if (!found || entry == NULL) { + LWLockRelease(&ClusterPcm->htab_lock.lock); + return PCM_X_TRANSFER_COMMIT_NOT_FOUND; + } + + LWLockAcquire(&entry->entry_lock.lock, LW_EXCLUSIVE); + pcm_authority_snapshot_locked(entry, ¤t); + if (!pcm_authority_snapshot_equal(¤t, expected)) { + LWLockRelease(&entry->entry_lock.lock); + LWLockRelease(&ClusterPcm->htab_lock.lock); + return PCM_X_TRANSFER_COMMIT_STALE; + } + if (current.state != PCM_STATE_X || current.x_holder_node != holder_node + || current.s_holders_bitmap != 0 || current.pending_x_requester_node != -1 + || current.master_holder.node_id != (uint32)holder_node) { + LWLockRelease(&entry->entry_lock.lock); + LWLockRelease(&ClusterPcm->htab_lock.lock); + return PCM_X_TRANSFER_COMMIT_BAD_STATE; + } - pcm_entry_lock_exclusive(entry); pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_X); entry->x_holder_node = cluster_node_id; pg_atomic_write_u32(&entry->s_holders_bitmap, 0); - /* HC110: keep master_holder coherent with the X holder so subsequent - * forward/ship decisions (and spec-5.2 D11 path-B detection of - * master==holder==self) read the right holder identity. */ - pcm_master_holder_set_node(entry, cluster_node_id); - /* spec-2.41 D2 (§2.8.1) — local-page source advances BOTH watermarks - * monotonically: lsn for the redo-coverage serve-gate, scn for the - * lost-write detector. page_scn comes from the installed page's - * pd_block_scn (InvalidScn-safe). */ + entry->s_holder_refcount_local = 0; + pcm_master_holder_set_exact(entry, &requester); if ((uint64)page_lsn > entry->pi_watermark_lsn) entry->pi_watermark_lsn = (uint64)page_lsn; - /* monotone-max by local_scn (scn_time_cmp order); a raw SCN compare would be - * node_id-dominated — see cluster_scn.h + gcs_block_lost_write_verdict. */ if (SCN_VALID(page_scn)) { SCN wm_old = entry->pi_watermark_scn; - if (scn_local(page_scn) - > scn_local(entry->pi_watermark_scn)) { /* SCN_CMP_OK: scn_time_cmp via scn_local */ + if (scn_local(page_scn) > scn_local(entry->pi_watermark_scn)) { entry->pi_watermark_scn = page_scn; - /* step 1b: record the shipping holder's wire identity on advance. */ pcm_wm_prov_record(tag, wm_old, page_scn, CLUSTER_PCM_WM_SRC_TAKE_X, holder_node, request_id, epoch); } } + entry->last_transition_at = GetCurrentTimestamp(); + pg_atomic_fetch_add_u64(&entry->transition_count_local, 1); LWLockRelease(&entry->entry_lock.lock); + LWLockRelease(&ClusterPcm->htab_lock.lock); + return PCM_X_TRANSFER_COMMIT_OK; } /* @@ -2127,16 +2174,31 @@ cluster_pcm_transition_apply(struct GrdEntry *entry, PcmLockTransition trans, in pg_atomic_fetch_add_u64(&entry->transition_count_local, 1); } +/* The pending-X cookie and S-holder bitmap are both protected by entry_lock. + * This is the final admission point for every local/remote S publication: + * existing S residency may re-enter without changing authority, but a new bit + * cannot cross any live writer barrier (including the same-node case). */ +static bool +pcm_s_admission_allowed_locked(struct GrdEntry *entry, uint32 holder_bit) +{ + Assert(entry != NULL); + Assert(LWLockHeldByMeInMode(&entry->entry_lock.lock, LW_EXCLUSIVE)); + return (pg_atomic_read_u32(&entry->s_holders_bitmap) & holder_bit) != 0 + || entry->pending_x_requester_node == -1; +} + /* * Apply a GCS-requested PCM transition on the master side. * - * Unlike the public local APIs, this helper returns false on state - * incompatibility so the GCS request handler can send a DENIED reply instead - * of raising ERROR and leaking the caller's reply wait. Caller is the GCS - * request handler; sender-side code must not call this after a GRANTED reply. + * Unlike the public local APIs, the exact helper distinguishes a raced + * pending-X barrier from structural incompatibility so the block handler can + * return the retryable denial without raising ERROR or leaking the caller's + * reply wait. The bool wrapper preserves existing callers' applied/not- + * applied contract. */ -bool -cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, int holder_node_id) +PcmGcsTransitionApplyResult +cluster_pcm_lock_apply_gcs_transition_result(BufferTag tag, PcmLockTransition trans, + int holder_node_id) { struct GrdEntry *entry; PcmState cur; @@ -2145,25 +2207,29 @@ cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, in bool broadcast_needed = false; if (cluster_pcm_htab == NULL) - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; if (holder_node_id < 0 || holder_node_id >= 32) - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; if (trans < PCM_TRANS_N_TO_S || trans > PCM_TRANS_S_TO_X_CLEANOUT) - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; if (trans == PCM_TRANS_S_TO_X_CLEANOUT) - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; if (trans == PCM_TRANS_N_TO_S || trans == PCM_TRANS_N_TO_X) entry = pcm_get_or_create_entry(tag); else entry = pcm_find_entry(tag); if (entry == NULL) - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; holder_bit = pcm_holder_bit(holder_node_id); target = pcm_transition_target(trans); pcm_entry_lock_exclusive(entry); + if (trans == PCM_TRANS_N_TO_S && !pcm_s_admission_allowed_locked(entry, holder_bit)) { + LWLockRelease(&entry->entry_lock.lock); + return PCM_GCS_TRANSITION_PENDING_X; + } cur = (PcmState)pg_atomic_read_u32(&entry->master_state); /* @@ -2175,7 +2241,7 @@ cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, in if (trans == PCM_TRANS_N_TO_S && cur == PCM_STATE_S) { pg_atomic_fetch_or_u32(&entry->s_holders_bitmap, holder_bit); LWLockRelease(&entry->entry_lock.lock); - return true; + return PCM_GCS_TRANSITION_APPLIED; } /* @@ -2189,12 +2255,12 @@ cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, in || (cur == PCM_STATE_S && (pg_atomic_read_u32(&entry->s_holders_bitmap) & holder_bit) == 0))) { LWLockRelease(&entry->entry_lock.lock); - return true; + return PCM_GCS_TRANSITION_APPLIED; } if (!cluster_pcm_transition_legal(cur, target, trans)) { LWLockRelease(&entry->entry_lock.lock); - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; } switch (trans) { @@ -2205,7 +2271,7 @@ cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, in if ((pg_atomic_read_u32(&entry->s_holders_bitmap) & holder_bit) == 0 || (pg_atomic_read_u32(&entry->s_holders_bitmap) & ~holder_bit) != 0) { LWLockRelease(&entry->entry_lock.lock); - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; } break; case PCM_TRANS_X_TO_S_DOWNGRADE: @@ -2213,29 +2279,37 @@ cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, in case PCM_TRANS_X_TO_N_RELEASE: if (entry->x_holder_node != holder_node_id) { LWLockRelease(&entry->entry_lock.lock); - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; } break; case PCM_TRANS_S_TO_N_INVALIDATE: case PCM_TRANS_S_TO_N_RELEASE: if ((pg_atomic_read_u32(&entry->s_holders_bitmap) & holder_bit) == 0) { LWLockRelease(&entry->entry_lock.lock); - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; } break; case PCM_TRANS_S_TO_X_CLEANOUT: LWLockRelease(&entry->entry_lock.lock); - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; } cluster_pcm_transition_apply(entry, trans, holder_node_id); - if ((PcmState)pg_atomic_read_u32(&entry->master_state) == PCM_STATE_N) + if ((PcmState)pg_atomic_read_u32(&entry->master_state) == PCM_STATE_N + || trans == PCM_TRANS_X_TO_S_DOWNGRADE) broadcast_needed = true; LWLockRelease(&entry->entry_lock.lock); if (broadcast_needed) ConditionVariableBroadcast(&entry->wait_cv); - return true; + return PCM_GCS_TRANSITION_APPLIED; +} + +bool +cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, int holder_node_id) +{ + return cluster_pcm_lock_apply_gcs_transition_result(tag, trans, holder_node_id) + == PCM_GCS_TRANSITION_APPLIED; } @@ -2427,8 +2501,8 @@ cluster_pcm_get_trans_s_to_x_cleanout_count(void) "activate the spec-2.30 PCM state machine."))) -void -cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) +static bool +pcm_lock_acquire_local(BufferTag tag, PcmLockMode mode, PcmAuthoritySnapshot *remote_x_out) { struct GrdEntry *entry; int holder_node; @@ -2504,21 +2578,23 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) */ for (;;) { PcmState cur; + bool pending_x_blocks_new_s = false; pcm_entry_lock_exclusive(entry); cur = (PcmState)pg_atomic_read_u32(&entry->master_state); if (mode == PCM_LOCK_MODE_S) { - if (cur == PCM_STATE_N) { + pending_x_blocks_new_s = !pcm_s_admission_allowed_locked(entry, holder_bit); + if (!pending_x_blocks_new_s && cur == PCM_STATE_N) { cluster_pcm_transition_apply(entry, PCM_TRANS_N_TO_S, holder_node); entry->s_holder_refcount_local = 1; LWLockRelease(&entry->entry_lock.lock); if (cv_prepared) ConditionVariableCancelSleep(); - return; + return true; } - if (cur == PCM_STATE_S) { + if (!pending_x_blocks_new_s && cur == PCM_STATE_S) { /* Same-node S re-acquire: bump refcount; or join from other-node-S. */ if ((pg_atomic_read_u32(&entry->s_holders_bitmap) & holder_bit) != 0) entry->s_holder_refcount_local++; @@ -2529,7 +2605,7 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) LWLockRelease(&entry->entry_lock.lock); if (cv_prepared) ConditionVariableCancelSleep(); - return; + return true; } /* cur == X → fall through to wait */ } else /* mode == PCM_LOCK_MODE_X */ @@ -2541,7 +2617,7 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) LWLockRelease(&entry->entry_lock.lock); if (cv_prepared) ConditionVariableCancelSleep(); - return; + return true; } /* * PGRAC: spec-6.12a — idempotent X re-acquire. This NODE already @@ -2557,7 +2633,7 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) LWLockRelease(&entry->entry_lock.lock); if (cv_prepared) ConditionVariableCancelSleep(); - return; + return true; } holders = pg_atomic_read_u32(&entry->s_holders_bitmap); @@ -2575,7 +2651,7 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) LWLockRelease(&entry->entry_lock.lock); if (cv_prepared) ConditionVariableCancelSleep(); - return; + return true; } /* cur == S or X → fall through to wait */ } @@ -2596,7 +2672,7 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) * conflicting holder under the already-held entry_lock (no extra * htab_lock → no lock-order inversion). */ - if (cluster_gcs_block_local_cache) { + if (cluster_gcs_block_local_cache && !pending_x_blocks_new_s) { int32 confl_x = entry->x_holder_node; uint32 confl_s = pg_atomic_read_u32(&entry->s_holders_bitmap) & ~holder_bit; bool remote_live = false; @@ -2611,6 +2687,19 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) remote_live = true; if (remote_live) { + /* P0-26: the optimistic buffer-aware precheck may race a queue + * handoff. Capture the complete remote-X authority under the + * same entry lock instead of escaping through the tag-only + * legacy terminal; the caller owns the BufferDesc transfer. */ + if (cur == PCM_STATE_X && confl_x >= 0 && confl_x != holder_node + && remote_x_out != NULL) { + pcm_authority_snapshot_locked(entry, remote_x_out); + LWLockRelease(&entry->entry_lock.lock); + if (cv_prepared) + ConditionVariableCancelSleep(); + return false; + } + /* * PGRAC: spec-6.12a — LOCAL-master S->X upgrade. When the * conflict is ONLY remote S copies (the quiescent X->S @@ -2641,7 +2730,7 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) pcm_entry_lock_exclusive(entry); entry->s_holder_refcount_local = 0; LWLockRelease(&entry->entry_lock.lock); - return; + return true; } /* Invalidate did not complete — fail closed, retryable * (Rule 8.A: never write past an unconfirmed invalidate). */ @@ -2701,6 +2790,14 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) } } +void +cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) +{ + /* A tag-only caller has no BufferDesc and therefore retains the + * historical fail-closed behavior for a remote-X conflict. */ + (void)pcm_lock_acquire_local(tag, mode, NULL); +} + /* * PGRAC: spec-5.2a D2 — clean-page X-transfer arm (backend-local, one-shot). @@ -2883,11 +2980,15 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode, bool *out_ret * (durable S; caller mirrors pcm_state = S). */ if (mode == PCM_LOCK_MODE_S) { - PcmLockMode master_state = cluster_pcm_lock_query(tag); - int32 holder = cluster_pcm_master_holder_node_by_tag(tag); - - if (master_state == PCM_LOCK_MODE_X && holder >= 0 && holder != cluster_node_id) - return cluster_gcs_local_master_read_image_and_wait(buf, holder); + PcmAuthoritySnapshot authority; + bool have_authority = cluster_pcm_lock_authority_snapshot(tag, &authority); + + if (have_authority && authority.state == PCM_STATE_X + && authority.x_holder_node >= 0 && authority.x_holder_node != cluster_node_id + && authority.s_holders_bitmap == 0 + && authority.master_holder.node_id == (uint32)authority.x_holder_node) + return cluster_gcs_local_master_read_image_and_wait(buf, &authority, + out_retry_denied); } else /* mode == PCM_LOCK_MODE_X */ { /* @@ -2901,11 +3002,15 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode, bool *out_ret * The heap AM then sees the remote row lock and enters the cross-node TX * completion wait (spec-5.2 D4/D5). */ - PcmLockMode master_state = cluster_pcm_lock_query(tag); - int32 holder = cluster_pcm_master_holder_node_by_tag(tag); + PcmAuthoritySnapshot authority; + bool have_authority = cluster_pcm_lock_authority_snapshot(tag, &authority); + PcmLockMode master_state = have_authority ? (PcmLockMode)authority.state : PCM_LOCK_MODE_N; - if (master_state == PCM_LOCK_MODE_X && holder >= 0 && holder != cluster_node_id) - return cluster_gcs_local_master_x_transfer_and_wait(buf, holder, clean_eligible); + if (have_authority && authority.state == PCM_STATE_X && authority.x_holder_node >= 0 + && authority.x_holder_node != cluster_node_id && authority.s_holders_bitmap == 0 + && authority.master_holder.node_id == (uint32)authority.x_holder_node) + return cluster_gcs_local_master_x_transfer_and_wait(buf, &authority, clean_eligible, + out_retry_denied); /* * PGRAC: spec-4.6a BUG-C2 follow-through for shared_catalog DDL after @@ -2922,9 +3027,16 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode, bool *out_ret if ((cluster_pcm_lock_query_s_holders_bitmap(tag) & self_bit) == 0) { struct GrdEntry *entry; + PcmAuthoritySnapshot raced_remote_x; bool upgraded = false; - cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + /* The S bootstrap is still BufferDesc-aware: if a queue handoff + * replaces the sampled S authority with remote X before this entry + * lock, preserve that exact authority and use the normal X-transfer + * path instead of the tag-only legacy terminal. */ + if (!pcm_lock_acquire_local(tag, PCM_LOCK_MODE_S, &raced_remote_x)) + return cluster_gcs_local_master_x_transfer_and_wait( + buf, &raced_remote_x, clean_eligible, out_retry_denied); /* Amendment v1.2 (R5): the upgrade waits on remote ACKs with * CHECK_FOR_INTERRUPTS in the loop, so a cancel can THROW out * of it — release the temporary S claim on that path too, not @@ -2972,7 +3084,17 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode, bool *out_ret } } - cluster_pcm_lock_acquire(tag, mode); + { + PcmAuthoritySnapshot raced_remote_x; + + if (!pcm_lock_acquire_local(tag, mode, &raced_remote_x)) { + if (mode == PCM_LOCK_MODE_S) + return cluster_gcs_local_master_read_image_and_wait(buf, &raced_remote_x, + out_retry_denied); + return cluster_gcs_local_master_x_transfer_and_wait(buf, &raced_remote_x, + clean_eligible, out_retry_denied); + } + } /* * PGRAC: GCS-race round-4c FUNC-1 (local-master flavour) — the tag-only * grant ships no image, so the buffer keeps this backend's pre-read diff --git a/src/backend/cluster/cluster_pcm_own.c b/src/backend/cluster/cluster_pcm_own.c index 348a780299..320da62ae8 100644 --- a/src/backend/cluster/cluster_pcm_own.c +++ b/src/backend/cluster/cluster_pcm_own.c @@ -55,6 +55,7 @@ cluster_pcm_own_shmem_init(void) for (i = 0; i < NBuffers; i++) { pg_atomic_init_u64(&ClusterPcmOwnArray[i].generation, 0); pg_atomic_init_u64(&ClusterPcmOwnArray[i].reservation_token, 0); + pg_atomic_init_u64(&ClusterPcmOwnArray[i].writer_activation_token, 0); pg_atomic_init_u32(&ClusterPcmOwnArray[i].flags, 0); ClusterPcmOwnArray[i]._pad = 0; } @@ -88,6 +89,7 @@ cluster_pcm_own_reservation_begin_exact(int buf_id, uint64 expected_generation, ClusterPcmOwnResult live_result; uint64 generation; uint64 old_token; + uint64 writer_activation_token; uint32 flags; if (out_token == NULL) @@ -100,12 +102,23 @@ cluster_pcm_own_reservation_begin_exact(int buf_id, uint64 expected_generation, generation = pg_atomic_read_u64(&entry->generation); old_token = pg_atomic_read_u64(&entry->reservation_token); + writer_activation_token = pg_atomic_read_u64(&entry->writer_activation_token); flags = pg_atomic_read_u32(&entry->flags); if (generation != expected_generation) return CLUSTER_PCM_OWN_STALE; + if (writer_activation_token != 0 && writer_activation_token != old_token) + return CLUSTER_PCM_OWN_CORRUPT; live_result = cluster_pcm_own_classify_live_flags(flags, old_token); - if (live_result != CLUSTER_PCM_OWN_OK) + if (live_result != CLUSTER_PCM_OWN_OK) { + if (writer_activation_token != 0) + return CLUSTER_PCM_OWN_CORRUPT; return live_result; + } + if (writer_activation_token != 0) { + if (reservation_flag == PCM_OWN_FLAG_REVOKING) + return CLUSTER_PCM_OWN_BUSY; + return CLUSTER_PCM_OWN_CORRUPT; + } if (generation == UINT64_MAX || old_token == UINT64_MAX) return CLUSTER_PCM_OWN_EXHAUSTED; @@ -129,6 +142,8 @@ cluster_pcm_own_reservation_abort_exact(int buf_id, uint64 expected_generation, return CLUSTER_PCM_OWN_INVALID; if (!cluster_pcm_own_entry_for_buf(buf_id, &entry)) return CLUSTER_PCM_OWN_NOT_READY; + if (pg_atomic_read_u64(&entry->writer_activation_token) != 0) + return CLUSTER_PCM_OWN_CORRUPT; if (pg_atomic_read_u64(&entry->generation) != expected_generation || pg_atomic_read_u64(&entry->reservation_token) != reservation_token || pg_atomic_read_u32(&entry->flags) != reservation_flag) @@ -155,6 +170,8 @@ cluster_pcm_own_reservation_handoff_exact(int buf_id, uint64 expected_generation return CLUSTER_PCM_OWN_NOT_READY; if (pg_atomic_read_u64(&entry->generation) != expected_generation) return CLUSTER_PCM_OWN_STALE; + if (pg_atomic_read_u64(&entry->writer_activation_token) != 0) + return CLUSTER_PCM_OWN_CORRUPT; live_token = pg_atomic_read_u64(&entry->reservation_token); flags = pg_atomic_read_u32(&entry->flags); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -205,6 +222,8 @@ cluster_pcm_own_grant_commit_exact(int buf_id, uint64 expected_generation, uint6 return CLUSTER_PCM_OWN_STALE; live_token = pg_atomic_read_u64(&entry->reservation_token); flags = pg_atomic_read_u32(&entry->flags); + if (pg_atomic_read_u64(&entry->writer_activation_token) != 0) + return CLUSTER_PCM_OWN_CORRUPT; live_result = cluster_pcm_own_classify_live_flags(flags, live_token); if (live_result == CLUSTER_PCM_OWN_CORRUPT) return live_result; @@ -224,6 +243,73 @@ cluster_pcm_own_grant_commit_exact(int buf_id, uint64 expected_generation, uint6 return CLUSTER_PCM_OWN_OK; } +ClusterPcmOwnResult +cluster_pcm_own_writer_grant_commit_exact(int buf_id, uint64 expected_generation, + uint64 reservation_token, + uint64 *out_committed_generation) +{ + ClusterPcmOwnEntry *entry; + ClusterPcmOwnResult live_result; + uint64 generation; + uint64 live_token; + uint32 flags; + + if (out_committed_generation == NULL || reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + *out_committed_generation = 0; + if (!cluster_pcm_own_entry_for_buf(buf_id, &entry)) + return CLUSTER_PCM_OWN_NOT_READY; + generation = pg_atomic_read_u64(&entry->generation); + live_token = pg_atomic_read_u64(&entry->reservation_token); + flags = pg_atomic_read_u32(&entry->flags); + if (pg_atomic_read_u64(&entry->writer_activation_token) != 0) + return CLUSTER_PCM_OWN_CORRUPT; + if (generation != expected_generation) + return CLUSTER_PCM_OWN_STALE; + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + return live_result; + if (flags == 0) + return CLUSTER_PCM_OWN_STALE; + if (flags != PCM_OWN_FLAG_GRANT_PENDING) + return CLUSTER_PCM_OWN_BUSY; + if (live_token != reservation_token) + return CLUSTER_PCM_OWN_STALE; + if (generation == UINT64_MAX) + return CLUSTER_PCM_OWN_EXHAUSTED; + /* Caller holds the matching BufferDesc header lock. The active flag is + * cleared and the fence published before that lock can be released, so a + * REVOKING begin can never pass between the two stores. */ + generation++; + pg_atomic_write_u64(&entry->generation, generation); + pg_atomic_write_u32(&entry->flags, 0); + pg_atomic_write_u64(&entry->writer_activation_token, reservation_token); + *out_committed_generation = generation; + return CLUSTER_PCM_OWN_OK; +} + +ClusterPcmOwnResult +cluster_pcm_own_writer_activation_clear_exact(int buf_id, uint64 expected_generation, + uint64 reservation_token) +{ + ClusterPcmOwnEntry *entry; + uint64 live_activation; + + if (reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + if (!cluster_pcm_own_entry_for_buf(buf_id, &entry)) + return CLUSTER_PCM_OWN_NOT_READY; + if (pg_atomic_read_u64(&entry->generation) != expected_generation + || pg_atomic_read_u64(&entry->reservation_token) != reservation_token + || pg_atomic_read_u32(&entry->flags) != 0) + return CLUSTER_PCM_OWN_STALE; + live_activation = pg_atomic_read_u64(&entry->writer_activation_token); + if (live_activation == 0 || live_activation != reservation_token) + return CLUSTER_PCM_OWN_STALE; + pg_atomic_write_u64(&entry->writer_activation_token, 0); + return CLUSTER_PCM_OWN_OK; +} + ClusterPcmOwnResult cluster_pcm_own_revoke_commit_exact(int buf_id, uint64 expected_generation, uint64 reservation_token, uint64 *out_committed_generation) @@ -243,6 +329,8 @@ cluster_pcm_own_revoke_commit_exact(int buf_id, uint64 expected_generation, generation = pg_atomic_read_u64(&entry->generation); live_token = pg_atomic_read_u64(&entry->reservation_token); flags = pg_atomic_read_u32(&entry->flags); + if (pg_atomic_read_u64(&entry->writer_activation_token) != 0) + return CLUSTER_PCM_OWN_CORRUPT; if (generation != expected_generation) return CLUSTER_PCM_OWN_STALE; live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -284,6 +372,8 @@ cluster_pcm_own_revoke_retain_commit_exact(int buf_id, uint64 expected_generatio generation = pg_atomic_read_u64(&entry->generation); live_token = pg_atomic_read_u64(&entry->reservation_token); flags = pg_atomic_read_u32(&entry->flags); + if (pg_atomic_read_u64(&entry->writer_activation_token) != 0) + return CLUSTER_PCM_OWN_CORRUPT; if (generation != expected_generation) return CLUSTER_PCM_OWN_STALE; live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -321,6 +411,8 @@ cluster_pcm_own_revoke_retain_release_exact(int buf_id, uint64 committed_generat return CLUSTER_PCM_OWN_NOT_READY; if (pg_atomic_read_u64(&entry->generation) != committed_generation) return CLUSTER_PCM_OWN_STALE; + if (pg_atomic_read_u64(&entry->writer_activation_token) != 0) + return CLUSTER_PCM_OWN_CORRUPT; live_token = pg_atomic_read_u64(&entry->reservation_token); flags = pg_atomic_read_u32(&entry->flags); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -352,10 +444,14 @@ cluster_pcm_own_gen_bump_checked(int buf_id, uint64 *out_generation) flags = pg_atomic_read_u32(&entry->flags); if (out_generation != NULL) *out_generation = generation; - if (generation == UINT64_MAX || flags != 0) + if (generation == UINT64_MAX || flags != 0 + || pg_atomic_read_u64(&entry->writer_activation_token) != 0) return false; generation++; pg_atomic_write_u64(&entry->generation, generation); + /* Descriptor/tag reuse and every ordinary transition start with no + * activation lifecycle. A nonzero fence was rejected above. */ + pg_atomic_write_u64(&entry->writer_activation_token, 0); if (out_generation != NULL) *out_generation = generation; return true; diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index 0ec2289f73..e5ffc8620f 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -1328,15 +1328,24 @@ cluster_pcm_x_local_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, memcpy(©, locked, sizeof(copy)); LWLockRelease(&header->local_locks[partition].lock); snprintf(buf, buflen, - "rel=%u blk=%u flags=0x%x round=%u master=%d ref_node=%d ref_ticket=" UINT64_FORMAT + "rel=%u fork=%d blk=%u flags=0x%x round=%u master=%d ref_node=%d ref_ticket=" UINT64_FORMAT " ref_grant=" UINT64_FORMAT " holder_ticket=" UINT64_FORMAT - " blocker_ticket=" UINT64_FORMAT " drain_gen=" UINT64_FORMAT - " holder_drain_gen=" UINT64_FORMAT + " blocker_ticket=" UINT64_FORMAT " blocker_gen=" UINT64_FORMAT + " blocker_op=%u blocker_phase=%u blocker_last=%u blocker_last_node=%u" + " blocker_tomb=0x%llx blocker_count=%u blocker_head=%ld blocker_next=%u" + " blocker_crc=%u drain_gen=" UINT64_FORMAT " holder_drain_gen=" UINT64_FORMAT " members=%zu head=%ld leader=%ld active_writer=%ld", - copy.tag.relNumber, copy.tag.blockNum, pcm_x_slot_flags_read(©.slot), + copy.tag.relNumber, (int)copy.tag.forkNum, copy.tag.blockNum, pcm_x_slot_flags_read(©.slot), copy.local_round, copy.master_node, copy.ref.identity.node_id, copy.ref.handle.ticket_id, copy.ref.grant_generation, copy.holder_ref.handle.ticket_id, copy.blocker_snapshot_ref.handle.ticket_id, + copy.blocker_set_generation, copy.blocker_snapshot_reliable.pending_opcode, + copy.blocker_snapshot_reliable.phase, + copy.blocker_snapshot_reliable.last_response_opcode, + copy.blocker_snapshot_reliable.last_responder_node, + (unsigned long long)copy.blocker_snapshot_reliable.response_tombstone_mask, + copy.blocker_snapshot_count, pcm_x_debug_index(copy.blocker_snapshot_head_index), + copy.blocker_snapshot_next_chunk, copy.blocker_snapshot_crc32c, copy.terminal_drain_generation, copy.holder_terminal_drain_generation, copy.membership_count, pcm_x_debug_index(copy.head_index), pcm_x_debug_index(copy.leader_index), pcm_x_debug_index(copy.active_writer_index)); @@ -4837,6 +4846,31 @@ pcm_x_master_drive_capture_locked(PcmXRuntimeSnapshot runtime, PcmXMasterTagSlot } +/* Revalidate a snapshot-token locator after crossing allocator -> master + * lock domains. The exact ticket may have completed, retired, or been + * reclaimed in that gap. Those monotone successors consume the old drive + * token; they are not evidence that the live queue is corrupt. */ +static PcmXQueueResult +pcm_x_master_drive_expected_capture_locked( + PcmXRuntimeSnapshot runtime, PcmXMasterTagSlot *tag_slot, PcmXSlotRef tag_ref, + PcmXMasterTicketSlot *ticket, PcmXSlotRef ticket_ref, const PcmXTicketRef *expected_ref, + PcmXMasterDriveSnapshot *snapshot) +{ + uint32 state; + + if (expected_ref == NULL || snapshot == NULL) + return PCM_X_QUEUE_CORRUPT; + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, expected_ref)) + return PCM_X_QUEUE_STALE; + state = pcm_x_slot_state_read(&ticket->slot); + if (state == PCM_XT_COMPLETE || state == PCM_XT_CANCELLED + || state == PCM_XT_RETIRE_CREDIT) + return PCM_X_QUEUE_NOT_READY; + return pcm_x_master_drive_capture_locked(runtime, tag_slot, tag_ref, ticket, ticket_ref, + snapshot); +} + + static bool pcm_x_master_drive_snapshot_base_equal(const PcmXMasterDriveSnapshot *left, const PcmXMasterDriveSnapshot *right) @@ -4976,8 +5010,8 @@ cluster_pcm_x_master_commit_retry_exact(const PcmXMasterDriveSnapshot *expected, ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, &expected->ref.identity.tag, PCM_X_MASTER_TICKET_DOMAIN_STATES); - result = pcm_x_master_drive_capture_locked(runtime, tag_slot, tag_ref, ticket, ticket_ref, - ¤t); + result = pcm_x_master_drive_expected_capture_locked( + runtime, tag_slot, tag_ref, ticket, ticket_ref, &expected->ref, ¤t); if (result != PCM_X_QUEUE_OK) goto retry_done; if (!pcm_x_master_drive_snapshot_equal(¤t, expected)) { @@ -5016,6 +5050,101 @@ cluster_pcm_x_master_commit_retry_exact(const PcmXMasterDriveSnapshot *expected, } +PcmXQueueResult +cluster_pcm_x_master_invalidate_busy_backoff_exact(const PcmXMasterDriveSnapshot *expected, + int32 busy_node, uint64 now_ms, + uint64 retry_delay_ms, + PcmXMasterDriveSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXMasterDriveSnapshot current; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint64 next_retry_deadline_ms; + uint32 busy_bit; + uint32 partition; + + pcm_x_master_drive_snapshot_clear(snapshot_out); + if (header == NULL || expected == NULL || snapshot_out == NULL || busy_node < 0 + || busy_node >= PCM_X_PROTOCOL_NODE_LIMIT || retry_delay_ms == 0 || now_ms == UINT64_MAX + || expected->ticket_state != PCM_XT_ACTIVE_TRANSFER + || expected->pending_opcode != PGRAC_IC_MSG_PCM_X_REVOKE + || expected->phase != PGRAC_IC_MSG_PCM_X_REVOKE || expected->reserved != 0 + || expected->expected_responder_node < 0 + || expected->expected_responder_node >= PCM_X_PROTOCOL_NODE_LIMIT + || expected->expected_responder_session == 0 + || !pcm_x_wait_identity_valid(&expected->ref.identity)) + return PCM_X_QUEUE_INVALID; + busy_bit = UINT32_C(1) << (uint32)busy_node; + if ((expected->pending_s_holders_bitmap & busy_bit) == 0 + || (expected->acked_s_holders_bitmap & busy_bit) != 0) + return PCM_X_QUEUE_INVALID; + next_retry_deadline_ms + = now_ms > UINT64_MAX - retry_delay_ms ? UINT64_MAX : now_ms + retry_delay_ms; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(&expected->ref, &ticket_ref, &ticket); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&expected->ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TAG, tag_ref, + &expected->ref.identity.tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &expected->ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + result = pcm_x_master_drive_expected_capture_locked( + runtime, tag_slot, tag_ref, ticket, ticket_ref, &expected->ref, ¤t); + if (result != PCM_X_QUEUE_OK) + goto busy_done; + if (!pcm_x_master_drive_snapshot_equal(¤t, expected)) { + result = PCM_X_QUEUE_STALE; + goto busy_done; + } + if (!pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_REVOKE, + ticket->reliable.expected_responder_node, + ticket->reliable.expected_responder_session) + || !pcm_x_transfer_peer_exact(ticket, ticket->reliable.expected_responder_node, + ticket->reliable.expected_responder_session)) { + result = PCM_X_QUEUE_STALE; + goto busy_done; + } + if (ticket->reliable.retry_deadline_ms != 0 + && now_ms < ticket->reliable.retry_deadline_ms) { + *snapshot_out = current; + result = PCM_X_QUEUE_DUPLICATE; + goto busy_done; + } + + /* BUSY is only scheduling evidence. Do not credit the holder, clear GRD, + * change the reliable identity, or advance the retry-attempt counter. */ + ticket->reliable.retry_deadline_ms = next_retry_deadline_ms; + pg_write_barrier(); + result = pcm_x_master_drive_capture_locked(runtime, tag_slot, tag_ref, ticket, ticket_ref, + snapshot_out); + +busy_done: + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + PcmXQueueResult cluster_pcm_x_master_commit_retry_drive(const PcmXMasterDriveSnapshot *expected, uint64 now_ms, uint64 retry_delay_ms, PcmXStageFrameCallback stage_frame, @@ -5088,8 +5217,8 @@ cluster_pcm_x_master_drive_bitmap_replace_exact(const PcmXMasterDriveSnapshot *e ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, &expected->ref.identity.tag, PCM_X_MASTER_TICKET_DOMAIN_STATES); - result = pcm_x_master_drive_capture_locked(runtime, tag_slot, tag_ref, ticket, ticket_ref, - ¤t); + result = pcm_x_master_drive_expected_capture_locked( + runtime, tag_slot, tag_ref, ticket, ticket_ref, &expected->ref, ¤t); if (result != PCM_X_QUEUE_OK) goto replace_done; *snapshot_out = current; @@ -10249,6 +10378,7 @@ pcm_x_local_tag_init_common(PcmXLocalTagSlot *tag_slot, const BufferTag *tag, ui tag_slot->grant_base_own_generation = 0; memset(&tag_slot->holder_ref, 0, sizeof(tag_slot->holder_ref)); memset(&tag_slot->holder_image, 0, sizeof(tag_slot->holder_image)); + tag_slot->holder_required_page_scn = 0; memset(&tag_slot->holder_reliable, 0, sizeof(tag_slot->holder_reliable)); tag_slot->holder_terminal_drain_generation = 0; memset(&tag_slot->blocker_snapshot_ref, 0, sizeof(tag_slot->blocker_snapshot_ref)); @@ -11117,6 +11247,7 @@ pcm_x_local_holder_detach_exact(const PcmXLocalHolderHandle *handle, uint32 allo && pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref) && pcm_x_transfer_leg_is_clear(&tag_slot->reliable) && pcm_x_image_token_equal(&tag_slot->holder_image, &zero_image) + && tag_slot->holder_required_page_scn == 0 && pcm_x_transfer_leg_is_pristine(&tag_slot->holder_reliable) && tag_slot->holder_terminal_drain_generation == 0 && tag_slot->blocker_snapshot_head_index == PCM_X_INVALID_SLOT_INDEX @@ -12367,6 +12498,28 @@ pcm_x_local_blocker_ack_tombstone_exact(const PcmXLocalTagSlot *tag_slot, } +/* A same-ticket re-PROBE can stage a later empty generation after the master + * consumed an earlier ACK. Keep this exception exact: only the authenticated + * master/session COMMIT, with a fully empty canonical payload, is eligible. */ +static bool +pcm_x_local_empty_blocker_commit_exact(const PcmXLocalTagSlot *tag_slot, + int32 authenticated_master_node, + uint64 authenticated_master_session) +{ + return tag_slot != NULL && tag_slot->blocker_set_generation != 0 + && tag_slot->blocker_set_generation != UINT64_MAX + && tag_slot->blocker_snapshot_reliable.state_sequence != 0 + && tag_slot->blocker_snapshot_reliable.response_tombstone_mask == 0 + && pcm_x_transfer_leg_exact(&tag_slot->blocker_snapshot_reliable, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, + authenticated_master_node, + authenticated_master_session) + && tag_slot->blocker_snapshot_head_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->blocker_snapshot_count == 0 && tag_slot->blocker_snapshot_crc32c == 0 + && tag_slot->blocker_snapshot_next_chunk == 0; +} + + PcmXQueueResult cluster_pcm_x_local_blocker_snapshot_arm_exact( const PcmXTicketRef *ref, int32 authenticated_master_node, uint64 authenticated_master_session, @@ -12921,9 +13074,10 @@ cluster_pcm_x_local_blocker_ack_exact(const PcmXTicketRef *ref, uint64 set_gener PcmXQueueResult -cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, - int32 authenticated_master_node, - uint64 authenticated_master_session) +cluster_pcm_x_local_holder_revoke_apply_floor_exact(const PcmXRevokePayload *revoke, + uint64 required_page_scn, + int32 authenticated_master_node, + uint64 authenticated_master_session) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; PcmXRuntimeSnapshot runtime; @@ -12996,7 +13150,8 @@ cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, } if (!pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref)) { if (!pcm_x_ticket_ref_equal(&tag_slot->holder_ref, &revoke->ref) - || tag_slot->holder_image.image_id != revoke->image_id) { + || tag_slot->holder_image.image_id != revoke->image_id + || tag_slot->holder_required_page_scn != required_page_scn) { result = PCM_X_QUEUE_STALE; goto revoke_release_gate; } @@ -13014,10 +13169,9 @@ cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, goto revoke_release_gate; } /* Type 49 is the fail-closed boundary that replaces the legacy type-48 - * unconditional ACK. The same ticket locator's immutable blocker set must - * have received an exact type-48 ACK and reclaimed every blocker slot before - * holder revoke state can become visible. A premature 49 is retryable; an - * inconsistent tombstone is corruption. */ + * unconditional ACK. The same ticket locator must carry either its exact + * type-48 ACK tombstone or the narrowly authenticated empty re-PROBE replay + * below. A premature 49 is retryable; inconsistent evidence is corruption. */ if (pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref)) { result = PCM_X_QUEUE_NOT_READY; goto revoke_release_gate; @@ -13027,10 +13181,17 @@ cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, goto revoke_release_gate; } if (!pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable)) { - result = PCM_X_QUEUE_NOT_READY; - goto revoke_release_gate; - } - if (tag_slot->blocker_set_generation == 0 + /* Authenticated exact REVOKE proves the master already crossed into + * transfer after staging this generation's ACK on the same tag FIFO. The + * ACK handler can nevertheless observe a transient local admission gate and + * leave the exact COMMIT armed. Consume only that canonical empty set (for + * the first or a replay generation), since no blocker edge can be omitted. */ + if (!pcm_x_local_empty_blocker_commit_exact( + tag_slot, authenticated_master_node, authenticated_master_session)) { + result = PCM_X_QUEUE_NOT_READY; + goto revoke_release_gate; + } + } else if (tag_slot->blocker_set_generation == 0 || tag_slot->blocker_snapshot_reliable.state_sequence == 0 || tag_slot->blocker_snapshot_reliable.last_response_opcode != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK @@ -13052,6 +13213,7 @@ cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, tag_slot->holder_ref = revoke->ref; memset(&tag_slot->holder_image, 0, sizeof(tag_slot->holder_image)); tag_slot->holder_image.image_id = revoke->image_id; + tag_slot->holder_required_page_scn = required_page_scn; pcm_x_transfer_leg_clear(&tag_slot->holder_reliable, PGRAC_IC_MSG_PCM_X_REVOKE, authenticated_master_node); memset(&tag_slot->blocker_snapshot_ref, 0, sizeof(tag_slot->blocker_snapshot_ref)); @@ -13086,6 +13248,16 @@ cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, } +PcmXQueueResult +cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, + int32 authenticated_master_node, + uint64 authenticated_master_session) +{ + return cluster_pcm_x_local_holder_revoke_apply_floor_exact( + revoke, 0, authenticated_master_node, authenticated_master_session); +} + + PcmXQueueResult cluster_pcm_x_local_holder_progress_exact(const BufferTag *tag, PcmXLocalHolderProgress *progress_out) @@ -13132,6 +13304,7 @@ cluster_pcm_x_local_holder_progress_exact(const BufferTag *tag, } else { progress_out->ref = tag_slot->holder_ref; progress_out->image = tag_slot->holder_image; + progress_out->required_page_scn = tag_slot->holder_required_page_scn; progress_out->reliable_state_sequence = tag_slot->holder_reliable.state_sequence; progress_out->pending_opcode = tag_slot->holder_reliable.pending_opcode; progress_out->last_response_opcode = tag_slot->holder_reliable.last_response_opcode; @@ -13218,11 +13391,32 @@ cluster_pcm_x_local_queue_invalidate_authorize_exact(const BufferTag *tag, uint6 || tag_slot->blocker_snapshot_ref.identity.request_id != request_id || tag_slot->blocker_snapshot_ref.identity.node_id != requester_node) result = PCM_X_QUEUE_STALE; - else if (!pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable) - || tag_slot->blocker_set_generation == 0) + else if (tag_slot->blocker_set_generation == 0) result = PCM_X_QUEUE_NOT_READY; - else if (tag_slot->blocker_set_generation == UINT64_MAX - || tag_slot->blocker_snapshot_reliable.state_sequence == 0 + else if (tag_slot->blocker_set_generation == UINT64_MAX) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else if (!pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable)) { + /* An authenticated exact INVALIDATE proves the master already crossed + * into transfer. A reordered same-ticket re-PROBE may have published + * another empty blocker COMMIT after the ACK that authorized that + * transition. No graph edge can be omitted in the empty set, so let + * passive pinned S become N+PI instead of waiting forever for an ACK + * the advanced master is no longer required to replay. */ + if (!pcm_x_transfer_leg_exact(&tag_slot->blocker_snapshot_reliable, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, + authenticated_master_node, authenticated_master_session) + || tag_slot->blocker_snapshot_count != 0) + result = PCM_X_QUEUE_NOT_READY; + else if (!pcm_x_local_empty_blocker_commit_exact( + tag_slot, authenticated_master_node, authenticated_master_session) + || (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 + || tag_slot->holder_terminal_drain_generation != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else + result = PCM_X_QUEUE_OK; + } else if (tag_slot->blocker_snapshot_reliable.state_sequence == 0 || tag_slot->blocker_snapshot_reliable.last_response_opcode != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK || tag_slot->blocker_snapshot_reliable.last_responder_node @@ -13246,10 +13440,10 @@ cluster_pcm_x_local_queue_invalidate_authorize_exact(const BufferTag *tag, uint6 PcmXQueueResult -cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_ready, - int32 authenticated_master_node, - uint64 authenticated_master_session, - PcmXGrantPayload *replay_out) +cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( + const PcmXGrantPayload *image_ready, int32 authenticated_master_node, + uint64 authenticated_master_session, PcmXGrantPayload *replay_out, + PcmXLocalImageReadyRefusal *refusal_out) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; PcmXRuntimeSnapshot runtime; @@ -13262,6 +13456,8 @@ cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_r bool gate_claimed = false; bool fail_closed = false; + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_NONE; if (replay_out != NULL) memset(replay_out, 0, sizeof(*replay_out)); if (header == NULL || image_ready == NULL || replay_out == NULL @@ -13270,19 +13466,31 @@ cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_r || !pcm_x_image_token_valid(&image_ready->image) || !pcm_x_image_id_master_exact(image_ready->image.image_id, authenticated_master_node) || authenticated_master_node < 0 || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT - || authenticated_master_session == 0) + || authenticated_master_session == 0) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_INVALID; return PCM_X_QUEUE_INVALID; + } runtime = cluster_pcm_x_runtime_snapshot(); - if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_RUNTIME; return PCM_X_QUEUE_NOT_READY; + } LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); directory_result = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &image_ready->ref.identity.tag, &tag_ref); LWLockRelease(&header->allocator_lock.lock); - if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_DIRECTORY; return PCM_X_QUEUE_NOT_FOUND; - if (directory_result != PCM_X_DIRECTORY_OK) + } + if (directory_result != PCM_X_DIRECTORY_OK) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_DIRECTORY; return pcm_x_queue_result_from_directory(directory_result); + } partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&image_ready->ref.identity.tag)); LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); @@ -13290,46 +13498,64 @@ cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_r &image_ready->ref.identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); if (tag_slot == NULL) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_TAG_SLOT; result = PCM_X_QUEUE_STALE; goto ready_done; } result = pcm_x_local_gate_try_acquire(tag_slot); if (result != PCM_X_QUEUE_OK) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_LOCAL_GATE; fail_closed = result == PCM_X_QUEUE_CORRUPT; goto ready_done; } gate_claimed = true; if (!pcm_x_runtime_token_exact(&runtime, 0)) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_RUNTIME_RECHECK; result = PCM_X_QUEUE_NOT_READY; goto ready_release_gate; } if (!pcm_x_local_holder_transfer_peer_exact(tag_slot, authenticated_master_node, - authenticated_master_session) + authenticated_master_session) || !pcm_x_ticket_ref_equal(&tag_slot->holder_ref, &image_ready->ref) || tag_slot->holder_image.image_id != image_ready->image.image_id) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_IDENTITY; result = PCM_X_QUEUE_STALE; goto ready_release_gate; } if (tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_ACTIVE_HOLDER; result = PCM_X_QUEUE_NOT_READY; goto ready_release_gate; } if (!pcm_x_transfer_leg_is_clear(&tag_slot->holder_reliable)) { if (!pcm_x_transfer_leg_exact(&tag_slot->holder_reliable, PGRAC_IC_MSG_PCM_X_IMAGE_READY, authenticated_master_node, authenticated_master_session)) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_RELIABLE_LEG; result = PCM_X_QUEUE_BUSY; goto ready_release_gate; } result = pcm_x_image_token_equal(&tag_slot->holder_image, &image_ready->image) ? PCM_X_QUEUE_DUPLICATE : PCM_X_QUEUE_STALE; + if (result == PCM_X_QUEUE_STALE && refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_IDENTITY; goto ready_output; } if (tag_slot->holder_reliable.last_response_opcode != PGRAC_IC_MSG_PCM_X_REVOKE) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_BAD_PHASE; result = PCM_X_QUEUE_BAD_STATE; goto ready_release_gate; } if (!cluster_pcm_x_generation_next(tag_slot->holder_reliable.state_sequence, &next_sequence)) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_COUNTER; result = PCM_X_QUEUE_COUNTER_EXHAUSTED; fail_closed = true; goto ready_release_gate; @@ -13351,6 +13577,8 @@ cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_r } ready_release_gate: if (gate_claimed && !fail_closed && !pcm_x_local_gate_release(tag_slot)) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_GATE_RELEASE; result = PCM_X_QUEUE_CORRUPT; fail_closed = true; } @@ -13362,6 +13590,17 @@ cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_r } +PcmXQueueResult +cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_ready, + int32 authenticated_master_node, + uint64 authenticated_master_session, + PcmXGrantPayload *replay_out) +{ + return cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( + image_ready, authenticated_master_node, authenticated_master_session, replay_out, NULL); +} + + PcmXQueueResult cluster_pcm_x_local_join_begin(const PcmXWaitIdentity *identity, int32 master_node, uint64 master_session_incarnation, PcmXLocalHandle *handle_out) @@ -16922,6 +17161,7 @@ pcm_x_local_holder_lane_retire_state(const PcmXLocalTagSlot *tag_slot, uint32 fl || tag_slot->blocker_snapshot_next_chunk != 0 || !pcm_x_transfer_leg_is_pristine(&tag_slot->blocker_snapshot_reliable) || !pcm_x_image_token_equal(&tag_slot->holder_image, &zero_image) + || tag_slot->holder_required_page_scn != 0 || !pcm_x_transfer_leg_is_pristine(&tag_slot->holder_reliable) || holder_flags != PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK || tag_slot->holder_terminal_drain_generation == 0 @@ -16937,6 +17177,7 @@ pcm_x_local_holder_lane_retire_state(const PcmXLocalTagSlot *tag_slot, uint32 fl return PCM_X_QUEUE_NOT_READY; if (holder_flags != 0 || tag_slot->holder_terminal_drain_generation != 0 || !pcm_x_image_token_equal(&tag_slot->holder_image, &zero_image) + || tag_slot->holder_required_page_scn != 0 || !pcm_x_transfer_leg_is_pristine(&tag_slot->holder_reliable)) return PCM_X_QUEUE_CORRUPT; return blocker_state; @@ -18395,14 +18636,16 @@ pcm_x_local_final_member_round_plan_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotR /* Close the terminal leg for an S-holder node that was not selected as the - * image source. Type-48 leaves a generation-zero admission locator plus an - * exact ACK tombstone. The authenticated DRAIN carries the first final grant - * generation this node can know, so capture it exactly and make all replays + * image source. Type-48 normally leaves an exact ACK tombstone; a reordered + * same-ticket re-PROBE can instead leave a canonical empty COMMIT after the + * master advanced. Authenticated DRAIN consumes either exact form, captures + * the first final grant generation this node can know, and makes every replay * generation- and drain-exact. Caller holds the local tag lock. */ static PcmXQueueResult pcm_x_local_blocker_participant_drain_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, const PcmXDrainPollPayload *poll, - int32 authenticated_master_node) + int32 authenticated_master_node, + uint64 authenticated_master_session) { PcmXImageToken zero_image; PcmXLocalRoundClosePlan round_plan; @@ -18446,6 +18689,7 @@ pcm_x_local_blocker_participant_drain_locked(PcmXLocalTagSlot *tag_slot, PcmXSlo return PCM_X_QUEUE_NOT_READY; if (holder_flags != 0 || tag_slot->holder_terminal_drain_generation != 0 || !pcm_x_image_token_equal(&tag_slot->holder_image, &zero_image) + || tag_slot->holder_required_page_scn != 0 || !pcm_x_transfer_leg_is_pristine(&tag_slot->holder_reliable)) return PCM_X_QUEUE_CORRUPT; if (!pcm_x_local_admission_ref_valid(&tag_slot->blocker_snapshot_ref, @@ -18453,7 +18697,9 @@ pcm_x_local_blocker_participant_drain_locked(PcmXLocalTagSlot *tag_slot, PcmXSlo || !BufferTagsEqual(&tag_slot->tag, &tag_slot->blocker_snapshot_ref.identity.tag) || tag_slot->blocker_snapshot_ref.identity.cluster_epoch != tag_slot->cluster_epoch) return PCM_X_QUEUE_CORRUPT; - if (!pcm_x_local_blocker_ack_tombstone_exact(tag_slot, authenticated_master_node) + if ((!pcm_x_local_blocker_ack_tombstone_exact(tag_slot, authenticated_master_node) + && !pcm_x_local_empty_blocker_commit_exact( + tag_slot, authenticated_master_node, authenticated_master_session)) || tag_slot->blocker_snapshot_reliable.response_tombstone_mask != 0) { blocker_state = pcm_x_local_blocker_lane_retire_state(tag_slot); return blocker_state == PCM_X_QUEUE_CORRUPT ? blocker_state : PCM_X_QUEUE_NOT_READY; @@ -18536,7 +18782,8 @@ pcm_x_local_holder_drain_poll_exact(const PcmXDrainPollPayload *poll, } if (pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref)) { result = pcm_x_local_blocker_participant_drain_locked(tag_slot, tag_ref, poll, - authenticated_master_node); + authenticated_master_node, + authenticated_master_session); fail_closed = result == PCM_X_QUEUE_CORRUPT; goto holder_drain_done; } @@ -18834,8 +19081,8 @@ cluster_pcm_x_local_drain_poll_certificate_exact(const PcmXDrainPollPayload *pol } } else if (!pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref) && pcm_x_ticket_locator_equal(&tag_slot->blocker_snapshot_ref, &poll->ref)) { - external_result = pcm_x_local_blocker_participant_drain_locked(tag_slot, tag_ref, poll, - authenticated_master_node); + external_result = pcm_x_local_blocker_participant_drain_locked( + tag_slot, tag_ref, poll, authenticated_master_node, authenticated_master_session); if (external_result != PCM_X_QUEUE_OK && external_result != PCM_X_QUEUE_DUPLICATE) { result = external_result; fail_closed = result == PCM_X_QUEUE_CORRUPT; @@ -20012,6 +20259,7 @@ pcm_x_local_holder_detach_terminal_exact(const PcmXTicketRef *ref, int32 authent } else { memset(&tag_slot->holder_ref, 0, sizeof(tag_slot->holder_ref)); memset(&tag_slot->holder_image, 0, sizeof(tag_slot->holder_image)); + tag_slot->holder_required_page_scn = 0; memset(&tag_slot->holder_reliable, 0, sizeof(tag_slot->holder_reliable)); } tag_slot->holder_terminal_drain_generation = 0; diff --git a/src/backend/cluster/cluster_pcm_x_image_fetch.c b/src/backend/cluster/cluster_pcm_x_image_fetch.c index a73f60b6c0..3812cf3211 100644 --- a/src/backend/cluster/cluster_pcm_x_image_fetch.c +++ b/src/backend/cluster/cluster_pcm_x_image_fetch.c @@ -86,43 +86,88 @@ cluster_pcm_x_image_fetch_build_request(const PcmXLocalProgress *progress, int32 bool -cluster_pcm_x_image_fetch_request_exact(const ClusterICEnvelope *env, - const GcsBlockRequestPayload *request, - const PcmXLocalHolderProgress *holder, int32 holder_node, - int32 current_master_node, uint64 current_epoch) +cluster_pcm_x_image_fetch_request_exact_diagnosed( + const ClusterICEnvelope *env, const GcsBlockRequestPayload *request, + const PcmXLocalHolderProgress *holder, int32 holder_node, int32 current_master_node, + uint64 current_epoch, PcmXImageFetchRequestRefusal *refusal_out) { static const uint8 zero_reserved[sizeof(request->reserved_0)] = { 0 }; int32 decoded_backend_id; int32 decoded_master_node; int32 decoded_requester_node; - return env != NULL && request != NULL && holder != NULL && holder_node >= 0 - && holder_node < PCM_X_PROTOCOL_NODE_LIMIT && current_master_node >= 0 - && current_master_node < PCM_X_PROTOCOL_NODE_LIMIT - && env->source_node_id == (uint32)request->sender_node - && env->dest_node_id == (uint32)holder_node && env->payload_length == sizeof(*request) - && request->sender_node >= 0 && request->sender_node < PCM_X_PROTOCOL_NODE_LIMIT - && request->requester_backend_id > 0 && request->epoch == current_epoch - && request->transition_id == (uint8)PCM_TRANS_N_TO_S - && memcmp(request->reserved_0, zero_reserved, sizeof(zero_reserved)) == 0 - && holder->master_node == current_master_node && holder->master_session_incarnation != 0 - && holder->ref.grant_generation != 0 && holder->ref.handle.ticket_id != 0 - && holder->ref.handle.queue_generation != 0 - && holder->ref.identity.cluster_epoch == current_epoch - && holder->ref.identity.wait_seq != 0 - && BufferTagsEqual(&holder->ref.identity.tag, &request->tag) - && holder->ref.identity.node_id == request->sender_node - && holder->pending_opcode == PGRAC_IC_MSG_PCM_X_IMAGE_READY - && holder->phase == PGRAC_IC_MSG_PCM_X_IMAGE_READY && holder->flags == 0 - && pcm_x_image_fetch_token_valid(&holder->image, current_master_node) - && holder->image.image_id == request->request_id - && holder->image.source_node == (uint32)holder_node - && cluster_pcm_x_image_id_decode(request->request_id, &decoded_master_node, NULL) - && decoded_master_node == current_master_node - && cluster_gcs_requester_id_decode(holder->ref.identity.request_id, - &decoded_requester_node, &decoded_backend_id, NULL) - && decoded_requester_node == request->sender_node - && decoded_backend_id == request->requester_backend_id; + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_NONE; + if (env == NULL || request == NULL || holder == NULL || holder_node < 0 + || holder_node >= PCM_X_PROTOCOL_NODE_LIMIT || current_master_node < 0 + || current_master_node >= PCM_X_PROTOCOL_NODE_LIMIT) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_ARGUMENT; + return false; + } + if (env->source_node_id != (uint32)request->sender_node + || env->dest_node_id != (uint32)holder_node || env->payload_length != sizeof(*request)) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_ENVELOPE; + return false; + } + if (request->sender_node < 0 || request->sender_node >= PCM_X_PROTOCOL_NODE_LIMIT + || request->requester_backend_id <= 0 || request->epoch != current_epoch + || request->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(request->reserved_0, zero_reserved, sizeof(zero_reserved)) != 0) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_REQUEST; + return false; + } + if (holder->master_node != current_master_node || holder->master_session_incarnation == 0) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_MASTER; + return false; + } + if (holder->ref.grant_generation == 0 || holder->ref.handle.ticket_id == 0 + || holder->ref.handle.queue_generation == 0 + || holder->ref.identity.cluster_epoch != current_epoch || holder->ref.identity.wait_seq == 0 + || !BufferTagsEqual(&holder->ref.identity.tag, &request->tag) + || holder->ref.identity.node_id != request->sender_node) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_REF; + return false; + } + if (holder->pending_opcode != PGRAC_IC_MSG_PCM_X_IMAGE_READY + || holder->phase != PGRAC_IC_MSG_PCM_X_IMAGE_READY || holder->flags != 0) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_LEG; + return false; + } + if (!pcm_x_image_fetch_token_valid(&holder->image, current_master_node) + || holder->image.image_id != request->request_id + || holder->image.source_node != (uint32)holder_node + || !cluster_pcm_x_image_id_decode(request->request_id, &decoded_master_node, NULL) + || decoded_master_node != current_master_node) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_IMAGE; + return false; + } + if (!cluster_gcs_requester_id_decode(holder->ref.identity.request_id, + &decoded_requester_node, &decoded_backend_id, NULL) + || decoded_requester_node != request->sender_node + || decoded_backend_id != request->requester_backend_id) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_REQUESTER_ID; + return false; + } + return true; +} + + +bool +cluster_pcm_x_image_fetch_request_exact(const ClusterICEnvelope *env, + const GcsBlockRequestPayload *request, + const PcmXLocalHolderProgress *holder, int32 holder_node, + int32 current_master_node, uint64 current_epoch) +{ + return cluster_pcm_x_image_fetch_request_exact_diagnosed( + env, request, holder, holder_node, current_master_node, current_epoch, NULL); } diff --git a/src/backend/cluster/cluster_sf_dep.c b/src/backend/cluster/cluster_sf_dep.c index 90d2d0d1d0..1cea155b8f 100644 --- a/src/backend/cluster/cluster_sf_dep.c +++ b/src/backend/cluster/cluster_sf_dep.c @@ -387,6 +387,63 @@ cluster_sf_peer_supports_pcm_x_rebase(int32 peer_id) return (capabilities & PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1) != 0; } +bool +cluster_sf_peer_supports_pcm_x_source_floor(int32 peer_id) +{ + uint32 capabilities; + + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return false; + + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + capabilities = cluster_sf_peer_cap_bits(&ClusterSfDep->peer_capabilities[peer_id]); + LWLockRelease(&ClusterSfDep->lock); + return (capabilities & PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1) != 0; +} + +/* One lock-coherent sample for choosing the type-49 wire version. CONVERT + * authenticates the PCM-X family; SOURCE_FLOOR and its connection generation + * are sampled from that same record so callers never pair bits across a + * reconnect. */ +bool +cluster_sf_peer_pcm_x_source_floor_sample(int32 peer_id, bool *source_floor_out, + uint32 *generation_out) +{ + bool supported; + + if (source_floor_out != NULL) + *source_floor_out = false; + if (generation_out != NULL) + *generation_out = 0; + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return false; + + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + supported = cluster_sf_peer_cap_family_sample( + &ClusterSfDep->peer_capabilities[peer_id], PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1, source_floor_out, generation_out); + LWLockRelease(&ClusterSfDep->lock); + return supported; +} + +/* Drain-side exact fence for a capability-bound LMS slot. */ +bool +cluster_sf_peer_capability_generation_matches(int32 peer_id, uint32 required_capabilities, + uint32 expected_generation) +{ + bool matches; + + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES + || required_capabilities == 0) + return false; + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + matches = cluster_sf_peer_cap_generation_matches_exact( + &ClusterSfDep->peer_capabilities[peer_id], required_capabilities, + expected_generation); + LWLockRelease(&ClusterSfDep->lock); + return matches; +} + /* * cluster_sf_peer_pcm_x_capability_sample * diff --git a/src/backend/cluster/cluster_tt_local.c b/src/backend/cluster/cluster_tt_local.c index c3fbd7324c..34f02775fa 100644 --- a/src/backend/cluster/cluster_tt_local.c +++ b/src/backend/cluster/cluster_tt_local.c @@ -57,6 +57,7 @@ #include "cluster/cluster_undo_record_api.h" /* spec-3.12 D2b cluster_undo_tt_rollover_locked */ #include "cluster/cluster_tt_status.h" #include "cluster/cluster_tt_status_hint.h" /* spec-3.2 D4 wire emit append */ +#include "cluster/cluster_uba.h" /* P0-33 exact data-ref alias */ #include "cluster/storage/cluster_undo_alloc.h" /* cluster_undo_active_segment_for_node_or_create */ #ifdef USE_PGRAC_CLUSTER @@ -152,6 +153,12 @@ typedef struct ClusterTTLocalBinding { * segment away (cluster_tt_slot_get_wrap would then ERROR on the stale id). */ uint16 wrap; uint32 cluster_epoch; /* snapshot at bind time */ + /* Real undo records use an independent record cursor. Its segment may + * differ from segment_id after rollover, while the page ref carries that + * record segment as its exact-key segment. */ + uint16 *active_alias_segments; + uint16 active_alias_count; + uint16 active_alias_capacity; } ClusterTTLocalBinding; static ClusterTTLocalBinding *cluster_tt_local_bindings = NULL; @@ -468,40 +475,83 @@ cluster_tt_local_finish_bindings(bool committed, SCN commit_scn) cluster_tt_local_reset_binding(); } -/* - * build_local_key -- compose ClusterTTStatusKey for `xid`. - * - * Returns true with the key populated when an xact-local binding - * exists for `xid` (F11 real allocator path); returns false otherwise - * and leaves `*out` zeroed. spec-3.4b F7: production code paths MUST - * NOT take a provisional id when the binding is absent -- this would - * create a real/provisional key collision risk. Callers should - * silently skip install when build_local_key returns false (the - * matching DML never stamped an ITL slot, so no overlay entry is - * needed). - */ +/* Compose either the canonical key or one page-ref segment alias. */ static bool -build_local_key(TransactionId xid, ClusterTTStatusKey *out) +build_binding_key(const ClusterTTLocalBinding *binding, uint16 segment_id, + ClusterTTStatusKey *out) { - uint32 seg; - uint16 off; - uint32 tt_id; - uint32 epoch; - memset(out, 0, sizeof(*out)); - - if (!cluster_tt_local_peek_binding(xid, &seg, &off, &tt_id, &epoch, NULL)) + if (binding == NULL || !TransactionIdIsValid(binding->top_xid) + || segment_id == 0) return false; out->origin_node_id = (uint16)cluster_node_id; - out->undo_segment_id = (uint16)seg; - out->tt_slot_id = tt_id; - out->cluster_epoch = epoch; - out->local_xid = xid; - /* _reserved + _reserved2 already zero from memset. */ + out->undo_segment_id = segment_id; + out->tt_slot_id = cluster_tt_slot_offset_to_id(binding->slot_offset); + out->cluster_epoch = binding->cluster_epoch; + out->local_xid = binding->top_xid; return true; } +static bool +build_local_key(TransactionId xid, ClusterTTStatusKey *out) +{ + int idx = cluster_tt_local_find_binding(xid); + + if (idx < 0) + { + memset(out, 0, sizeof(*out)); + return false; + } + return build_binding_key(&cluster_tt_local_bindings[idx], + (uint16)cluster_tt_local_bindings[idx].segment_id, out); +} + +/* Install and emit one already-minted exact key. */ +static void +install_key(const ClusterTTStatusKey *key, ClusterTTStatus status, SCN commit_scn) +{ + bool installed = cluster_tt_status_install_local(key, status, commit_scn); + + if (!installed) + return; + +#ifdef USE_ASSERT_CHECKING + { + ClusterTTStatusResult res; + bool hit = false; + bool epoch_stable = ((uint32)cluster_epoch_get_current() == key->cluster_epoch); + + if (epoch_stable) + { + hit = cluster_tt_status_lookup_exact(key, &res); + epoch_stable = ((uint32)cluster_epoch_get_current() == key->cluster_epoch); + } + if (epoch_stable) + Assert(hit && res.authoritative && res.status == status); + if (hit && res.authoritative && res.status == status) + cluster_tt_status_bump_self_consumer_hit(); + } +#endif + + cluster_tt_status_hint_emit(key, status, commit_scn); +} + +static void +install_binding_aliases(const ClusterTTLocalBinding *binding, + ClusterTTStatus status, SCN commit_scn) +{ + uint16 i; + + for (i = 0; i < binding->active_alias_count; i++) + { + ClusterTTStatusKey key; + + if (build_binding_key(binding, binding->active_alias_segments[i], &key)) + install_key(&key, status, commit_scn); + } +} + /* * install_status -- common path for commit / abort install + N7 * self-consumer lookup. @@ -515,7 +565,8 @@ static void install_status(TransactionId xid, ClusterTTStatus status, SCN commit_scn) { ClusterTTStatusKey key; - bool installed; + ClusterTTLocalBinding *binding; + int idx; if (!cluster_enabled || cluster_node_id < 0) return; @@ -528,8 +579,10 @@ install_status(TransactionId xid, ClusterTTStatus status, SCN commit_scn) * (DDL-only or read-only); no overlay entry is needed and the * provisional fallback path is forbidden in production. */ - if (!build_local_key(xid, &key)) + idx = cluster_tt_local_find_binding(xid); + if (idx < 0 || !build_local_key(xid, &key)) return; + binding = &cluster_tt_local_bindings[idx]; /* * spec-3.3 D6 (L181 chain step 2): install in-memory overlay entry @@ -537,56 +590,8 @@ install_status(TransactionId xid, ClusterTTStatus status, SCN commit_scn) * carries commit_scn to (a) self-consumer N7 lookup and (b) the wire * emit path which will ship it to peers in V2 hints (D8). */ - installed = cluster_tt_status_install_local(&key, status, commit_scn); - if (!installed) - return; - - /* - * spec-3.1 v0.4 N7 self-consumer: re-lookup to prove the just-installed key - * is reachable + bump self_consumer_hit_count (D9 T8 + D10 L2). - * - * P0 perf hardening: this is now a DEBUG-ONLY invariant check. In every - * commit it cost an extra HTAB lookup + LW_SHARED on the hot path; the - * production install path does not need to re-read its own write. Assert - * builds still fail fast if the install cannot read its own key, and the - * self_consumer_hit_count counter is now a debug-build signal. - */ -#ifdef USE_ASSERT_CHECKING - { - ClusterTTStatusResult res; - bool hit = false; - bool epoch_stable = ((uint32)cluster_epoch_get_current() == key.cluster_epoch); - - /* - * lookup_exact() is the production reader API, so it intentionally fences - * against the current cluster epoch. A transaction can abort while a - * clean-leave/reconfig advances the epoch; the local old-epoch status - * install is still valid evidence for that transaction, but public lookup - * must reject it. Keep this debug wiring check only when the epoch stayed - * stable across the install/lookup window. - */ - if (epoch_stable) { - hit = cluster_tt_status_lookup_exact(&key, &res); - epoch_stable = ((uint32)cluster_epoch_get_current() == key.cluster_epoch); - } - - if (epoch_stable) - Assert(hit && res.authoritative && res.status == status); - if (hit && res.authoritative && res.status == status) - cluster_tt_status_bump_self_consumer_hit(); - } -#endif - - /* - * spec-3.2 D4 + spec-3.3 D6/D8 (L181 chain step 4): cross-node TT - * status hint wire emit. Uses the EXACT key just minted + installed - * (HC184 — no raw-xid rebuild); commit_scn flows through so peers - * see the real value via V2 wire. Fire-and-forget; emit_mode = - * disabled is a no-op inside the function. commit/abort 对称 - * (install_status is called from both record_commit + record_abort; - * abort path passes InvalidScn). - */ - cluster_tt_status_hint_emit(&key, status, commit_scn); + install_key(&key, status, commit_scn); + install_binding_aliases(binding, status, commit_scn); } /* ------------------------------------------------------------ */ @@ -700,6 +705,71 @@ cluster_tt_local_record_active(TransactionId xid) * running and may stamp additional lock-only or data ITL slots. */ } +void +cluster_tt_local_record_data_active(TransactionId xid, UBA uba) +{ + ClusterTTLocalBinding *binding; + uint32 record_segment; + uint32 block_no; + uint16 slot_offset; + uint16 row_offset; + int idx; + uint16 i; + + if (!cluster_enabled || cluster_node_id < 0 || !TransactionIdIsNormal(xid)) + return; + idx = cluster_tt_local_find_binding(xid); + if (idx < 0) + return; + binding = &cluster_tt_local_bindings[idx]; + + /* The writer made this UBA from the same binding before entering CRIT. + * If that invariant is ever broken, publish no guessed alias. */ + if (!uba_decode(uba, &record_segment, &block_no, &slot_offset, &row_offset) + || uba_origin_node_id(uba) != (NodeId)cluster_node_id + || slot_offset != binding->slot_offset) + { + Assert(false); + return; + } + (void)block_no; + (void)row_offset; + + if (record_segment != binding->segment_id) + { + for (i = 0; i < binding->active_alias_count; i++) + if (binding->active_alias_segments[i] == (uint16)record_segment) + break; + + if (i == binding->active_alias_count) + { + MemoryContext oldcxt; + + oldcxt = MemoryContextSwitchTo(TopTransactionContext); + if (binding->active_alias_segments == NULL) + { + binding->active_alias_capacity = 4; + binding->active_alias_segments = (uint16 *)palloc( + sizeof(uint16) * binding->active_alias_capacity); + } + else if (binding->active_alias_count == binding->active_alias_capacity) + { + binding->active_alias_capacity *= 2; + binding->active_alias_segments = (uint16 *)repalloc( + binding->active_alias_segments, + sizeof(uint16) * binding->active_alias_capacity); + } + MemoryContextSwitchTo(oldcxt); + binding->active_alias_segments[binding->active_alias_count++] + = (uint16)record_segment; + } + } + + /* install_status publishes canonical + every registered alias. Commit and + * abort use the same path, closing each alias terminally. */ + install_status(xid, CLUSTER_TT_STATUS_IN_PROGRESS, InvalidScn); +} + uint32 cluster_tt_local_slot_seq_peek(void) { @@ -743,6 +813,13 @@ cluster_tt_local_record_active(TransactionId xid) (void)xid; } +void +cluster_tt_local_record_data_active(TransactionId xid, UBA uba) +{ + (void)xid; + (void)uba; +} + uint32 cluster_tt_local_slot_seq_peek(void) { diff --git a/src/backend/cluster/cluster_visibility_verdict.c b/src/backend/cluster/cluster_visibility_verdict.c index ee3d6119d9..e6617390d1 100644 --- a/src/backend/cluster/cluster_visibility_verdict.c +++ b/src/backend/cluster/cluster_visibility_verdict.c @@ -30,9 +30,34 @@ #ifdef USE_PGRAC_CLUSTER +#include "access/htup_details.h" + #include "cluster/cluster_tt_status.h" #include "cluster/cluster_visibility_resolve.h" +/* + * P0-27: VACUUM freeze is an authoritative xmin-committed proof. Only the + * exact FROZEN bit pair bypasses old xmin identity resolution; a lone native + * COMMITTED or INVALID hint still follows cluster evidence and fails closed + * when that evidence is stale or ambiguous. + */ +bool +cluster_vis_xmin_needs_resolution(uint16 infomask) +{ + return (infomask & HEAP_XMIN_FROZEN) != HEAP_XMIN_FROZEN; +} + +/* + * P0-28: local GlobalVis/CLOG state cannot prove that no peer still names a + * shared-storage TID. Reclamation is legal only after a cluster-wide + * non-removable horizon is available. + */ +bool +cluster_vis_prune_must_defer(bool storage_mode, bool cluster_horizon_available) +{ + return storage_mode && !cluster_horizon_available; +} + /* ============================================================ * spec-3.14 §2.2 OBS truth tables (pure verdict functions). * diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 3cfcb1a1e7..2117dacdff 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -107,8 +107,7 @@ extern bool ignore_checksum_failure; #define BufferGetLSN(bufHdr) (PageGetLSN(BufHdrGetBlock(bufHdr))) /* Note: this macro only works on local buffers, not shared ones! */ -#define LocalBufHdrGetBlock(bufHdr) \ - LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)] +#define LocalBufHdrGetBlock(bufHdr) LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)] /* Bits in SyncOneBuffer's return value */ #define BUF_WRITTEN 0x01 @@ -117,31 +116,10 @@ extern bool ignore_checksum_failure; #define RELS_BSEARCH_THRESHOLD 20 #ifdef USE_PGRAC_CLUSTER -/* - * spec-6.14 D4: single PCM-tracking criterion by relfilenumber. Under - * cluster.shared_catalog the catalog lives in the single shared tree, so its - * pages need the same N/S/X + CR + PI lost-write coherency as user pages - * (routing a catalog page into the shared tree without PCM tracking would be a - * double-write hazard -- pairs with the D3 smgr flip, INV-14-1). All shared - * buffers under shared_catalog=on hold permanent (non-temp) relations: temp - * relations use local buffers and never reach these paths, and unlogged - * permanent relations are rejected at DDL time (Q12). Off mode keeps the - * historic FirstNormalObjectId user-only boundary. - * - * The eviction/release hooks (HC112) MUST use this SAME criterion as the - * acquire path, otherwise a tracked catalog page's PCM lock would leak on - * eviction (spec-6.14 R6 -- do not let a gate drift). - */ -static inline bool -cluster_bufmgr_reln_pcm_tracked(RelFileNumber relnum) -{ - return cluster_shared_catalog || relnum >= (RelFileNumber) FirstNormalObjectId; -} - static inline bool cluster_bufmgr_should_pcm_track(BufferDesc *buf) { - return cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(&buf->tag)); + return cluster_pcm_x_buffer_tag_tracked(&buf->tag, cluster_shared_catalog); } /* Defined with the GCS copy/drop substrate below. The queue ownership @@ -152,6 +130,15 @@ static void cluster_bufmgr_pin_for_gcs_locked(BufferDesc *buf, uint32 buf_state) static void cluster_bufmgr_unpin_for_gcs(BufferDesc *buf); static void FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, IOContext io_context); +static ClusterPcmOwnResult cluster_bufmgr_pcm_own_finish_x_to_s_downgrade( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, + ClusterPcmOwnSnapshot *out_shared); + +/* Process-local dynamic scope for the exact retain-finish FlushBuffer call. */ +static bool cluster_pcm_x_finish_retain_flush_active = false; +static bool cluster_pcm_x_finish_retain_flush_io_active = false; +static bool cluster_pcm_x_finish_retain_flush_error_context_pushed = false; +static ErrorContextCallback *cluster_pcm_x_finish_retain_flush_error_context_previous = NULL; static ClusterPcmOwnResult cluster_pcm_own_bump_failure(BufferDesc *buf, uint64 generation, uint32 *out_flags) @@ -257,10 +244,11 @@ cluster_pcm_own_transition(BufferDesc *buf, uint8 new_pcm_state, uint32 set_flag UnlockBufHdr(buf, buf_state); if (fx_log) elog(LOG, - "cluster PCM own S/X over !BM_VALID: site=own-transition buffer=%d rel=%u fork=%d blk=%u target=%u refcount=%u old_state=%u gen=%llu token=%llu", - buf->buf_id, fx_tag.relNumber, (int) fx_tag.forkNum, fx_tag.blockNum, - (unsigned) new_pcm_state, fx_refcount, (unsigned) fx_old_state, - (unsigned long long) new_generation, (unsigned long long) fx_token); + "cluster PCM own S/X over !BM_VALID: site=own-transition buffer=%d rel=%u fork=%d " + "blk=%u target=%u refcount=%u old_state=%u gen=%llu token=%llu", + buf->buf_id, fx_tag.relNumber, (int)fx_tag.forkNum, fx_tag.blockNum, + (unsigned)new_pcm_state, fx_refcount, (unsigned)fx_old_state, + (unsigned long long)new_generation, (unsigned long long)fx_token); } /* @@ -297,10 +285,34 @@ cluster_pcm_own_snapshot_locked(BufferDesc *buf, ClusterPcmOwnSnapshot *out) out->tag = buf->tag; out->generation = cluster_pcm_own_gen_get(buf->buf_id); out->reservation_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + out->writer_activation_token + = cluster_pcm_own_writer_activation_token_get(buf->buf_id); out->flags = cluster_pcm_own_flags_get(buf->buf_id); out->pcm_state = buf->pcm_state; } +/* Diagnostic only: callers capture under the BufferDesc header lock and emit + * after releasing it. Keeping one format for commit, exact clear, and the + * unguarded legacy N boundaries makes a stale activation fence traceable to + * its last exact tag/descriptor tuple without changing ownership semantics. */ +static void +cluster_pcm_own_activation_diag_emit(const char *site, int buffer_id, + const ClusterPcmOwnSnapshot *snapshot, + ClusterPcmOwnResult result) +{ + if (site == NULL || snapshot == NULL) + return; + elog(LOG, + "cluster PCM writer activation diagnostic: site=%s buffer=%d rel=%u fork=%d blk=%u " + "generation=%llu reservation_token=%llu writer_activation_token=%llu flags=0x%x " + "pcm_state=%u result=%d", + site, buffer_id, snapshot->tag.relNumber, (int)snapshot->tag.forkNum, + snapshot->tag.blockNum, (unsigned long long)snapshot->generation, + (unsigned long long)snapshot->reservation_token, + (unsigned long long)snapshot->writer_activation_token, snapshot->flags, + (unsigned int)snapshot->pcm_state, (int)result); +} + /* A normal D-h1 PI is !BM_VALID. PCM-X retained-image transfer deliberately * uses the otherwise-unoccupied PI+BM_VALID shape while N+REVOKING keeps the * exact descriptor pinned to the image record. Write/reuse gates recognize @@ -606,10 +618,13 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, return result; content_lock = BufferDescriptorGetContentLock(buf); + if (!LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) { + cluster_bufmgr_unpin_for_gcs(buf); + return CLUSTER_PCM_OWN_BUSY; + } + content_locked = true; PG_TRY(); { - LWLockAcquire(content_lock, LW_EXCLUSIVE); - content_locked = true; buf_state = LockBufHdr(buf); if (!cluster_pcm_own_snapshot_matches_locked(buf, &expected_s) || (buf_state & BM_VALID) == 0 @@ -903,13 +918,16 @@ cluster_bufmgr_pcm_own_handoff_s_revoke_to_x_reservation( static ClusterPcmOwnResult cluster_pcm_own_finish_grant_reservation(BufferDesc *buf, const ClusterPcmOwnSnapshot *base, uint64 reservation_token, uint8 new_pcm_state, + bool arm_writer_activation, uint64 *out_committed_generation, ClusterPcmOwnSnapshot *live_out) { + ClusterPcmOwnSnapshot activation_diag; ClusterPcmOwnSnapshot live; ClusterPcmXGrantReservationKind kind; ClusterPcmOwnResult result; uint32 buf_state; + bool activation_diag_valid = false; if (buf == NULL || base == NULL || out_committed_generation == NULL) return CLUSTER_PCM_OWN_INVALID; @@ -948,15 +966,26 @@ cluster_pcm_own_finish_grant_reservation(BufferDesc *buf, const ClusterPcmOwnSna * caller initializes it under content EXCLUSIVE. */ result = CLUSTER_PCM_OWN_CORRUPT; else { - result = cluster_pcm_own_grant_commit_exact(buf->buf_id, base->generation, - reservation_token, out_committed_generation); + if (new_pcm_state == (uint8)PCM_STATE_X && arm_writer_activation) + result = cluster_pcm_own_writer_grant_commit_exact( + buf->buf_id, base->generation, reservation_token, out_committed_generation); + else + result = cluster_pcm_own_grant_commit_exact(buf->buf_id, base->generation, + reservation_token, out_committed_generation); if (result == CLUSTER_PCM_OWN_OK) { buf->buffer_type = new_pcm_state == (uint8)PCM_STATE_S ? (uint8)BUF_TYPE_SCUR : (uint8)BUF_TYPE_XCUR; buf->pcm_state = new_pcm_state; + if (new_pcm_state == (uint8)PCM_STATE_X && arm_writer_activation) { + cluster_pcm_own_snapshot_locked(buf, &activation_diag); + activation_diag_valid = true; + } } } UnlockBufHdr(buf, buf_state); + if (activation_diag_valid) + cluster_pcm_own_activation_diag_emit("writer-commit", buf->buf_id, + &activation_diag, result); /* Every X ownership commit funnels through this exact commit; the S-grant * finish and every STALE/INVALID/CORRUPT refusal stay uncounted. */ if (result == CLUSTER_PCM_OWN_OK && new_pcm_state == (uint8)PCM_STATE_X) @@ -973,8 +1002,9 @@ cluster_bufmgr_pcm_own_finish_x_commit(BufferDesc *buf, const ClusterPcmOwnSnaps && expected->pcm_state != (uint8)PCM_STATE_X)) return CLUSTER_PCM_OWN_STALE; return cluster_pcm_own_finish_grant_reservation(buf, expected, reservation_token, - (uint8)PCM_STATE_X, out_committed_generation, - NULL); + (uint8)PCM_STATE_X, true, + out_committed_generation, + NULL); } static ClusterPcmOwnResult @@ -1159,7 +1189,7 @@ cluster_pcm_own_finish_grant_or_rollback(BufferDesc *buf, const ClusterPcmOwnSna memset(&live, 0, sizeof(live)); finish_result = cluster_pcm_own_finish_grant_reservation( - buf, base, reservation_token, new_pcm_state, out_committed_generation, &live); + buf, base, reservation_token, new_pcm_state, false, out_committed_generation, &live); if (finish_result == CLUSTER_PCM_OWN_OK) return; @@ -1301,6 +1331,9 @@ typedef struct ClusterPcmXWriterLedgerEntry { PcmXLocalWriterClaim claim; ClusterPcmOwnSnapshot granted; bool claim_handed_off; + bool claim_cleanup_complete; + bool grant_snapshot_exact; + bool activation_fence_armed; } ClusterPcmXWriterLedgerEntry; static ClusterPcmXWriterLedgerEntry cluster_bufmgr_pcm_x_writer_ledger[LWLOCK_MAX_HELD_BY_PROC]; @@ -1426,13 +1459,12 @@ cluster_bufmgr_pcm_legacy_begin_probe(BufferDesc *buf, PcmLockMode pcm_mode, return; buf_state = pg_atomic_read_u32(&buf->state); elog(LOG, - "cluster PCM legacy reservation opened from a non-N base: site=%s mode=%d buffer=%d rel=%u fork=%d blk=%u valid=%d base_state=%u base_gen=%llu base_token=%llu base_flags=0x%x pcm_state_now=%u", - site, (int) pcm_mode, buf->buf_id, - buf->tag.relNumber, (int) buf->tag.forkNum, buf->tag.blockNum, - (buf_state & BM_VALID) != 0 ? 1 : 0, - base->pcm_state, - (unsigned long long) base->generation, - (unsigned long long) base->reservation_token, + "cluster PCM legacy reservation opened from a non-N base: site=%s mode=%d buffer=%d " + "rel=%u fork=%d blk=%u valid=%d base_state=%u base_gen=%llu base_token=%llu " + "base_flags=0x%x pcm_state_now=%u", + site, (int)pcm_mode, buf->buf_id, buf->tag.relNumber, (int)buf->tag.forkNum, + buf->tag.blockNum, (buf_state & BM_VALID) != 0 ? 1 : 0, base->pcm_state, + (unsigned long long)base->generation, (unsigned long long)base->reservation_token, base->flags, buf->pcm_state); } @@ -1443,6 +1475,31 @@ typedef enum ClusterBufmgrPcmRetryRearmResult CLUSTER_BUFMGR_PCM_RETRY_BARRIER_REFUSED } ClusterBufmgrPcmRetryRearmResult; +/* A queue INVALIDATE that meets mirror-N + GRANT_PENDING returns BUSY. Its + * master retries no sooner than max(starvation backoff, LMON tick), while the + * DENIED_PENDING_X requester owns this function's wait. Keep the exact old + * reservation absent for two such intervals before publishing the successor: + * one interval lets the master stage its retry and the second lets the DATA + * worker consume it. Later denials widen that receive window, capped at the + * configured GUC maximum (60s base, 120s quiesce) rather than overflowing. + */ +static long +cluster_bufmgr_pcm_pending_x_retry_delay_ms(uint32 wait_index) +{ + uint64 retry_delay_ms; + uint32 shift = Min(wait_index, UINT32_C(16)); + + retry_delay_ms = (uint64)Max(Max(cluster_gcs_block_starvation_backoff_ms, + cluster_lmon_main_loop_interval), + 1); + if (retry_delay_ms >= UINT64_C(60000) + || retry_delay_ms > (UINT64_C(60000) >> shift)) + retry_delay_ms = UINT64_C(60000); + else + retry_delay_ms <<= shift; + return (long)(retry_delay_ms * 2); +} + /* Consume one authoritative DENIED_PENDING_X. The exact old reservation is * aborted before sleeping; STALE is an ABA-safe zero-mutation result and is * handled by re-sampling the complete ownership tuple. A successful rearm @@ -1476,6 +1533,15 @@ cluster_bufmgr_pcm_retry_denied_rearm(BufferDesc *buf, PcmLockMode pcm_mode, * already a stable covering grant, rejoin the normal post-content-lock * generation check instead of opening another reservation over it. */ cluster_pcm_own_read(buf, ¤t_state, covered_generation, ¤t_flags); + elog(LOG, + "cluster PCM pending-X exact abort observation: buffer=%d rel=%u fork=%d blk=%u " + "mode=%d wait_index=%u base_state=%u base_generation=%llu token=%llu " + "abort_result=%d live_state=%u live_generation=%llu live_token=%llu live_flags=0x%x", + buf->buf_id, buf->tag.relNumber, (int)buf->tag.forkNum, buf->tag.blockNum, + (int)pcm_mode, wait_index, base->pcm_state, + (unsigned long long)base->generation, (unsigned long long)*reservation_token, + (int)own_result, current_state, (unsigned long long)*covered_generation, + (unsigned long long)cluster_pcm_own_reservation_token_get(buf->buf_id), current_flags); if (current_flags == 0 && cluster_gcs_block_local_cache && cluster_pcm_mode_covers((PcmLockMode) current_state, pcm_mode)) return CLUSTER_BUFMGR_PCM_RETRY_COVERED; @@ -1493,20 +1559,14 @@ cluster_bufmgr_pcm_retry_denied_rearm(BufferDesc *buf, PcmLockMode pcm_mode, "pending-X retry nested guard"); } - backoff_ms = (long) cluster_gcs_block_starvation_backoff_ms - * (1L << (wait_index < 16 ? wait_index : 16)); - if (backoff_ms <= 0) - backoff_ms = 1; - if (backoff_ms > 25000) - backoff_ms = 25000; + backoff_ms = cluster_bufmgr_pcm_pending_x_retry_delay_ms(wait_index); CHECK_FOR_INTERRUPTS(); (void) WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, backoff_ms, WAIT_EVENT_GCS_BLOCK_STARVATION_RETRY); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); - own_result = cluster_bufmgr_pcm_begin_grant_reservation_wait(buf, base, - reservation_token); + own_result = cluster_bufmgr_pcm_begin_grant_reservation_wait(buf, base, reservation_token); if (own_result != CLUSTER_PCM_OWN_OK) cluster_pcm_own_report_bump_failure(buf, own_result, base->generation, base->flags, "pending-X retry reservation"); @@ -1634,7 +1694,9 @@ cluster_bufmgr_pcm_x_holder_drain_deferred_nowait(void) } if (LWLockHeldByMe(entry->content_lock)) { - elog(LOG, "could not drain deferred cluster PCM-X holder while content lock is held: buffer=%d", + elog(LOG, + "could not drain deferred cluster PCM-X holder while content lock is held: " + "buffer=%d", entry->buffer_id); continue; } @@ -1686,24 +1748,20 @@ cluster_bufmgr_pcm_x_holder_drain_deferred(ClusterPcmXHolderLedgerEntry *entry) if (action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT) { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); - cluster_bufmgr_pcm_x_holder_report_failure(result, - GetBufferDescriptor(entry->buffer_id), - "deferred detach"); + cluster_bufmgr_pcm_x_holder_report_failure( + result, GetBufferDescriptor(entry->buffer_id), "deferred detach"); } - cluster_bufmgr_pcm_x_holder_retry_wait( - entry->content_lock, entry->buffer_id, - wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS, NULL); + cluster_bufmgr_pcm_x_holder_retry_wait(entry->content_lock, entry->buffer_id, + wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS, + NULL); wait_index++; - if (wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS == 0) - { + if (wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS == 0) { runtime = cluster_pcm_x_runtime_snapshot(); - if (runtime.state != PCM_X_RUNTIME_ACTIVE - || runtime.master_session_incarnation == 0) - { + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); - cluster_bufmgr_pcm_x_holder_report_failure( - PCM_X_QUEUE_NOT_READY, GetBufferDescriptor(entry->buffer_id), - "deferred detach runtime"); + cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_NOT_READY, + GetBufferDescriptor(entry->buffer_id), + "deferred detach runtime"); } } } @@ -2122,6 +2180,72 @@ cluster_bufmgr_pcm_x_writer_clear(ClusterPcmXWriterLedgerEntry *entry) MemSet(entry, 0, sizeof(*entry)); } +/* Clear the shared grant->content activation fence only for the exact writer + * grant captured by this process-local ledger. Activate calls this while it + * owns content EXCLUSIVE; exceptional cleanup calls it only after the queue + * claim has reached its exact terminal. */ +static ClusterPcmOwnResult +cluster_bufmgr_pcm_x_writer_activation_clear(ClusterPcmXWriterLedgerEntry *entry, + BufferDesc *buf, + ClusterPcmOwnSnapshot *live_out) +{ + ClusterPcmOwnSnapshot live; + ClusterPcmOwnResult result; + uint32 buf_state; + + if (entry == NULL || buf == NULL || !entry->activation_fence_armed) + return CLUSTER_PCM_OWN_INVALID; + buf_state = LockBufHdr(buf); + cluster_pcm_own_snapshot_locked(buf, &live); + if (live_out != NULL) + *live_out = live; + if (!cluster_pcm_x_writer_grant_snapshot_exact(&entry->claim, &entry->granted, &live) + || cluster_pcm_own_writer_activation_token_get(buf->buf_id) + != entry->granted.reservation_token) + result = CLUSTER_PCM_OWN_STALE; + else + result = cluster_pcm_own_writer_activation_clear_exact( + buf->buf_id, entry->granted.generation, entry->granted.reservation_token); + UnlockBufHdr(buf, buf_state); + cluster_pcm_own_activation_diag_emit("writer-activation-clear", buf->buf_id, &live, result); + if (result == CLUSTER_PCM_OWN_OK) + entry->activation_fence_armed = false; + return result; +} + +static bool +cluster_bufmgr_pcm_x_writer_finish_claim_cleanup(ClusterPcmXWriterLedgerEntry *entry, + BufferDesc *buf, const char *context) +{ + ClusterPcmOwnResult own_result; + + if (entry == NULL || buf == NULL || !entry->claim_cleanup_complete) + return false; + if (!entry->grant_snapshot_exact) { + entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; + cluster_pcm_x_runtime_fail_closed(); + elog(LOG, "preserving cluster PCM-X writer with unproven grant snapshot: " + "context=%s buffer=%d", context, entry->buffer_id); + return false; + } + if (entry->activation_fence_armed) { + own_result = cluster_bufmgr_pcm_x_writer_activation_clear(entry, buf, NULL); + if (own_result != CLUSTER_PCM_OWN_OK) { + entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; + cluster_pcm_x_runtime_fail_closed(); + elog(LOG, + "could not clear exact cluster PCM-X writer activation fence: " + "context=%s buffer=%d generation=%llu token=%llu result=%d", + context, entry->buffer_id, + (unsigned long long)entry->granted.generation, + (unsigned long long)entry->granted.reservation_token, (int)own_result); + return false; + } + } + cluster_bufmgr_pcm_x_writer_clear(entry); + return true; +} + static void cluster_bufmgr_pcm_x_writer_release(ClusterPcmXWriterLedgerEntry *entry); static void @@ -2142,9 +2266,15 @@ cluster_bufmgr_pcm_x_writer_drain_deferred_nowait(void) entry->buffer_id); continue; } - result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); - if (result == PCM_X_QUEUE_OK) - cluster_bufmgr_pcm_x_writer_clear(entry); + if (entry->claim_cleanup_complete) + result = PCM_X_QUEUE_OK; + else + result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); + if (result == PCM_X_QUEUE_OK) { + entry->claim_cleanup_complete = true; + (void)cluster_bufmgr_pcm_x_writer_finish_claim_cleanup(entry, + GetBufferDescriptor(entry->buffer_id), "deferred drain"); + } else if (result != PCM_X_QUEUE_GATE_RETRY && result != PCM_X_QUEUE_BUSY) { cluster_pcm_x_runtime_fail_closed(); elog(LOG, "could not drain deferred exact cluster PCM-X writer: buffer=%d result=%d", @@ -2190,6 +2320,9 @@ cluster_bufmgr_pcm_x_writer_prepare(BufferDesc *buf, PcmLockMode mode, bool *bar entry->content_lock = content_lock; entry->phase = PCM_X_WRITER_LEDGER_HANDOFF; entry->claim_handed_off = false; + entry->claim_cleanup_complete = false; + entry->grant_snapshot_exact = false; + entry->activation_fence_armed = false; buf_state = LockBufHdr(buf); cluster_bufmgr_pcm_direct_init_snapshot_locked(buf, buf_state, false, &observed); @@ -2234,9 +2367,37 @@ cluster_bufmgr_pcm_x_writer_prepare(BufferDesc *buf, PcmLockMode mode, bool *bar own_result = cluster_bufmgr_pcm_own_snapshot(buf, &granted); if (own_result != CLUSTER_PCM_OWN_OK || !cluster_pcm_x_writer_grant_snapshot_exact(&entry->claim, &granted, &granted)) { + ereport(LOG, + (errmsg("cluster PCM-X writer grant snapshot mismatch"), + errdetail("buffer=%d own_result=%d claim_base=%llu claim_grant_base=%llu " + "claim_generation=%llu claim_flags=%u writer_flags=%u " + "claim_role=%u writer_role=%u claim_round=%u writer_round=%u " + "claim_slot=%zu/%llu writer_slot=%zu/%llu live_generation=%llu " + "live_token=%llu live_flags=%u live_state=%u tag_exact=%d", + buf->buf_id, (int)own_result, + (unsigned long long)entry->claim.writer.identity.base_own_generation, + (unsigned long long)entry->claim.grant_base_own_generation, + (unsigned long long)entry->claim.claim_generation, + (unsigned int)entry->claim.flags, + (unsigned int)entry->claim.writer.flags, + (unsigned int)entry->claim.role, + (unsigned int)entry->claim.writer.role, entry->claim.local_round, + entry->claim.writer.local_round, entry->claim.active_slot.slot_index, + (unsigned long long)entry->claim.active_slot.slot_generation, + entry->claim.writer.membership_slot.slot_index, + (unsigned long long)entry->claim.writer.membership_slot.slot_generation, + (unsigned long long)granted.generation, + (unsigned long long)granted.reservation_token, granted.flags, + (unsigned int)granted.pcm_state, + BufferTagsEqual(&granted.tag, &entry->claim.writer.identity.tag) ? 1 : 0))); release_result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); - if (release_result == PCM_X_QUEUE_OK) - cluster_bufmgr_pcm_x_writer_clear(entry); + if (release_result == PCM_X_QUEUE_OK) { + /* The queue claim is terminal, but the ownership grant did not + * match it. Preserve the ledger/fence evidence; guessing a clear + * here could release another grant's activation barrier. */ + entry->claim_cleanup_complete = true; + entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; + } else { /* Keep the exact claim reachable for owner-exit/deferred retry. */ entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; @@ -2249,6 +2410,8 @@ cluster_bufmgr_pcm_x_writer_prepare(BufferDesc *buf, PcmLockMode mode, bool *bar } entry->granted = granted; + entry->grant_snapshot_exact = true; + entry->activation_fence_armed = true; entry->phase = PCM_X_WRITER_LEDGER_ACQUIRING; return entry; } @@ -2267,9 +2430,30 @@ cluster_bufmgr_pcm_x_writer_activate(ClusterPcmXWriterLedgerEntry *entry) || !cluster_bufmgr_pcm_x_writer_entry_exact(entry, buf) || entry->content_lock == NULL || !LWLockHeldByMe(entry->content_lock)) cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_BAD_STATE, buf, "activate phase"); - own_result = cluster_bufmgr_pcm_own_snapshot(buf, &live); + own_result = cluster_bufmgr_pcm_x_writer_activation_clear(entry, buf, &live); if (own_result != CLUSTER_PCM_OWN_OK || !cluster_pcm_x_writer_grant_snapshot_exact(&entry->claim, &entry->granted, &live)) { + ereport(LOG, + (errmsg("cluster PCM-X writer activate ownership mismatch"), + errdetail("buffer=%d own_result=%d claim_base=%llu claim_grant_base=%llu " + "claim_generation=%llu claim_round=%u claim_role=%u claim_slot=%zu/%llu " + "granted_generation=%llu granted_token=%llu granted_flags=%u " + "granted_state=%u live_generation=%llu live_token=%llu live_flags=%u " + "live_state=%u tag_exact=%d", + buf->buf_id, (int)own_result, + (unsigned long long)entry->claim.writer.identity.base_own_generation, + (unsigned long long)entry->claim.grant_base_own_generation, + (unsigned long long)entry->claim.claim_generation, + entry->claim.local_round, (unsigned int)entry->claim.role, + entry->claim.active_slot.slot_index, + (unsigned long long)entry->claim.active_slot.slot_generation, + (unsigned long long)entry->granted.generation, + (unsigned long long)entry->granted.reservation_token, + entry->granted.flags, (unsigned int)entry->granted.pcm_state, + (unsigned long long)live.generation, + (unsigned long long)live.reservation_token, live.flags, + (unsigned int)live.pcm_state, + BufferTagsEqual(&live.tag, &entry->granted.tag) ? 1 : 0))); cluster_pcm_x_runtime_fail_closed(); cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_CORRUPT, buf, "activate ownership"); } @@ -2348,8 +2532,11 @@ cluster_bufmgr_pcm_x_writer_abort_acquiring(ClusterPcmXWriterLedgerEntry *entry) if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)) return; result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); - if (result == PCM_X_QUEUE_OK) - cluster_bufmgr_pcm_x_writer_clear(entry); + if (result == PCM_X_QUEUE_OK) { + entry->claim_cleanup_complete = true; + (void)cluster_bufmgr_pcm_x_writer_finish_claim_cleanup( + entry, GetBufferDescriptor(entry->buffer_id), "abort acquiring"); + } else if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BUSY) entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; else { @@ -2378,9 +2565,15 @@ cluster_bufmgr_pcm_x_writer_exception_cleanup_all(void) } if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)) continue; - result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); - if (result == PCM_X_QUEUE_OK) - cluster_bufmgr_pcm_x_writer_clear(entry); + if (entry->claim_cleanup_complete) + result = PCM_X_QUEUE_OK; + else + result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); + if (result == PCM_X_QUEUE_OK) { + entry->claim_cleanup_complete = true; + (void)cluster_bufmgr_pcm_x_writer_finish_claim_cleanup( + entry, GetBufferDescriptor(entry->buffer_id), "exception cleanup"); + } else if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BUSY) entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; else { @@ -2436,10 +2629,16 @@ cluster_bufmgr_pcm_x_writer_owner_exit_drain_once(bool runtime_active) entry->buffer_id, (int)entry->phase); continue; } - result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); + if (entry->claim_cleanup_complete) + result = PCM_X_QUEUE_OK; + else + result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); action = cluster_pcm_x_owner_exit_action(result, false, runtime_active); - if (action == CLUSTER_PCM_X_OWNER_EXIT_COMPLETE) - cluster_bufmgr_pcm_x_writer_clear(entry); + if (action == CLUSTER_PCM_X_OWNER_EXIT_COMPLETE) { + entry->claim_cleanup_complete = true; + if (!cluster_bufmgr_pcm_x_writer_finish_claim_cleanup(entry, buf, "owner exit")) + retry = runtime_active; + } else if (action == CLUSTER_PCM_X_OWNER_EXIT_RETRY) { entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; retry = true; @@ -3077,16 +3276,10 @@ ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref) * NOTE: what we check here is that *this* backend holds a pin on * the buffer. We do not care whether some other backend does. */ -#define BufferIsPinned(bufnum) \ -( \ - !BufferIsValid(bufnum) ? \ - false \ - : \ - BufferIsLocal(bufnum) ? \ - (LocalRefCount[-(bufnum) - 1] > 0) \ - : \ - (GetPrivateRefCount(bufnum) > 0) \ -) +#define BufferIsPinned(bufnum) \ + (!BufferIsValid(bufnum) ? false \ + : BufferIsLocal(bufnum) ? (LocalRefCount[-(bufnum) - 1] > 0) \ + : (GetPrivateRefCount(bufnum) > 0)) static Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, @@ -4317,7 +4510,7 @@ InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, } old_pcm_mode = eviction_capture.pcm_state; release_pcm_holder = cluster_pcm_is_active() - && cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(oldTag)) + && cluster_pcm_x_buffer_tag_tracked(oldTag, cluster_shared_catalog) && (old_pcm_mode == (uint8) PCM_LOCK_MODE_S || old_pcm_mode == (uint8) PCM_LOCK_MODE_X); #endif @@ -4574,7 +4767,7 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) * before BufTableDelete completes the eviction. */ if (cluster_pcm_is_active() - && cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(&tag)) + && cluster_pcm_x_buffer_tag_tracked(&tag, cluster_shared_catalog) && buf_hdr->pcm_state != (uint8) PCM_STATE_N) { PcmLockMode old_mode = (PcmLockMode) buf_hdr->pcm_state; @@ -4907,11 +5100,11 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ClusterExtendEngage engage = cluster_extend_liveness_engage(true); if (engage == CLUSTER_EXTEND_ENGAGE_FAIL_CLOSED) - ereport(ERROR, - (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), - errmsg("could not acquire the cluster relation-extend lock for \"%s\"", - RelationGetRelationName(bmr.rel)), - errdetail("The cluster coordination substrate is not ready and an alive peer could not be ruled out."))); + ereport(ERROR, (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), + errmsg("could not acquire the cluster relation-extend lock for \"%s\"", + RelationGetRelationName(bmr.rel)), + errdetail("The cluster coordination substrate is not ready and an " + "alive peer could not be ruled out."))); if (engage == CLUSTER_EXTEND_ENGAGE_COORDINATE) { @@ -4923,11 +5116,12 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, */ hwc = cluster_hw_classify_persistence(bmr.rel->rd_rel->relpersistence, true); if (hwc == CLUSTER_HW_FAIL_CLOSED) - ereport(ERROR, - (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), - errmsg("unlogged relation \"%s\" cannot be safely extended in a multi-node cluster", - RelationGetRelationName(bmr.rel)), - errhint("Unlogged relations have no WAL authority to coordinate cross-node extension."))); + ereport(ERROR, (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), + errmsg("unlogged relation \"%s\" cannot be safely extended in a " + "multi-node cluster", + RelationGetRelationName(bmr.rel)), + errhint("Unlogged relations have no WAL authority to coordinate " + "cross-node extension."))); } /* @@ -5086,11 +5280,11 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, StrategyFreeBuffer(buf_hdr); UnpinBuffer(buf_hdr); } - ereport(ERROR, - (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), - errmsg("cluster relation-extend authority unavailable for \"%s\"", - RelationGetRelationName(bmr.rel)), - errhint("The HW_ALLOC round trip to the resource master could not be proven; retry."))); + ereport(ERROR, (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), + errmsg("cluster relation-extend authority unavailable for \"%s\"", + RelationGetRelationName(bmr.rel)), + errhint("The HW_ALLOC round trip to the resource master could not be " + "proven; retry."))); } if (hw_granted < extend_by) @@ -5230,9 +5424,10 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ereport(ERROR, (errmsg("unexpected data beyond EOF in block %u of relation %s", existing_hdr->tag.blockNum, relpath(bmr.smgr->smgr_rlocator, fork)), - errhint("This has been seen to occur with buggy kernels; consider updating your system."))); + errhint("This has been seen to occur with buggy kernels; consider " + "updating your system."))); - /* + /* * We *must* do smgr[zero]extend before succeeding, else the page * will not be reserved by the kernel, and the next P_NEW call * will decide to return the same page. Clear the BM_VALID bit, @@ -6307,11 +6502,11 @@ BgBufferSync(WritebackContext *wb_context) PendingBgWriterStats.buf_written_clean += num_written; #ifdef BGW_DEBUG - elog(DEBUG1, "bgwriter: recent_alloc=%u smoothed=%.2f delta=%ld ahead=%d density=%.2f reusable_est=%d upcoming_est=%d scanned=%d wrote=%d reusable=%d", - recent_alloc, smoothed_alloc, strategy_delta, bufs_ahead, - smoothed_density, reusable_buffers_est, upcoming_alloc_est, - bufs_to_lap - num_to_scan, - num_written, + elog(DEBUG1, + "bgwriter: recent_alloc=%u smoothed=%.2f delta=%ld ahead=%d density=%.2f reusable_est=%d " + "upcoming_est=%d scanned=%d wrote=%d reusable=%d", + recent_alloc, smoothed_alloc, strategy_delta, bufs_ahead, smoothed_density, + reusable_buffers_est, upcoming_alloc_est, bufs_to_lap - num_to_scan, num_written, reusable_buffers - reusable_buffers_est); #endif @@ -6809,6 +7004,19 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, if (!StartBufferIO(buf, false)) return; +#ifdef USE_PGRAC_CLUSTER + /* The retain-finish caller catches and absorbs this ERROR at the DATA + * worker boundary. Remember the exact ResourceOwner-tracked BufferIO so + * that caller can abort it before releasing its raw pin. Ordinary + * FlushBuffer errors keep PostgreSQL's transaction-abort cleanup. */ + if (cluster_pcm_x_finish_retain_flush_active) + { + Assert(!cluster_pcm_x_finish_retain_flush_io_active); + Assert(!cluster_pcm_x_finish_retain_flush_error_context_pushed); + cluster_pcm_x_finish_retain_flush_io_active = true; + } +#endif + #ifdef USE_PGRAC_CLUSTER /* @@ -6819,6 +7027,8 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, */ if (cluster_smart_fusion && cluster_sf_dep_buffer_flush_blocked(buf)) { TerminateBufferIO(buf, false, 0); + if (cluster_pcm_x_finish_retain_flush_io_active) + cluster_pcm_x_finish_retain_flush_io_active = false; return; } #endif @@ -6828,6 +7038,13 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, errcallback.arg = (void *) buf; errcallback.previous = error_context_stack; error_context_stack = &errcallback; +#ifdef USE_PGRAC_CLUSTER + if (cluster_pcm_x_finish_retain_flush_io_active) + { + cluster_pcm_x_finish_retain_flush_error_context_previous = errcallback.previous; + cluster_pcm_x_finish_retain_flush_error_context_pushed = true; + } +#endif /* Find smgr relation for buffer */ if (reln == NULL) @@ -6909,6 +7126,20 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, io_start = pgstat_prepare_io_time(); +#ifdef USE_PGRAC_CLUSTER + /* This dynamic scope is entered only by + * cluster_bufmgr_pcm_own_finish_revoke_retain after exact snapshot + * revalidation under caller pin + content EXCLUSIVE. Copy/materialize + * FlushBuffer calls cannot consume this one-shot. */ + if (cluster_pcm_x_finish_retain_flush_active + && cluster_pcm_own_flags_get(buf->buf_id) == PCM_OWN_FLAG_REVOKING) { + CLUSTER_INJECTION_POINT("cluster-pcm-x-retain-flush-error"); + if (cluster_injection_should_skip("cluster-pcm-x-retain-flush-error")) + ereport(ERROR, (errcode(ERRCODE_IO_ERROR), + errmsg("injected PCM-X retained-image FlushBuffer failure"))); + } +#endif + /* * bufToWrite is either the shared buffer or a copy, as appropriate. */ @@ -6967,6 +7198,10 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, * end the BM_IO_IN_PROGRESS state. */ TerminateBufferIO(buf, true, 0); +#ifdef USE_PGRAC_CLUSTER + if (cluster_pcm_x_finish_retain_flush_io_active) + cluster_pcm_x_finish_retain_flush_io_active = false; +#endif TRACE_POSTGRESQL_BUFFER_FLUSH_DONE(BufTagGetForkNum(&buf->tag), buf->tag.blockNum, @@ -6976,6 +7211,13 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, /* Pop the error context stack */ error_context_stack = errcallback.previous; +#ifdef USE_PGRAC_CLUSTER + if (cluster_pcm_x_finish_retain_flush_error_context_pushed) + { + cluster_pcm_x_finish_retain_flush_error_context_pushed = false; + cluster_pcm_x_finish_retain_flush_error_context_previous = NULL; + } +#endif } /* @@ -8697,8 +8939,8 @@ LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) "cluster-pcm-grant-finalize-deliver-invalidate")) { if (cluster_gcs_block_test_deliver_self_invalidate(buf->tag)) - elog(WARNING, - "cluster W3 delivery shim: synthetic INVALIDATE was ACKed instead of parked (GRANT_PENDING not honored)"); + elog(WARNING, "cluster W3 delivery shim: synthetic INVALIDATE was ACKed " + "instead of parked (GRANT_PENDING not honored)"); } } @@ -10009,15 +10251,43 @@ cluster_bufmgr_read_storage_scn_for_gcs(BufferTag tag, SCN *out_page_scn) return true; } +const char * +cluster_bufmgr_gcs_copy_refusal_name(ClusterBufmgrGcsCopyRefusal refusal) +{ + switch (refusal) { + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE: + return "none"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INVALID_ARGUMENT: + return "invalid_argument"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT: + return "not_resident"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID: + return "current_invalid"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST: + return "buffer_content_conditional_1"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_SECOND: + return "buffer_content_conditional_2"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY: + return "ownership_revoke_busy"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_HC89_LSN_DRIFT: + return "hc89_lsn_drift"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_SMART_FUSION_UNCLASSIFIED: + return "smart_fusion_unclassified"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INJECTED_EVICT: + return "injected_evict"; + } + return "unknown"; +} + /* * Copy the 8KB block bytes for `tag` into *dst, flushing WAL up to the * page's LSN before reading the bytes (HC82 I-WAL-before-ship), then making * any dirty source current in shared storage before it may be retired. Sets * *out_page_lsn to the page LSN observed at the second-stable revalidation. * - * Returns false on: - * - Buffer no longer in pool (evicted between probe and pin) - * - HC89 revalidation single-retry exhausted + * Returns false on a non-resident or non-current image, either conditional + * content-lock refusal, or HC89 revalidation single-retry exhaustion. When + * requested, *out_refusal identifies that exact terminal stage. * * Returns true with *dst populated and *out_page_lsn set otherwise. * @@ -10031,61 +10301,76 @@ cluster_bufmgr_read_storage_scn_for_gcs(BufferTag tag, SCN *out_page_scn) * before return. */ bool -cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char *dst) +cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char *dst, + ClusterBufmgrGcsCopyRefusal *out_refusal) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - LWLock *content_lock; - XLogRecPtr first_lsn; - XLogRecPtr second_lsn; - int retries; - bool stable; - bool needs_flush; - bool storage_current; - Page page; + LWLock *content_lock; + XLogRecPtr first_lsn; + XLogRecPtr second_lsn; + int retries; + bool stable; + bool needs_flush; + bool storage_current; + Page page; + volatile bool content_locked = false; + volatile bool caller_pinned = false; + + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE; Assert(dst != NULL); Assert(out_page_lsn != NULL); + if (dst == NULL || out_page_lsn == NULL) { + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INVALID_ARGUMENT; + return false; + } hashcode = BufTableHashCode(&tag); partition_lock = BufMappingPartitionLock(hashcode); LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT; return false; } buf = GetBufferDescriptor(buf_id); /* Partition lock keeps the buffer from being recycled before we raw-pin. */ { - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf); /* Re-verify tag under header lock to defend against tag-rewrite * races between the partition-lock-protected lookup and the pin. */ - if (!BufferTagsEqual(&buf->tag, &tag)) - { + if (!BufferTagsEqual(&buf->tag, &tag)) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT; return false; } - if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) - { + if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; return false; } cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); + caller_pinned = true; } LWLockRelease(partition_lock); content_lock = BufferDescriptorGetContentLock(buf); - page = (Page) BufHdrGetBlock(buf); + page = (Page)BufHdrGetBlock(buf); /* * HC89: revalidation loop with single retry budget. retries = 0 — first @@ -10093,121 +10378,159 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char * fail-closed */ stable = false; - for (retries = 0; retries < 2; retries++) + PG_TRY(); { - /* Read page_lsn under content_lock SHARED. Never park a GCS DATA + for (retries = 0; retries < 2; retries++) { + /* Read page_lsn under content_lock SHARED. Never park a GCS DATA * worker behind a backend that may itself be waiting for this worker. */ - if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) - break; - first_lsn = PageGetLSN(page); - LWLockRelease(content_lock); + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) { + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST; + break; + } + content_locked = true; + first_lsn = PageGetLSN(page); + LWLockRelease(content_lock); + content_locked = false; - /* + /* * HC82: flush WAL up to the page LSN before shipping the bytes. Use * XLogFlush(page_lsn) specifically — NOT XLogFlush(insert pointer) * which would be correct but over-flushes and doesn't express the * "flush this page before ship" safety contract. */ #ifdef USE_CLUSTER_UNIT - if (cluster_gcs_block_test_xlog_flush_hook != NULL) - cluster_gcs_block_test_xlog_flush_hook((uint64) first_lsn); + if (cluster_gcs_block_test_xlog_flush_hook != NULL) + cluster_gcs_block_test_xlog_flush_hook((uint64)first_lsn); #endif - if (!XLogRecPtrIsInvalid(first_lsn)) - XLogFlush(cluster_gcs_clamp_ship_flush_lsn(first_lsn)); + if (!XLogRecPtrIsInvalid(first_lsn)) + XLogFlush(cluster_gcs_clamp_ship_flush_lsn(first_lsn)); - /* + /* * Reacquire content_lock SHARED and revalidate that the page LSN has * not advanced past first_lsn AND the buffer tag still matches. * Either signals concurrent mutation that would break HC82's "ship * the bytes that I just flushed WAL for" contract. */ - if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) - break; - { - uint32 buf_state = LockBufHdr(buf); - bool current = BufferTagsEqual(&buf->tag, &tag) - && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) { + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_SECOND; + break; + } + content_locked = true; + { + uint32 buf_state = LockBufHdr(buf); + bool tag_matches = BufferTagsEqual(&buf->tag, &tag); + bool current = tag_matches + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); - needs_flush = current - && (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | - BM_CHECKPOINT_NEEDED)) != 0; - storage_current = current && (buf_state & BM_IO_ERROR) == 0; + needs_flush + = current + && (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0; + storage_current = current && (buf_state & BM_IO_ERROR) == 0; - UnlockBufHdr(buf, buf_state); - if (!storage_current) - { - LWLockRelease(content_lock); - break; + UnlockBufHdr(buf, buf_state); + if (!storage_current) { + if (out_refusal != NULL) + *out_refusal = tag_matches + ? CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID + : CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT; + LWLockRelease(content_lock); + content_locked = false; + break; + } } - } - second_lsn = PageGetLSN(page); + second_lsn = PageGetLSN(page); #ifdef USE_CLUSTER_UNIT - /* + /* * Spec L25/L26 hook: if the test injection returns N > 0 we mimic N * consecutive LSN drift events. retries 0 + drift available means * second_lsn != first_lsn synthetically. */ - if (cluster_gcs_block_test_lsn_drift_hook != NULL) - { - int drift_remaining = cluster_gcs_block_test_lsn_drift_hook(); + if (cluster_gcs_block_test_lsn_drift_hook != NULL) { + int drift_remaining = cluster_gcs_block_test_lsn_drift_hook(); - if (drift_remaining > retries) - second_lsn = first_lsn + 1; /* synthetic mismatch */ - } + if (drift_remaining > retries) + second_lsn = first_lsn + 1; /* synthetic mismatch */ + } #endif - if (first_lsn == second_lsn) - { - uint32 buf_state; + if (first_lsn == second_lsn) { + uint32 buf_state; - /* A queue handoff may retire this descriptor immediately after + /* A queue handoff may retire this descriptor immediately after * copying it. Make the shared-storage fallback at least as current * as the shipped image before publishing that handoff watermark. */ - if (needs_flush) - FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); - - buf_state = LockBufHdr(buf); - storage_current = BufferTagsEqual(&buf->tag, &tag) - && cluster_bufmgr_pcm_current_image_locked(buf, buf_state) - && PageGetLSN(page) == second_lsn - && (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | - BM_IO_ERROR | BM_IO_IN_PROGRESS)) == 0; - UnlockBufHdr(buf, buf_state); - if (!storage_current) - { - LWLockRelease(content_lock); - continue; - } + if (needs_flush) + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + + buf_state = LockBufHdr(buf); + storage_current = BufferTagsEqual(&buf->tag, &tag) + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state) + && PageGetLSN(page) == second_lsn + && (buf_state + & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED + | BM_IO_ERROR | BM_IO_IN_PROGRESS)) + == 0; + UnlockBufHdr(buf, buf_state); + if (!storage_current) { + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + LWLockRelease(content_lock); + content_locked = false; + continue; + } - memcpy(dst, page, BLCKSZ); + memcpy(dst, page, BLCKSZ); - /* Hint-bit dirties may occur under a shared content lock. Do not + /* Hint-bit dirties may occur under a shared content lock. Do not * certify a copy if one raced the memcpy after the first clean check. */ - buf_state = LockBufHdr(buf); - storage_current = BufferTagsEqual(&buf->tag, &tag) - && cluster_bufmgr_pcm_current_image_locked(buf, buf_state) - && PageGetLSN(page) == second_lsn - && (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | - BM_IO_ERROR | BM_IO_IN_PROGRESS)) == 0; - UnlockBufHdr(buf, buf_state); - if (!storage_current) - { + buf_state = LockBufHdr(buf); + storage_current = BufferTagsEqual(&buf->tag, &tag) + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state) + && PageGetLSN(page) == second_lsn + && (buf_state + & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED + | BM_IO_ERROR | BM_IO_IN_PROGRESS)) + == 0; + UnlockBufHdr(buf, buf_state); + if (!storage_current) { + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + LWLockRelease(content_lock); + content_locked = false; + continue; + } + + *out_page_lsn = second_lsn; LWLockRelease(content_lock); - continue; + content_locked = false; + stable = true; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE; + break; } - - *out_page_lsn = second_lsn; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_HC89_LSN_DRIFT; LWLockRelease(content_lock); - stable = true; - break; + content_locked = false; + /* fall through: retry once if budget remains */ } - LWLockRelease(content_lock); - /* fall through: retry once if budget remains */ } + PG_CATCH(); + { + if (content_locked && LWLockHeldByMe(content_lock)) + LWLockRelease(content_lock); + if (caller_pinned) + cluster_bufmgr_unpin_for_gcs(buf); + PG_RE_THROW(); + } + PG_END_TRY(); - cluster_bufmgr_unpin_for_gcs(buf); + if (caller_pinned) + cluster_bufmgr_unpin_for_gcs(buf); return stable; } @@ -10429,17 +10752,36 @@ cluster_bufmgr_finish_direct_land_target_for_gcs(BufferDesc *buf, bool valid, * harmless). Runs in the LMON IC-dispatch context; the FlushBuffer * call mirrors the checkpointer contract (pin + content lock held). * ======================================================================== */ -bool -cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) +ClusterBufmgrGcsDowngradeOutcome +cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image( + BufferTag tag, XLogRecPtr *out_page_lsn, char *dst, + ClusterBufmgrGcsCopyRefusal *out_refusal) { uint32 hashcode; LWLock *partition_lock; int buf_id; BufferDesc *buf; LWLock *content_lock; + ClusterPcmOwnSnapshot current; + ClusterPcmOwnSnapshot revoking; + ClusterPcmOwnSnapshot shared; + ClusterPcmOwnResult own_result; + Page page; + XLogRecPtr first_lsn; + XLogRecPtr second_lsn; uint32 buf_state; bool dirty; + if (out_page_lsn != NULL) + *out_page_lsn = InvalidXLogRecPtr; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE; + if (out_page_lsn == NULL || dst == NULL) { + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INVALID_ARGUMENT; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + hashcode = BufTableHashCode(&tag); partition_lock = BufMappingPartitionLock(hashcode); @@ -10448,7 +10790,9 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) if (buf_id < 0) { LWLockRelease(partition_lock); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } buf = GetBufferDescriptor(buf_id); @@ -10457,7 +10801,9 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); LWLockRelease(partition_lock); @@ -10469,13 +10815,17 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) if (!LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) { cluster_bufmgr_unpin_for_gcs(buf); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } /* @@ -10490,34 +10840,124 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + own_result = cluster_bufmgr_pcm_own_snapshot(buf, ¤t); + if (own_result != CLUSTER_PCM_OWN_OK || !BufferTagsEqual(¤t.tag, &tag) + || current.pcm_state != (uint8)PCM_STATE_X || current.flags != 0) { + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + own_result = cluster_bufmgr_pcm_own_begin_x_revoke(buf, ¤t, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) { + if (own_result != CLUSTER_PCM_OWN_BUSY && own_result != CLUSTER_PCM_OWN_STALE) + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } buf_state = LockBufHdr(buf); dirty = (buf_state & BM_DIRTY) != 0; UnlockBufHdr(buf, buf_state); - if (dirty) - FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); - - if (!cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_X_TO_S_DOWNGRADE, - cluster_node_id)) + PG_TRY(); { - /* Master refused (state moved under us) — leave local X untouched. */ + if (dirty) + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + } + PG_CATCH(); + { + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return false; + PG_RE_THROW(); } + PG_END_TRY(); - /* PGRAC ownership-generation wave: the X->S downgrade is a committed - * ownership transition -- set pcm_state and bump the generation atomically - * (header spinlock) so a cached-X writer racing the content-lock window - * detects the revoke even across an X->S->X ABA. */ - cluster_pcm_own_transition(buf, (uint8) PCM_STATE_S, 0, 0); - - LWLockRelease(content_lock); + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0 + || (buf_state & (BM_DIRTY | BM_IO_IN_PROGRESS)) != 0 + || buf->pcm_state != (uint8)PCM_STATE_X + || cluster_pcm_own_gen_get(buf->buf_id) != revoking.generation + || cluster_pcm_own_reservation_token_get(buf->buf_id) != revoking.reservation_token + || cluster_pcm_own_flags_get(buf->buf_id) != PCM_OWN_FLAG_REVOKING + || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { + UnlockBufHdr(buf, buf_state); + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + UnlockBufHdr(buf, buf_state); + page = (Page)BufHdrGetBlock(buf); + first_lsn = PageGetLSN(page); + memcpy(dst, page, BLCKSZ); + second_lsn = PageGetLSN(page); + if (first_lsn != second_lsn) { + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_HC89_LSN_DRIFT; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + + if (!cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_X_TO_S_DOWNGRADE, + cluster_node_id)) + { + /* Master refused (state moved under us) — leave local X untouched. */ + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + + own_result = cluster_bufmgr_pcm_own_finish_x_to_s_downgrade(buf, &revoking, &shared); + if (own_result != CLUSTER_PCM_OWN_OK || revoking.generation == UINT64_MAX + || shared.generation != revoking.generation + 1 + || shared.reservation_token != revoking.reservation_token || shared.flags != 0 + || shared.pcm_state != (uint8)PCM_STATE_S || !BufferTagsEqual(&shared.tag, &tag)) { + /* The local master transition already committed. Do not invent a + * rollback or expose an S grant from a divergent local tuple. */ + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + return CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY; + } + *out_page_lsn = second_lsn; + + LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return true; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED; +} + +bool +cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) +{ + PGAlignedBlock block; + XLogRecPtr page_lsn; + + return cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image( + tag, &page_lsn, block.data, NULL) + == CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED; } /* ======================================================================== @@ -10657,19 +11097,93 @@ cluster_bufmgr_unlock_resident_stamp(Buffer buffer) * requester's registration try-ACK fails closed to one-shot semantics — * no path serves a copy the master does not track (Rule 8.A). * ======================================================================== */ -bool -cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) +static ClusterPcmOwnResult +cluster_bufmgr_pcm_own_finish_x_to_s_downgrade( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, + ClusterPcmOwnSnapshot *out_shared) +{ + ClusterPcmOwnResult live_result; + ClusterPcmOwnResult result; + uint64 committed_generation = 0; + uint64 live_token; + uint32 flags; + uint32 buf_state; + + if (buf == NULL || expected_revoking == NULL || out_shared == NULL) + return CLUSTER_PCM_OWN_INVALID; + memset(out_shared, 0, sizeof(*out_shared)); + if (!LWLockHeldByMe(BufferDescriptorGetContentLock(buf)) + || expected_revoking->pcm_state != (uint8)PCM_STATE_X + || expected_revoking->flags != PCM_OWN_FLAG_REVOKING + || expected_revoking->reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, &expected_revoking->tag) + || (buf_state & BM_VALID) == 0 + || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation + || buf->pcm_state != (uint8)PCM_STATE_X) + result = CLUSTER_PCM_OWN_STALE; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + result = CLUSTER_PCM_OWN_CORRUPT; + else { + live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if (flags == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (flags != PCM_OWN_FLAG_REVOKING) + result = CLUSTER_PCM_OWN_BUSY; + else if (live_token != expected_revoking->reservation_token) + result = CLUSTER_PCM_OWN_STALE; + else { + result = cluster_pcm_own_revoke_commit_exact( + buf->buf_id, expected_revoking->generation, + expected_revoking->reservation_token, &committed_generation); + if (result == CLUSTER_PCM_OWN_OK) { + buf->buffer_type = (uint8)BUF_TYPE_SCUR; + buf->pcm_state = (uint8)PCM_STATE_S; + cluster_pcm_own_snapshot_locked(buf, out_shared); + } + } + } + UnlockBufHdr(buf, buf_state); + return result; +} + +ClusterBufmgrGcsDowngradeOutcome +cluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image( + BufferTag tag, int32 master_node, XLogRecPtr *out_page_lsn, char *dst, + ClusterBufmgrGcsCopyRefusal *out_refusal) { uint32 hashcode; LWLock *partition_lock; int buf_id; BufferDesc *buf; LWLock *content_lock; + ClusterPcmOwnSnapshot current; + ClusterPcmOwnSnapshot revoking; + ClusterPcmOwnSnapshot shared; + ClusterPcmOwnResult own_result; + Page page; + XLogRecPtr first_lsn; + XLogRecPtr second_lsn; uint32 buf_state; bool dirty; + bool notify_handed_off = false; - if (master_node < 0 || master_node == cluster_node_id) - return false; + if (out_page_lsn != NULL) + *out_page_lsn = InvalidXLogRecPtr; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE; + + if (master_node < 0 || master_node == cluster_node_id || out_page_lsn == NULL || dst == NULL) { + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INVALID_ARGUMENT; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } hashcode = BufTableHashCode(&tag); partition_lock = BufMappingPartitionLock(hashcode); @@ -10679,7 +11193,9 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) if (buf_id < 0) { LWLockRelease(partition_lock); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } buf = GetBufferDescriptor(buf_id); @@ -10688,7 +11204,9 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); LWLockRelease(partition_lock); @@ -10700,13 +11218,17 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) if (!LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) { cluster_bufmgr_unpin_for_gcs(buf); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } /* Same re-verify as the local variant: tag, PCM X, quiescent. */ @@ -10716,15 +11238,87 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + + own_result = cluster_bufmgr_pcm_own_snapshot(buf, ¤t); + if (own_result != CLUSTER_PCM_OWN_OK || !BufferTagsEqual(¤t.tag, &tag) + || current.pcm_state != (uint8)PCM_STATE_X || current.flags != 0) { + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + own_result = cluster_bufmgr_pcm_own_begin_x_revoke(buf, ¤t, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) { + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY; + if (own_result != CLUSTER_PCM_OWN_BUSY && own_result != CLUSTER_PCM_OWN_STALE) + cluster_pcm_x_runtime_fail_closed(); + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } buf_state = LockBufHdr(buf); dirty = (buf_state & BM_DIRTY) != 0; UnlockBufHdr(buf, buf_state); - if (dirty) - FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + PG_TRY(); + { + if (dirty) + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + } + PG_CATCH(); + { + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + PG_RE_THROW(); + } + PG_END_TRY(); + + /* Content EXCLUSIVE keeps the page bytes stable. Prove the exact + * REVOKING ownership and clean current-image shape both before and after + * the stack copy, while the notify has not yet crossed the wire boundary. */ + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0 + || (buf_state & (BM_DIRTY | BM_IO_IN_PROGRESS)) != 0 + || buf->pcm_state != (uint8)PCM_STATE_X + || cluster_pcm_own_gen_get(buf->buf_id) != revoking.generation + || cluster_pcm_own_reservation_token_get(buf->buf_id) != revoking.reservation_token + || cluster_pcm_own_flags_get(buf->buf_id) != PCM_OWN_FLAG_REVOKING + || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { + UnlockBufHdr(buf, buf_state); + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + UnlockBufHdr(buf, buf_state); + page = (Page)BufHdrGetBlock(buf); + first_lsn = PageGetLSN(page); + memcpy(dst, page, BLCKSZ); + second_lsn = PageGetLSN(page); + if (first_lsn != second_lsn) { + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_HC89_LSN_DRIFT; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } /* * PGRAC: GCS-race round-4c P1 (yield-notify liveness) — deterministic @@ -10737,24 +11331,50 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) if (cluster_injection_should_skip("cluster-gcs-block-yield-notify-drop")) { /* simulated post-handoff loss: fall through to the S flip */ + notify_handed_off = true; } - else if (!cluster_gcs_send_transition_nowait(tag, PCM_TRANS_X_TO_S_DOWNGRADE, master_node)) + else + notify_handed_off + = cluster_gcs_send_transition_nowait(tag, PCM_TRANS_X_TO_S_DOWNGRADE, master_node); + if (!notify_handed_off) { - /* Notify not handed to transport — keep local X, master unchanged. */ + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return false; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } - /* PGRAC ownership-generation wave: the X->S downgrade is a committed - * ownership transition -- set pcm_state and bump the generation atomically - * (header spinlock) so a cached-X writer racing the content-lock window - * detects the revoke even across an X->S->X ABA. */ - cluster_pcm_own_transition(buf, (uint8) PCM_STATE_S, 0, 0); + own_result = cluster_bufmgr_pcm_own_finish_x_to_s_downgrade(buf, &revoking, &shared); + if (own_result != CLUSTER_PCM_OWN_OK + || revoking.generation == UINT64_MAX + || shared.generation != revoking.generation + 1 + || shared.reservation_token != revoking.reservation_token || shared.flags != 0 + || shared.pcm_state != (uint8)PCM_STATE_S || !BufferTagsEqual(&shared.tag, &tag)) { + /* Notify may already be visible to the master. Never fabricate a local + * rollback or S grant after this irreversible boundary. */ + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + return CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY; + } + *out_page_lsn = second_lsn; LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return true; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED; +} + +bool +cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) +{ + PGAlignedBlock block; + XLogRecPtr page_lsn; + + return cluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image( + tag, master_node, &page_lsn, block.data, NULL) + == CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED; } /* ======================================================================== @@ -11038,6 +11658,7 @@ ClusterBufmgrGcsDropResult cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode, XLogRecPtr *out_page_lsn, SCN *out_page_scn) { + ClusterPcmOwnSnapshot activation_diag; uint32 hashcode; LWLock *partition_lock; int buf_id; @@ -11048,6 +11669,8 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode bool was_dirty = false; uint8 saved_pcm_state; uint64 staged_gen; /* PGRAC W2: ownership gen captured at stage-N */ + bool activation_diag_valid = false; + bool invalidate_succeeded; (void) expected_mode; @@ -11114,7 +11737,10 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode LWLock *content_lock = BufferDescriptorGetContentLock(buf); Page page = (Page) BufHdrGetBlock(buf); - LWLockAcquire(content_lock, LW_SHARED); + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) { + cluster_bufmgr_unpin_for_gcs(buf); + return CLUSTER_BUFMGR_GCS_DROP_PINNED; + } page_lsn = PageGetLSN(page); page_scn = ((PageHeader) page)->pd_block_scn; LWLockRelease(content_lock); @@ -11164,10 +11790,16 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode saved_pcm_state = buf->pcm_state; staged_gen = cluster_pcm_own_gen_get(buf->buf_id); /* PGRAC W2 */ buf->pcm_state = (uint8) PCM_STATE_N; + cluster_pcm_own_snapshot_locked(buf, &activation_diag); + activation_diag_valid = activation_diag.writer_activation_token != 0; /* PGRAC: spec-6.12h D-h1 — keep a Past Image instead of dropping. */ - if (cluster_bufmgr_convert_to_pi_locked(buf, buf_state)) + if (cluster_bufmgr_convert_to_pi_locked(buf, buf_state)) { + if (activation_diag_valid) + cluster_pcm_own_activation_diag_emit("invalidate-stage-n-pi", buf->buf_id, + &activation_diag, CLUSTER_PCM_OWN_CORRUPT); return CLUSTER_BUFMGR_GCS_DROP_DROPPED; + } /* * PGRAC: GCS serve-stall round-5 (A2) — bounded drop. A pinner racing @@ -11177,7 +11809,11 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode * its true PCM state. */ cluster_bufmgr_in_gcs_drop = true; /* gates the drop-prepin inject */ - if (!InvalidateBufferTry(buf)) /* releases the header spinlock */ + invalidate_succeeded = InvalidateBufferTry(buf); /* releases the header spinlock */ + if (activation_diag_valid) + cluster_pcm_own_activation_diag_emit("invalidate-stage-n-drop", buf->buf_id, + &activation_diag, CLUSTER_PCM_OWN_CORRUPT); + if (!invalidate_succeeded) { cluster_bufmgr_in_gcs_drop = false; @@ -11521,6 +12157,7 @@ ClusterBufmgrGcsDropResult cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn, XLogRecPtr *out_page_lsn) { + ClusterPcmOwnSnapshot activation_diag; uint32 hashcode; LWLock *partition_lock; int buf_id; @@ -11530,6 +12167,8 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn bool was_dirty = false; uint8 saved_pcm_state; uint64 staged_gen; /* PGRAC W2: ownership gen captured at stage-N */ + bool activation_diag_valid = false; + bool invalidate_succeeded; if (out_page_lsn != NULL) *out_page_lsn = InvalidXLogRecPtr; @@ -11649,18 +12288,28 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn saved_pcm_state = buf->pcm_state; staged_gen = cluster_pcm_own_gen_get(buf->buf_id); /* PGRAC W2 */ buf->pcm_state = (uint8) PCM_STATE_N; + cluster_pcm_own_snapshot_locked(buf, &activation_diag); + activation_diag_valid = activation_diag.writer_activation_token != 0; /* * PGRAC: spec-6.12h D-h1 — keep a Past Image instead of dropping (the * shipped current went to the new holder; our copy becomes the PI). */ - if (cluster_bufmgr_convert_to_pi_locked(buf, buf_state)) + if (cluster_bufmgr_convert_to_pi_locked(buf, buf_state)) { + if (activation_diag_valid) + cluster_pcm_own_activation_diag_emit("drop-no-wire-stage-n-pi", buf->buf_id, + &activation_diag, CLUSTER_PCM_OWN_CORRUPT); return CLUSTER_BUFMGR_GCS_DROP_DROPPED; + } /* PGRAC: GCS serve-stall round-5 (A2) — bounded drop; restore the * residency mode on a raced pin (mirrors the invalidate wrapper). */ cluster_bufmgr_in_gcs_drop = true; /* gates the drop-prepin inject */ - if (!InvalidateBufferTry(buf)) /* releases the header spinlock */ + invalidate_succeeded = InvalidateBufferTry(buf); /* releases the header spinlock */ + if (activation_diag_valid) + cluster_pcm_own_activation_diag_emit("drop-no-wire-stage-n-drop", buf->buf_id, + &activation_diag, CLUSTER_PCM_OWN_CORRUPT); + if (!invalidate_succeeded) { cluster_bufmgr_in_gcs_drop = false; @@ -11768,6 +12417,7 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, uint64 reservation_token = 0; uint32 buf_state; volatile bool content_locked = false; + volatile bool private_pinned = false; if (out_revoking != NULL) memset(out_revoking, 0, sizeof(*out_revoking)); @@ -11807,14 +12457,33 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, * and report BUSY: one flush converges the state and the image pump * retries against a clean page. */ + content_lock = BufferDescriptorGetContentLock(buf); PinBuffer_Locked(buf); /* consumes the buffer header lock */ - LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED); - FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); - LWLockRelease(BufferDescriptorGetContentLock(buf)); - UnpinBuffer(buf); + private_pinned = true; + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) { + UnpinBuffer(buf); + return CLUSTER_PCM_OWN_BUSY; + } + content_locked = true; + PG_TRY(); + { + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + LWLockRelease(content_lock); + content_locked = false; + UnpinBuffer(buf); + private_pinned = false; + } + PG_CATCH(); + { + if (content_locked && LWLockHeldByMe(content_lock)) + LWLockRelease(content_lock); + if (private_pinned) + UnpinBuffer(buf); + PG_RE_THROW(); + } + PG_END_TRY(); return CLUSTER_PCM_OWN_BUSY; - } - else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + } else if ((buf_state & BM_IO_IN_PROGRESS) != 0) result = CLUSTER_PCM_OWN_BUSY; else { result = cluster_pcm_own_reservation_begin_exact(buf->buf_id, expected_n->generation, @@ -11837,8 +12506,7 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, result = CLUSTER_PCM_OWN_CORRUPT; } - if (result == CLUSTER_PCM_OWN_OK) { - LWLockAcquire(content_lock, LW_EXCLUSIVE); + if (result == CLUSTER_PCM_OWN_OK && LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) { content_locked = true; buf_state = LockBufHdr(buf); if (!cluster_pcm_own_snapshot_matches_locked(buf, &live) @@ -11868,7 +12536,8 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, UnlockBufHdr(buf, buf_state); LWLockRelease(content_lock); content_locked = false; - } + } else if (result == CLUSTER_PCM_OWN_OK) + result = CLUSTER_PCM_OWN_BUSY; } PG_CATCH(); { @@ -11961,6 +12630,403 @@ cluster_bufmgr_pcm_own_begin_s_revoke(BufferDesc *buf, return result; } + +typedef enum ClusterPcmOwnSSourceHardFailureReason +{ + CLUSTER_PCM_S_SOURCE_HARD_NONE = 0, + CLUSTER_PCM_S_SOURCE_HARD_INITIAL_CURRENT_IMAGE, + CLUSTER_PCM_S_SOURCE_HARD_INITIAL_IO_ERROR, + CLUSTER_PCM_S_SOURCE_HARD_BEGIN_REVOKE_CORRUPT, + CLUSTER_PCM_S_SOURCE_HARD_STORAGE_READ_ERROR, + CLUSTER_PCM_S_SOURCE_HARD_STORAGE_VERIFY, + CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_CURRENT_IMAGE, + CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_IO_ERROR, + CLUSTER_PCM_S_SOURCE_HARD_NO_COVER, + CLUSTER_PCM_S_SOURCE_HARD_ABORT_FAILURE +} ClusterPcmOwnSSourceHardFailureReason; + +typedef struct ClusterPcmOwnSSourceHardFailureObservation +{ + bool valid; + ClusterPcmOwnSSourceHardFailureReason reason; + ClusterPcmOwnSSourceHardFailureReason cause; + BufferTag tag; + int buffer_id; + uint8 pcm_state; + uint8 buffer_type; + uint32 flags; + uint32 buffer_state; + uint64 generation; + uint64 reservation_token; + uint64 required_scn; + uint64 local_scn; + uint64 storage_scn; + int result; + int abort_result; +} ClusterPcmOwnSSourceHardFailureObservation; + +static const char * +cluster_bufmgr_pcm_own_s_source_hard_failure_reason_name( + ClusterPcmOwnSSourceHardFailureReason reason) +{ + switch (reason) + { + case CLUSTER_PCM_S_SOURCE_HARD_NONE: + return "none"; + case CLUSTER_PCM_S_SOURCE_HARD_INITIAL_CURRENT_IMAGE: + return "initial-current-image"; + case CLUSTER_PCM_S_SOURCE_HARD_INITIAL_IO_ERROR: + return "initial-io-error"; + case CLUSTER_PCM_S_SOURCE_HARD_BEGIN_REVOKE_CORRUPT: + return "begin-revoke-corrupt"; + case CLUSTER_PCM_S_SOURCE_HARD_STORAGE_READ_ERROR: + return "storage-read-error"; + case CLUSTER_PCM_S_SOURCE_HARD_STORAGE_VERIFY: + return "storage-verify"; + case CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_CURRENT_IMAGE: + return "post-lock-current-image"; + case CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_IO_ERROR: + return "post-lock-io-error"; + case CLUSTER_PCM_S_SOURCE_HARD_NO_COVER: + return "no-cover"; + case CLUSTER_PCM_S_SOURCE_HARD_ABORT_FAILURE: + return "abort-failure"; + } + return "unknown"; +} + +/* One record for the first exact hard-failure shape and every later state + * change. The small per-process cache prevents a retrying LMS worker from + * flooding the server log without hiding a different ownership tuple or SCN + * comparison. This is observation only: no ownership or buffer state is + * read or changed here beyond the caller's already coherent samples. */ +static void +cluster_bufmgr_pcm_own_observe_s_source_hard_failure( + ClusterPcmOwnSSourceHardFailureReason reason, + ClusterPcmOwnSSourceHardFailureReason cause, BufferDesc *buf, + const ClusterPcmOwnSnapshot *identity, SCN required_scn, SCN local_scn, + SCN storage_scn, uint32 buffer_state, uint8 buffer_type, + ClusterPcmOwnResult result, ClusterPcmOwnResult abort_result) +{ + static ClusterPcmOwnSSourceHardFailureObservation cache[8]; + static uint32 next_cache_slot = 0; + ClusterPcmOwnSSourceHardFailureObservation obs; + int cache_idx = -1; + int i; + + if (reason == CLUSTER_PCM_S_SOURCE_HARD_NONE || buf == NULL || identity == NULL) + return; + memset(&obs, 0, sizeof(obs)); + obs.valid = true; + obs.reason = reason; + obs.cause = cause; + obs.tag = identity->tag; + obs.buffer_id = buf->buf_id; + obs.pcm_state = identity->pcm_state; + obs.buffer_type = buffer_type; + obs.flags = identity->flags; + obs.buffer_state = buffer_state; + obs.generation = identity->generation; + obs.reservation_token = identity->reservation_token; + obs.required_scn = (uint64)required_scn; + obs.local_scn = (uint64)local_scn; + obs.storage_scn = (uint64)storage_scn; + obs.result = (int)result; + obs.abort_result = (int)abort_result; + + for (i = 0; i < lengthof(cache); i++) + { + if (cache[i].valid && cache[i].reason == obs.reason + && BufferTagsEqual(&cache[i].tag, &obs.tag)) + { + cache_idx = i; + break; + } + } + if (cache_idx >= 0 && memcmp(&cache[cache_idx], &obs, sizeof(obs)) == 0) + return; + if (cache_idx < 0) + { + cache_idx = (int)(next_cache_slot % lengthof(cache)); + next_cache_slot++; + } + cache[cache_idx] = obs; + + elog(LOG, + "cluster PCM S-source hard failure observation: " + "reason=%s cause=%s result=%d abort_result=%d " + "buffer=%d spc=%u db=%u rel=%u fork=%d blk=%u " + "state=%u generation=%llu token=%llu flags=0x%x " + "required_scn=%llu local_scn=%llu storage_scn=%llu " + "buffer_state=0x%x buffer_type=%u", + cluster_bufmgr_pcm_own_s_source_hard_failure_reason_name(obs.reason), + cluster_bufmgr_pcm_own_s_source_hard_failure_reason_name(obs.cause), + obs.result, obs.abort_result, obs.buffer_id, obs.tag.spcOid, obs.tag.dbOid, + obs.tag.relNumber, (int)obs.tag.forkNum, obs.tag.blockNum, obs.pcm_state, + (unsigned long long)obs.generation, + (unsigned long long)obs.reservation_token, obs.flags, + (unsigned long long)obs.required_scn, (unsigned long long)obs.local_scn, + (unsigned long long)obs.storage_scn, obs.buffer_state, + (unsigned int)obs.buffer_type); +} + +/* Freeze an exact S source and materialize the newest safe image visible in + * either its clean resident page or shared storage. REVOKING prevents local + * data writes/reuse while the conditional content-lock hold binds the final + * comparison and optional storage refresh to the same ownership tuple. */ +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_prepare_s_source_image( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_s, SCN required_page_scn, + ClusterPcmOwnSnapshot *out_revoking, char block_data[BLCKSZ], + XLogRecPtr *out_page_lsn, uint64 *out_page_scn, + ClusterPcmOwnSourcePrepareRefusal *out_refusal) +{ + PGIOAlignedBlock scratch; + ClusterPcmOwnResult abort_result; + ClusterPcmOwnResult result; + ClusterPcmOwnSnapshot live; + SMgrRelation reln; + LWLock *content_lock; + Page local_page; + Page storage_page; + SCN local_scn; + SCN storage_scn; + uint32 buf_state; + uint32 observed_buf_state = 0; + uint8 observed_buffer_type = 0; + bool local_covers; + bool storage_covers; + volatile bool content_locked = false; + volatile bool private_pinned = false; + ClusterPcmOwnSSourceHardFailureReason hard_failure_reason = + CLUSTER_PCM_S_SOURCE_HARD_NONE; + + if (out_revoking != NULL) + memset(out_revoking, 0, sizeof(*out_revoking)); + if (block_data != NULL) + memset(block_data, 0, BLCKSZ); + if (out_page_lsn != NULL) + *out_page_lsn = InvalidXLogRecPtr; + if (out_page_scn != NULL) + *out_page_scn = 0; + if (out_refusal != NULL) + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_NONE; + if (buf == NULL || expected_s == NULL || out_revoking == NULL || block_data == NULL + || out_page_lsn == NULL || out_page_scn == NULL || out_refusal == NULL) + return CLUSTER_PCM_OWN_INVALID; + if (expected_s->pcm_state != (uint8)PCM_STATE_S || expected_s->flags != 0) + return CLUSTER_PCM_OWN_INVALID; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + + /* A read can install a commit hint after the checkpoint that made this S + * image storage-consistent. Flush that legal dirt before REVOKING and + * return BUSY after making progress; otherwise the pump would reserve, + * reject the dirt, abort, and repeat forever. This is the same WAL-first + * convergence used by requester-as-source N. */ + ReservePrivateRefCountEntry(); + ResourceOwnerEnlargeBuffers(CurrentResourceOwner); + buf_state = LockBufHdr(buf); + observed_buf_state = buf_state; + observed_buffer_type = buf->buffer_type; + if (!cluster_pcm_own_snapshot_matches_locked(buf, expected_s) || (buf_state & BM_VALID) == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + { + hard_failure_reason = CLUSTER_PCM_S_SOURCE_HARD_INITIAL_CURRENT_IMAGE; + result = CLUSTER_PCM_OWN_CORRUPT; + } + else if ((buf_state & BM_IO_ERROR) != 0) + { + hard_failure_reason = CLUSTER_PCM_S_SOURCE_HARD_INITIAL_IO_ERROR; + result = CLUSTER_PCM_OWN_CORRUPT; + } + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) + { + content_lock = BufferDescriptorGetContentLock(buf); + PinBuffer_Locked(buf); /* consumes the buffer header lock */ + private_pinned = true; + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) { + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_CONTENT_LOCK; + UnpinBuffer(buf); + return CLUSTER_PCM_OWN_BUSY; + } + content_locked = true; + PG_TRY(); + { + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_DIRTY_FLUSHED; + LWLockRelease(content_lock); + content_locked = false; + UnpinBuffer(buf); + private_pinned = false; + } + PG_CATCH(); + { + if (content_locked && LWLockHeldByMe(content_lock)) + LWLockRelease(content_lock); + if (private_pinned) + UnpinBuffer(buf); + PG_RE_THROW(); + } + PG_END_TRY(); + return CLUSTER_PCM_OWN_BUSY; + } + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) { + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_IO_IN_PROGRESS; + result = CLUSTER_PCM_OWN_BUSY; + } + else + result = CLUSTER_PCM_OWN_OK; + UnlockBufHdr(buf, buf_state); + if (result != CLUSTER_PCM_OWN_OK) { + if (result == CLUSTER_PCM_OWN_CORRUPT) + cluster_bufmgr_pcm_own_observe_s_source_hard_failure( + hard_failure_reason, CLUSTER_PCM_S_SOURCE_HARD_NONE, buf, expected_s, + required_page_scn, InvalidScn, InvalidScn, observed_buf_state, + observed_buffer_type, result, CLUSTER_PCM_OWN_INVALID); + return result; + } + + result = cluster_bufmgr_pcm_own_begin_s_revoke(buf, expected_s, &live); + if (result != CLUSTER_PCM_OWN_OK) { + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_BEGIN_REVOKE; + if (result == CLUSTER_PCM_OWN_CORRUPT) + cluster_bufmgr_pcm_own_observe_s_source_hard_failure( + CLUSTER_PCM_S_SOURCE_HARD_BEGIN_REVOKE_CORRUPT, + CLUSTER_PCM_S_SOURCE_HARD_NONE, buf, expected_s, required_page_scn, + InvalidScn, InvalidScn, observed_buf_state, observed_buffer_type, result, + CLUSTER_PCM_OWN_INVALID); + return result; + } + content_lock = BufferDescriptorGetContentLock(buf); + storage_page = (Page)scratch.data; + local_scn = InvalidScn; + storage_scn = InvalidScn; + + PG_TRY(); + { + reln = smgropen(BufTagGetRelFileLocator(&expected_s->tag), InvalidBackendId); + smgrread(reln, BufTagGetForkNum(&expected_s->tag), expected_s->tag.blockNum, + scratch.data); + if (!PageIsVerifiedExtended(storage_page, expected_s->tag.blockNum, + PIV_LOG_WARNING | PIV_REPORT_STAT)) { + hard_failure_reason = CLUSTER_PCM_S_SOURCE_HARD_STORAGE_VERIFY; + result = CLUSTER_PCM_OWN_CORRUPT; + } + if (result == CLUSTER_PCM_OWN_OK + && LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) { + content_locked = true; + buf_state = LockBufHdr(buf); + observed_buf_state = buf_state; + observed_buffer_type = buf->buffer_type; + if (!cluster_pcm_own_snapshot_matches_locked(buf, &live) + || live.flags != PCM_OWN_FLAG_REVOKING + || live.pcm_state != (uint8)PCM_STATE_S || (buf_state & BM_VALID) == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { + hard_failure_reason = CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_CURRENT_IMAGE; + result = CLUSTER_PCM_OWN_CORRUPT; + } + else if ((buf_state & BM_IO_ERROR) != 0) { + hard_failure_reason = CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_IO_ERROR; + result = CLUSTER_PCM_OWN_CORRUPT; + } + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) { + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_DIRTY_RACED; + result = CLUSTER_PCM_OWN_BUSY; + } + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) { + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_IO_IN_PROGRESS; + result = CLUSTER_PCM_OWN_BUSY; + } + else { + local_page = (Page)BufHdrGetBlock(buf); + local_scn = ((PageHeader)local_page)->pd_block_scn; + storage_scn = ((PageHeader)storage_page)->pd_block_scn; + local_covers + = !SCN_VALID(required_page_scn) + || (SCN_VALID(local_scn) + && scn_local(local_scn) >= scn_local(required_page_scn)); + storage_covers + = !SCN_VALID(required_page_scn) + || (SCN_VALID(storage_scn) + && scn_local(storage_scn) >= scn_local(required_page_scn)); + /* Every S grant is storage-consistent before publication. Once an + * exact non-source ACK has published a higher floor, neither copy + * covering it is durable lost-write evidence, not retryable lag. */ + if (!local_covers && !storage_covers) { + hard_failure_reason = CLUSTER_PCM_S_SOURCE_HARD_NO_COVER; + result = CLUSTER_PCM_OWN_CORRUPT; + } + else if (local_covers + && (!storage_covers || !SCN_VALID(storage_scn) + || (SCN_VALID(local_scn) + && scn_local(local_scn) >= scn_local(storage_scn)))) { + memcpy(block_data, local_page, BLCKSZ); + *out_page_lsn = PageGetLSN(local_page); + *out_page_scn = (uint64)local_scn; + *out_revoking = live; + } else { + memcpy((char *)BufHdrGetBlock(buf), scratch.data, BLCKSZ); + buf->buffer_type = (uint8)BUF_TYPE_CURRENT; + memcpy(block_data, scratch.data, BLCKSZ); + *out_page_lsn = PageGetLSN(storage_page); + *out_page_scn = (uint64)storage_scn; + *out_revoking = live; + } + } + UnlockBufHdr(buf, buf_state); + LWLockRelease(content_lock); + content_locked = false; + } else if (result == CLUSTER_PCM_OWN_OK) { + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_CONTENT_LOCK; + result = CLUSTER_PCM_OWN_BUSY; + } + } + PG_CATCH(); + { + if (content_locked && LWLockHeldByMe(content_lock)) + LWLockRelease(content_lock); + abort_result = cluster_bufmgr_pcm_own_abort_s_revoke(buf, &live); + if (abort_result != CLUSTER_PCM_OWN_OK) { + cluster_bufmgr_pcm_own_observe_s_source_hard_failure( + CLUSTER_PCM_S_SOURCE_HARD_ABORT_FAILURE, + CLUSTER_PCM_S_SOURCE_HARD_STORAGE_READ_ERROR, buf, &live, + required_page_scn, local_scn, storage_scn, observed_buf_state, + observed_buffer_type, CLUSTER_PCM_OWN_CORRUPT, abort_result); + elog(LOG, + "could not abort exact PCM-X S-source reservation after storage read error: " + "buffer=%d generation=%llu token=%llu result=%d; evidence retained", + buf->buf_id, (unsigned long long)live.generation, + (unsigned long long)live.reservation_token, (int)abort_result); + } + PG_RE_THROW(); + } + PG_END_TRY(); + + if (result == CLUSTER_PCM_OWN_OK) + return result; + abort_result = cluster_bufmgr_pcm_own_abort_s_revoke(buf, &live); + if (abort_result != CLUSTER_PCM_OWN_OK) { + cluster_bufmgr_pcm_own_observe_s_source_hard_failure( + CLUSTER_PCM_S_SOURCE_HARD_ABORT_FAILURE, hard_failure_reason, buf, &live, + required_page_scn, local_scn, storage_scn, observed_buf_state, + observed_buffer_type, result, abort_result); + return CLUSTER_PCM_OWN_CORRUPT; + } + if (result == CLUSTER_PCM_OWN_CORRUPT) + cluster_bufmgr_pcm_own_observe_s_source_hard_failure( + hard_failure_reason, CLUSTER_PCM_S_SOURCE_HARD_NONE, buf, &live, + required_page_scn, local_scn, storage_scn, observed_buf_state, + observed_buffer_type, result, abort_result); + memset(out_revoking, 0, sizeof(*out_revoking)); + memset(block_data, 0, BLCKSZ); + *out_page_lsn = InvalidXLogRecPtr; + *out_page_scn = 0; + return result; +} + /* Abort only the matching S-source staging reservation. */ ClusterPcmOwnResult cluster_bufmgr_pcm_own_abort_s_revoke(BufferDesc *buf, @@ -12129,7 +13195,8 @@ static ClusterPcmOwnResult cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, XLogRecPtr expected_lsn, - ClusterPcmOwnSnapshot *out_finished) + ClusterPcmOwnSnapshot *out_finished, + ClusterPcmOwnFinishRefusal *out_refusal) { ClusterPcmXRevokeFinishMode finish_mode; ClusterPcmOwnResult live_result; @@ -12140,6 +13207,7 @@ cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(BufferDesc *buf, uint64 live_token; uint32 hashcode; uint32 flags; + uint32 shared_refcount; uint32 buf_state; int mapped_buf_id; @@ -12153,33 +13221,51 @@ cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(BufferDesc *buf, } buf_state = LockBufHdr(buf); - finish_mode = cluster_pcm_x_revoke_finish_mode(&tag, BUF_STATE_GET_REFCOUNT(buf_state)); + shared_refcount = BUF_STATE_GET_REFCOUNT(buf_state); + finish_mode = cluster_pcm_x_revoke_finish_mode(&tag, shared_refcount); if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0 || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation || buf->pcm_state != expected_revoking->pcm_state) result = CLUSTER_PCM_OWN_STALE; else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) result = CLUSTER_PCM_OWN_CORRUPT; - else if (finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_BUSY) - result = CLUSTER_PCM_OWN_BUSY; - else if (finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_DROP) - result = CLUSTER_PCM_OWN_CORRUPT; - else if ((buf_state & BM_IO_IN_PROGRESS) != 0) - result = CLUSTER_PCM_OWN_BUSY; else { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); - live_result = cluster_pcm_own_classify_live_flags(flags, live_token); - if (live_result == CLUSTER_PCM_OWN_CORRUPT) - result = live_result; - else if (flags == 0) - result = CLUSTER_PCM_OWN_STALE; - else if (flags != PCM_OWN_FLAG_REVOKING) + if (out_refusal != NULL) { + out_refusal->shared_refcount = shared_refcount; + out_refusal->bm_io_in_progress = (buf_state & BM_IO_IN_PROGRESS) != 0; + out_refusal->live_flags = flags; + out_refusal->live_token = live_token; + } + if (finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_BUSY) { + if (out_refusal != NULL) + out_refusal->reason = CLUSTER_PCM_OWN_FINISH_REFUSAL_VM_FSM_PINNED; result = CLUSTER_PCM_OWN_BUSY; - else if (live_token != expected_revoking->reservation_token) - result = CLUSTER_PCM_OWN_STALE; - else if (PageGetLSN((Page)BufHdrGetBlock(buf)) != expected_lsn) - result = CLUSTER_PCM_OWN_STALE; + } + else if (finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_DROP) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) { + if (out_refusal != NULL) + out_refusal->reason = CLUSTER_PCM_OWN_FINISH_REFUSAL_IO_IN_PROGRESS; + result = CLUSTER_PCM_OWN_BUSY; + } + else { + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if (flags == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (flags != PCM_OWN_FLAG_REVOKING) { + if (out_refusal != NULL) + out_refusal->reason = CLUSTER_PCM_OWN_FINISH_REFUSAL_LIVE_FLAGS; + result = CLUSTER_PCM_OWN_BUSY; + } + else if (live_token != expected_revoking->reservation_token) + result = CLUSTER_PCM_OWN_STALE; + else if (PageGetLSN((Page)BufHdrGetBlock(buf)) != expected_lsn) + result = CLUSTER_PCM_OWN_STALE; + } } if (result == CLUSTER_PCM_OWN_OK) @@ -12205,11 +13291,11 @@ cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(BufferDesc *buf, /* * Commit one staged S/X source as a retained image, except for VM/FSM. * - * REVOKING already binds the descriptor to its tag through D5a and the victim - * guard, so this path deliberately does not nest a mapping lock with the - * content lock. Content EXCLUSIVE is the byte/I/O linearization: a flush - * that already owns SHARE completes first; once this lock is acquired, no - * writer or output I/O can overlap the exact LSN check and transition. + * Mapping authority binds the tag to the descriptor while this DATA worker + * takes its own raw pin. The pin then closes descriptor-retag ABA after the + * mapping lock is released. Content EXCLUSIVE is the byte/I/O linearization: + * a flush that already owns SHARE completes first; once this lock is acquired, + * no writer or output I/O can overlap the exact LSN check and transition. * Passive pins remain untouched. The descriptor keeps tag, BM_VALID, * refcount, and bytes, while PI+N marks those bytes non-current and the exact * REVOKING token prevents reuse until DRAIN. @@ -12217,23 +13303,32 @@ cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(BufferDesc *buf, ClusterPcmOwnResult cluster_bufmgr_pcm_own_finish_revoke_retain( BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, - XLogRecPtr expected_lsn, ClusterPcmOwnSnapshot *out_retained) + XLogRecPtr expected_lsn, ClusterPcmOwnSnapshot *out_retained, + ClusterPcmOwnFinishRefusal *out_refusal) { ClusterPcmOwnResult live_result; ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; BufferTag tag; LWLock *content_lock; + LWLock *partition_lock; uint64 committed_generation; uint64 live_token; + uint32 hashcode; uint32 flags; uint32 buf_state; + int mapped_buf_id; bool source_is_s; bool source_is_x; bool needs_flush = false; + bool forced_test_flush = false; + volatile bool content_locked = false; + volatile bool caller_pinned = false; ClusterPcmXRevokeFinishMode finish_mode; if (out_retained != NULL) memset(out_retained, 0, sizeof(*out_retained)); + if (out_refusal != NULL) + memset(out_refusal, 0, sizeof(*out_refusal)); if (buf == NULL || expected_revoking == NULL || out_retained == NULL) return CLUSTER_PCM_OWN_INVALID; source_is_s = expected_revoking->pcm_state == (uint8) PCM_STATE_S; @@ -12251,16 +13346,28 @@ cluster_bufmgr_pcm_own_finish_revoke_retain( return CLUSTER_PCM_OWN_INVALID; if (finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_DROP) return cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(buf, expected_revoking, - expected_lsn, out_retained); + expected_lsn, out_retained, out_refusal); + if (finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_RETAIN) + return CLUSTER_PCM_OWN_INVALID; Assert(finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_RETAIN); - content_lock = BufferDescriptorGetContentLock(buf); - LWLockAcquire(content_lock, LW_EXCLUSIVE); + /* + * Bind tag -> descriptor under mapping authority, then add this DATA + * worker's own raw pin before dropping that authority. The pin, not a + * passive backend's incidental refcount, is the FlushBuffer lifetime + * contract and closes descriptor-retag ABA through the final commit. + */ + hashcode = BufTableHashCode(&tag); + partition_lock = BufMappingPartitionLock(hashcode); + LWLockAcquire(partition_lock, LW_SHARED); + mapped_buf_id = BufTableLookup(&tag, hashcode); + if (mapped_buf_id != buf->buf_id) { + LWLockRelease(partition_lock); + return CLUSTER_PCM_OWN_STALE; + } buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) - || (buf_state & BM_VALID) == 0 - || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation - || buf->pcm_state != expected_revoking->pcm_state) + if (!cluster_pcm_own_snapshot_matches_locked(buf, expected_revoking) + || (buf_state & BM_VALID) == 0) result = CLUSTER_PCM_OWN_STALE; else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) result = CLUSTER_PCM_OWN_CORRUPT; @@ -12268,42 +13375,29 @@ cluster_bufmgr_pcm_own_finish_revoke_retain( result = CLUSTER_PCM_OWN_BUSY; else if ((buf_state & BM_IO_ERROR) != 0) result = CLUSTER_PCM_OWN_CORRUPT; - else - { - live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); - flags = cluster_pcm_own_flags_get(buf->buf_id); - live_result = cluster_pcm_own_classify_live_flags(flags, live_token); - if (live_result == CLUSTER_PCM_OWN_CORRUPT) - result = live_result; - else if (flags == 0) - result = CLUSTER_PCM_OWN_STALE; - else if (flags != PCM_OWN_FLAG_REVOKING) - result = CLUSTER_PCM_OWN_BUSY; - else if (live_token != expected_revoking->reservation_token) - result = CLUSTER_PCM_OWN_STALE; - else if (PageGetLSN((Page) BufHdrGetBlock(buf)) != expected_lsn) - result = CLUSTER_PCM_OWN_STALE; - else - needs_flush - = (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0; + else { + cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); + caller_pinned = true; + buf_state = 0; } - UnlockBufHdr(buf, buf_state); + if (buf_state != 0) + UnlockBufHdr(buf, buf_state); + LWLockRelease(partition_lock); + if (result != CLUSTER_PCM_OWN_OK) + return result; - if (result == CLUSTER_PCM_OWN_OK) - { - /* A hint-bit update may legally dirty the page under the shared - * content lock after the immutable A-record copy. Under EXCLUSIVE no - * further byte change can race us: flush that same-LSN image, then - * re-prove both the ownership token and a clean storage fallback before - * retiring the descriptor. */ - if (needs_flush) - FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + content_lock = BufferDescriptorGetContentLock(buf); + if (!LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) { + cluster_bufmgr_unpin_for_gcs(buf); + return CLUSTER_PCM_OWN_BUSY; + } + content_locked = true; + PG_TRY(); + { buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) - || (buf_state & BM_VALID) == 0 - || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation - || buf->pcm_state != expected_revoking->pcm_state) + if (!cluster_pcm_own_snapshot_matches_locked(buf, expected_revoking) + || (buf_state & BM_VALID) == 0) result = CLUSTER_PCM_OWN_STALE; else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) result = CLUSTER_PCM_OWN_CORRUPT; @@ -12311,41 +13405,121 @@ cluster_bufmgr_pcm_own_finish_revoke_retain( result = CLUSTER_PCM_OWN_BUSY; else if ((buf_state & BM_IO_ERROR) != 0) result = CLUSTER_PCM_OWN_CORRUPT; - else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) - result = CLUSTER_PCM_OWN_BUSY; - else - { - live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); - flags = cluster_pcm_own_flags_get(buf->buf_id); - live_result = cluster_pcm_own_classify_live_flags(flags, live_token); - if (live_result == CLUSTER_PCM_OWN_CORRUPT) - result = live_result; - else if (flags == 0) - result = CLUSTER_PCM_OWN_STALE; - else if (flags != PCM_OWN_FLAG_REVOKING) - result = CLUSTER_PCM_OWN_BUSY; - else if (live_token != expected_revoking->reservation_token) - result = CLUSTER_PCM_OWN_STALE; - else if (PageGetLSN((Page) BufHdrGetBlock(buf)) != expected_lsn) - result = CLUSTER_PCM_OWN_STALE; + else if (PageGetLSN((Page)BufHdrGetBlock(buf)) != expected_lsn) + result = CLUSTER_PCM_OWN_STALE; + else { + /* + * The copy/materialize leg normally flushes a dirty source before + * finish. Test arming dirties the unchanged exact image under the + * caller pin + content EXCLUSIVE so the finish FlushBuffer boundary + * is deterministic, without fabricating different page bytes. + */ + forced_test_flush = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); + if (forced_test_flush) + buf_state |= BM_DIRTY | BM_JUST_DIRTIED; + needs_flush = (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0; } + UnlockBufHdr(buf, buf_state); - if (result == CLUSTER_PCM_OWN_OK) - result = cluster_pcm_own_revoke_retain_commit_exact( - buf->buf_id, expected_revoking->generation, - expected_revoking->reservation_token, &committed_generation); - if (result == CLUSTER_PCM_OWN_OK) + if (result == CLUSTER_PCM_OWN_OK) { + /* + * A hint-bit update may legally dirty the page under the shared + * content lock after the immutable A-record copy. EXCLUSIVE freezes + * that image; the caller-owned pin covers FlushBuffer and the exact + * ownership commit that follows it. + */ + if (needs_flush && !caller_pinned) + result = CLUSTER_PCM_OWN_CORRUPT; + else if (needs_flush) { + cluster_pcm_x_finish_retain_flush_active = true; + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + cluster_pcm_x_finish_retain_flush_active = false; + if (forced_test_flush) + elog(LOG, + "cluster PCM-X retained-image finish FlushBuffer succeeded: " + "buffer=%d rel=%u fork=%d blk=%u", + buf->buf_id, tag.relNumber, (int)tag.forkNum, tag.blockNum); + } + + if (result == CLUSTER_PCM_OWN_OK) { + buf_state = LockBufHdr(buf); + if (!cluster_pcm_own_snapshot_matches_locked(buf, expected_revoking) + || (buf_state & BM_VALID) == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + result = CLUSTER_PCM_OWN_BUSY; + else if ((buf_state & BM_IO_ERROR) != 0) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) + result = CLUSTER_PCM_OWN_BUSY; + else if (PageGetLSN((Page)BufHdrGetBlock(buf)) != expected_lsn) + result = CLUSTER_PCM_OWN_STALE; + else { + live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if (flags != PCM_OWN_FLAG_REVOKING) + result = flags == 0 ? CLUSTER_PCM_OWN_STALE : CLUSTER_PCM_OWN_BUSY; + else if (live_token != expected_revoking->reservation_token) + result = CLUSTER_PCM_OWN_STALE; + } + + if (result == CLUSTER_PCM_OWN_OK) + result = cluster_pcm_own_revoke_retain_commit_exact( + buf->buf_id, expected_revoking->generation, + expected_revoking->reservation_token, &committed_generation); + if (result == CLUSTER_PCM_OWN_OK) { + buf->pcm_state = (uint8)PCM_STATE_N; + buf->buffer_type = (uint8)BUF_TYPE_PI; + buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR); + cluster_pcm_own_snapshot_locked(buf, out_retained); + } + UnlockBufHdr(buf, buf_state); + } + } + + LWLockRelease(content_lock); + content_locked = false; + } + PG_CATCH(); + { + /* elog(ERROR) resets InterruptHoldoffCount before longjmp, but a + * still-held LWLock retains its ownership entry. Recreate that lock's + * interrupt hold only on the exact release path; LWLockRelease consumes + * the replacement hold itself. */ + cluster_pcm_x_finish_retain_flush_active = false; + /* FlushBuffer normally leaves BufferIO cleanup to transaction abort. + * This DATA worker intentionally absorbs the injected/physical write + * ERROR after preserving the immutable revoke record, so restore its + * stack-local error callback and terminate its exact ResourceOwner IO + * before dropping the caller-owned pin. */ + if (cluster_pcm_x_finish_retain_flush_error_context_pushed) { - buf->pcm_state = (uint8) PCM_STATE_N; - buf->buffer_type = (uint8) BUF_TYPE_PI; - buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED | - BM_CHECKPOINT_NEEDED | BM_IO_ERROR); - cluster_pcm_own_snapshot_locked(buf, out_retained); + error_context_stack = cluster_pcm_x_finish_retain_flush_error_context_previous; + cluster_pcm_x_finish_retain_flush_error_context_pushed = false; + cluster_pcm_x_finish_retain_flush_error_context_previous = NULL; } - UnlockBufHdr(buf, buf_state); + if (content_locked && LWLockHeldByMe(content_lock)) { + HOLD_INTERRUPTS(); + LWLockRelease(content_lock); + } + if (cluster_pcm_x_finish_retain_flush_io_active) + { + cluster_pcm_x_finish_retain_flush_io_active = false; + AbortBufferIO(BufferDescriptorGetBuffer(buf)); + } + if (caller_pinned) + cluster_bufmgr_unpin_for_gcs(buf); + PG_RE_THROW(); } + PG_END_TRY(); - LWLockRelease(content_lock); + if (caller_pinned) + cluster_bufmgr_unpin_for_gcs(buf); return result; } @@ -12425,7 +13599,8 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, return result; content_lock = BufferDescriptorGetContentLock(buf); - LWLockAcquire(content_lock, LW_EXCLUSIVE); + if (!LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) + return CLUSTER_PCM_OWN_BUSY; buf_state = LockBufHdr(buf); if (!BufferTagsEqual(&buf->tag, tag) @@ -12479,11 +13654,11 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, LWLockRelease(content_lock); if (fx_log) elog(LOG, - "cluster PCM retained image released: image=%s buffer=%d rel=%u fork=%d blk=%u refcount=%u gen=%llu token=%llu", - fx_dropped ? "dropped" : "kept-pinned-pi", buf->buf_id, - tag->relNumber, (int) tag->forkNum, tag->blockNum, - fx_refcount, (unsigned long long) committed_generation, - (unsigned long long) retained_token); + "cluster PCM retained image released: image=%s buffer=%d rel=%u fork=%d blk=%u " + "refcount=%u gen=%llu token=%llu", + fx_dropped ? "dropped" : "kept-pinned-pi", buf->buf_id, tag->relNumber, + (int)tag->forkNum, tag->blockNum, fx_refcount, + (unsigned long long)committed_generation, (unsigned long long)retained_token); return result; } @@ -12595,11 +13770,13 @@ cluster_bufmgr_block_is_pi(BufferTag tag) bool cluster_bufmgr_discard_pi_block(BufferTag tag) { + ClusterPcmOwnSnapshot activation_diag; uint32 hash = BufTableHashCode(&tag); LWLock *partition_lock = BufMappingPartitionLock(hash); int buf_id; BufferDesc *buf; uint32 buf_state; + bool activation_diag_valid; LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hash); @@ -12623,6 +13800,8 @@ cluster_bufmgr_discard_pi_block(BufferTag tag) * but clear it again under the lock so InvalidateBuffer's eviction hook * can never see a stale mode and emit a release wire from LMON. */ buf->pcm_state = (uint8) PCM_STATE_N; + cluster_pcm_own_snapshot_locked(buf, &activation_diag); + activation_diag_valid = activation_diag.writer_activation_token != 0; /* PGRAC: spec-6.12h D-h3a — hygiene: drop the shadow stamp with the PI. * Correctness never depends on this clear (the D-h3 consumer * re-validates the PI shape + tag under this same header lock, and @@ -12630,6 +13809,9 @@ cluster_bufmgr_discard_pi_block(BufferTag tag) * slot should not linger as a plausible-looking stamp. */ cluster_pi_shadow_clear(buf->buf_id); InvalidateBuffer(buf); /* releases the header spinlock */ + if (activation_diag_valid) + cluster_pcm_own_activation_diag_emit("discard-pi-stage-n", buf->buf_id, + &activation_diag, CLUSTER_PCM_OWN_CORRUPT); return true; } diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h index c103997703..a969b31555 100644 --- a/src/include/access/visibilitymap.h +++ b/src/include/access/visibilitymap.h @@ -33,6 +33,8 @@ extern bool visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags); extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk, Buffer *vmbuf); +extern bool visibilitymap_pin_recent(Relation rel, BlockNumber heapBlk, + Buffer recent_buffer, Buffer *vmbuf); extern bool visibilitymap_pin_ok(BlockNumber heapBlk, Buffer vmbuf); extern void visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf, XLogRecPtr recptr, Buffer vmBuf, TransactionId cutoff_xid, diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index 07d2d83b44..e2aaed2997 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -514,12 +514,13 @@ typedef struct GcsBlockPcmXImageIdentity { typedef struct GcsBlockPcmXImageBinding { GcsBlockPcmXImageIdentity identity; uint64 master_session; + uint64 required_page_scn; } GcsBlockPcmXImageBinding; StaticAssertDecl(sizeof(GcsBlockPcmXImageIdentity) == 128, "PCM-X image identity must fit the existing dedup 128-byte metadata cell"); -StaticAssertDecl(sizeof(GcsBlockPcmXImageBinding) == 136, - "PCM-X image binding is identity plus authenticated master session"); +StaticAssertDecl(sizeof(GcsBlockPcmXImageBinding) == 144, + "PCM-X image binding includes session and required source floor"); static inline bool GcsBlockPcmXImageIdentityEqual(const GcsBlockPcmXImageIdentity *left, @@ -533,6 +534,7 @@ GcsBlockPcmXImageBindingEqual(const GcsBlockPcmXImageBinding *left, const GcsBlockPcmXImageBinding *right) { return left != NULL && right != NULL && left->master_session == right->master_session + && left->required_page_scn == right->required_page_scn && GcsBlockPcmXImageIdentityEqual(&left->identity, &right->identity); } @@ -949,7 +951,9 @@ cluster_gcs_pcm_x_revoke_ingress_valid(const PcmXRevokePayload *request, Size pa int32 authenticated_node, uint64 current_epoch, int32 tag_master, int32 local_node) { - return request != NULL && payload_length == sizeof(*request) && authenticated_node >= 0 + return request != NULL + && (payload_length == sizeof(*request) || payload_length == sizeof(PcmXRevokePayloadV2)) + && authenticated_node >= 0 && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == authenticated_node && cluster_gcs_pcm_x_transfer_ref_wire_valid(&request->ref, current_epoch) @@ -1296,6 +1300,32 @@ cluster_gcs_block_direct_state_transition_ok(ClusterGcsBlockDirectState from, return false; } +/* + * A current-master INVALIDATE that meets a node-local mirror-N + * GRANT_PENDING is also an authoritative reason for the one exact N->S + * attempt to stand down: that reader cannot be granted while this X transfer + * is invalidating the node. Keep the delivery predicate attempt-exact and + * refuse any live direct-land target. The caller only synthesizes a local + * DENIED_PENDING_X; the INVALIDATE itself still returns RETRYABLE_BUSY until + * the owning backend aborts its reservation, so no holder bit is credited. + */ +static inline bool +GcsBlockLocalPendingSDenialMatches(bool in_use, bool reply_received, bool stale, + uint8 transition_id, const BufferTag *slot_tag, + uint64 request_epoch, int32 expected_master_node, + ClusterGcsBlockDirectState direct_state, + bool direct_target_prepared, + const BufferTag *invalidate_tag, uint64 invalidate_epoch, + int32 invalidate_master_node) +{ + return in_use && !reply_received && !stale && slot_tag != NULL && invalidate_tag != NULL + && transition_id == (uint8)PCM_TRANS_N_TO_S + && BufferTagsEqual(slot_tag, invalidate_tag) && request_epoch == invalidate_epoch + && expected_master_node == invalidate_master_node && !direct_target_prepared + && (direct_state == GCS_BLOCK_DIRECT_UNARMED + || direct_state == GCS_BLOCK_DIRECT_ABORTED); +} + /* ============================================================ * GcsBlockReplyStatus -- reply status code carried in @@ -1479,6 +1509,27 @@ typedef enum GcsBlockReplyStatus { * requester keeps 53R97 (Rule 8.A). */ } GcsBlockReplyStatus; +/* + * The block-request wire validator admits only legal transition ids, but the + * master's outer S/X decision and its final entry-lock transition apply are + * deliberately separated by buffer probing/copying. A concurrent handoff can + * therefore make an otherwise valid N->S/N->X request incompatible at that + * final apply. That is authority drift, not a client-terminal protocol error: + * reuse DENIED_PENDING_X's established fresh-request/token retry boundary. + * Keep every other transition's incompatibility terminal so malformed or + * structurally illegal control-plane transitions are never papered over. + */ +static inline GcsBlockReplyStatus +GcsBlockApplyRefusalStatus(PcmGcsTransitionApplyResult apply_result, + PcmLockTransition transition_id) +{ + if (apply_result == PCM_GCS_TRANSITION_PENDING_X + || (apply_result == PCM_GCS_TRANSITION_INCOMPATIBLE + && (transition_id == PCM_TRANS_N_TO_S || transition_id == PCM_TRANS_N_TO_X))) + return GCS_BLOCK_REPLY_DENIED_PENDING_X; + return GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE; +} + /* spec-5.16 D3b / r4 (spec-6.12a ㉕ extends) — every new reply status MUST be * appended as the tail value (no collision with any shipped status; r3 mis-read * a truncated enum as max 8, the real shipped max before spec-5.16 was @@ -3062,7 +3113,39 @@ StaticAssertDecl(GCS_BLOCK_DATA_SIZE == BLCKSZ, #include "cluster/cluster_pcm_lock.h" /* PcmLockMode for invalidate helper */ extern bool cluster_bufmgr_probe_block_for_gcs(BufferTag tag); extern bool cluster_bufmgr_read_storage_scn_for_gcs(BufferTag tag, SCN *out_page_scn); -extern bool cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char *dst); +/* Process-local diagnostic returned by the nonblocking holder-copy helper. + * This is observation only: the caller's bool success/deny contract and wire + * reply status remain unchanged. */ +typedef enum ClusterBufmgrGcsCopyRefusal { + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE = 0, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INVALID_ARGUMENT, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_SECOND, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_HC89_LSN_DRIFT, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_SMART_FUSION_UNCLASSIFIED, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INJECTED_EVICT +} ClusterBufmgrGcsCopyRefusal; + +/* A DATA worker cannot wait for BufferContent: its owner may itself be waiting + * for that worker to deliver a reply. Only the two conditional-lock misses + * are therefore retryable through the established fresh reservation/request + * boundary. Residency/current-image failures remain structural, while HC89 + * keeps its explicit one-retry hot-page bound. */ +static inline GcsBlockReplyStatus +GcsBlockMasterDirectCopyRefusalStatus(ClusterBufmgrGcsCopyRefusal refusal) +{ + if (refusal == CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST + || refusal == CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_SECOND) + return GCS_BLOCK_REPLY_DENIED_PENDING_X; + return GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; +} + +extern const char *cluster_bufmgr_gcs_copy_refusal_name(ClusterBufmgrGcsCopyRefusal refusal); +extern bool cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char *dst, + ClusterBufmgrGcsCopyRefusal *out_refusal); extern bool cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page_lsn, void **out_page_addr, BufferDesc **out_buf); @@ -3191,8 +3274,21 @@ extern bool cluster_gcs_block_local_x_upgrade_ext(BufferTag tag, bool *out_busy) * consistent), applies PCM_TRANS_X_TO_S_DOWNGRADE, flips the local * pcm_state cache X->S. False = not quiescent / not X / buffer gone / * master refused; caller falls back to the one-shot read-image ship. */ +typedef enum ClusterBufmgrGcsDowngradeOutcome { + CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY = 0, + CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED, + CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY +} ClusterBufmgrGcsDowngradeOutcome; extern bool cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag); +extern ClusterBufmgrGcsDowngradeOutcome +cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image( + BufferTag tag, XLogRecPtr *out_page_lsn, char *dst, + ClusterBufmgrGcsCopyRefusal *out_refusal); extern bool cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node); +extern ClusterBufmgrGcsDowngradeOutcome +cluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image( + BufferTag tag, int32 master_node, XLogRecPtr *out_page_lsn, char *dst, + ClusterBufmgrGcsCopyRefusal *out_refusal); /* PGRAC: GCS-race round-4c P1 — re-send the (idempotent) X->S downgrade * notify when a BAST nudge arrives for a block this node already holds in @@ -3297,14 +3393,22 @@ extern bool cluster_gcs_send_block_request_and_wait(BufferDesc *buf, * block a REMOTE node holds in X and a local reader needs an N→S image. * Forwards a read-image request to the holder and installs the shipped * current image for one read. Returns false (non-durable; caller leaves - * buf->pcm_state == N); fails closed (ereport) if no image is obtained. + * buf->pcm_state == N); fails closed (ereport) if no image is obtained while + * expected remains exact. Authority drift instead sets out_retry_denied so + * bufmgr aborts/rearms GRANT_PENDING and selects a fresh holder identity. */ -extern bool cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, int32 holder_node); +extern bool cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, + const PcmAuthoritySnapshot *expected, + bool *out_retry_denied); /* PGRAC: spec-5.2 D11 — local-master writer-transfer (revoke); durable X grant. * spec-5.2a D2/D3: clean_eligible routes a clean (sequence) page through the - * flush-data-before-drop holder path + stale-holder storage-fallback recovery. */ -extern bool cluster_gcs_local_master_x_transfer_and_wait(BufferDesc *buf, int32 holder_node, - bool clean_eligible); + * flush-data-before-drop holder path + stale-holder storage-fallback recovery. + * P0-26: expected is the entry-lock authority token; authority drift returns + * retry_denied so bufmgr aborts/rearms a fresh ownership/request identity. */ +extern bool cluster_gcs_local_master_x_transfer_and_wait(BufferDesc *buf, + const PcmAuthoritySnapshot *expected, + bool clean_eligible, + bool *out_retry_denied); /* * spec-4.7 D1 — GCS/PCM block resource recovery phase. @@ -3577,6 +3681,7 @@ extern uint64 cluster_gcs_get_x_vs_s_no_carrier_denied_count(void); /* PGRAC: spec-2.37 D12 — 4 NEW counter accessors for PI watermark + lost-write. */ extern uint64 cluster_gcs_get_pi_watermark_advance_count(void); extern uint64 cluster_gcs_get_pi_watermark_retire_count(void); +extern uint64 cluster_gcs_get_pi_durable_note_apply_count(void); extern uint64 cluster_gcs_get_lost_write_detected_count(void); extern uint64 cluster_gcs_get_lost_write_avoid_count(void); /* PGRAC: spec-2.41 D7 — SCN detector + redo-coverage observability accessors. */ diff --git a/src/include/cluster/cluster_gcs_block_dedup.h b/src/include/cluster/cluster_gcs_block_dedup.h index 93d6f4ea0d..62c7ab29dc 100644 --- a/src/include/cluster/cluster_gcs_block_dedup.h +++ b/src/include/cluster/cluster_gcs_block_dedup.h @@ -133,8 +133,9 @@ StaticAssertDecl(sizeof(GcsBlockDedupKey) == 24, "spec-2.34 D2 GcsBlockDedupKey * cell is an outbound-admission marker: nonzero * suppresses type-50 resends until exact type 49 * clears it. It is never application completion; - * exact DRAIN removes the dedicated entry, and all - * generic reclaim paths reject non-GENERIC kinds. + * exact DRAIN publishes a replay tombstone; exact + * RETIRE removes it, and all + generic reclaim paths * reject non-GENERIC kinds. * pinned_lifetime_us the legal-request-lifetime threshold, pinned * at REGISTRATION from the requester's wire * hint (2x margin applied) or, absent a hint, @@ -152,7 +153,10 @@ typedef enum GcsBlockDedupEntryKind { GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE = 2, /* Stable bytes exist, but X->N has not yet been positively committed. * This state is retained across LMS death and is never sendable. */ - GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED = 3 + GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED = 3, + /* Exact descriptor/byte cleanup completed after local TERMINAL_DRAINED. + * Keep the binding as an ACK replay tombstone until exact RETIRE. */ + GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED = 4 } GcsBlockDedupEntryKind; typedef union GcsBlockDedupPayloadMeta { @@ -169,12 +173,13 @@ typedef struct GcsBlockDedupEntry { uint8 transition_id; /* 1B — HC91 collision check */ uint8 status; /* 1B — GcsBlockReplyStatus */ uint8 entry_kind; /* 1B — GcsBlockDedupEntryKind */ - uint8 request_flags; /* 1B — original request properties */ + uint8 request_flags; /* 1B — generic request properties; PCM-X + * source state or drained master node + 1 */ uint64 pcm_x_master_session; /* 8B — PCM-X kind only */ GcsBlockReplyHeader reply_header; /* 48B — full reply header (HC99) */ - bool has_sf_dep; /* 1B — spec-6.2 v2 dep vector present */ - uint8 sf_flags; /* 1B — GCS_BLOCK_REPLY_SF_* */ - uint8 sf_dep_count; /* 1B — non-empty dep vector entries */ + bool has_sf_dep; /* 1B — generic SF metadata; PCM-X */ + uint8 sf_flags; /* 1B — metadata cell overlays the */ + uint8 sf_dep_count; /* 1B — exact reservation token */ uint8 _pad1[5]; /* 5B — dep_vec @ 112 */ GcsBlockDedupPayloadMeta payload_meta; /* 128B — deps or exact PCM-X identity */ char block_data[GCS_BLOCK_DATA_SIZE]; /* 8192B — full page payload */ @@ -212,7 +217,10 @@ typedef enum GcsBlockPcmXImageResult { GCS_BLOCK_PCM_X_IMAGE_NOT_READY, GCS_BLOCK_PCM_X_IMAGE_STALE, GCS_BLOCK_PCM_X_IMAGE_FULL, - GCS_BLOCK_PCM_X_IMAGE_INVALID + GCS_BLOCK_PCM_X_IMAGE_INVALID, + /* Immutable bytes exist, but ownership commit is still pending. This + * process-local work result is never a sendable READY classification. */ + GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING } GcsBlockPcmXImageResult; /* By-value LMS work descriptor. It deliberately carries no page bytes: @@ -221,13 +229,15 @@ typedef enum GcsBlockPcmXImageResult { typedef struct GcsBlockPcmXImageWork { GcsBlockDedupKey key; GcsBlockPcmXImageBinding binding; + uint64 reservation_token; BufferTag tag; + uint8 source_pcm_state; uint8 entry_kind; - uint8 _reserved[3]; + uint8 _reserved[2]; } GcsBlockPcmXImageWork; -StaticAssertDecl(sizeof(GcsBlockPcmXImageWork) == 184, - "PCM-X LMS work descriptor remains a by-value 184-byte token"); +StaticAssertDecl(sizeof(GcsBlockPcmXImageWork) == 200, + "PCM-X LMS work descriptor includes the exact source floor binding"); /* ============================================================ @@ -472,11 +482,13 @@ extern GcsBlockDedupResult cluster_gcs_block_dedup_lookup_or_register( * N->S grant/forward identity after PCM-X publishes its queue-kind claim. * A by-value cached denial is returned for initial send or periodic replay; * exact DONE removes it from the replay set. */ -extern GcsBlockPendingXDenyResult cluster_gcs_block_dedup_pending_x_deny_next( - int worker_id, const BufferTag *tag, GcsBlockDedupEntry *denied_out); -extern GcsBlockPendingXDenyResult cluster_gcs_block_dedup_pending_x_deny_exact( - int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, uint8 transition_id, - GcsBlockDedupEntry *denied_out); +extern GcsBlockPendingXDenyResult +cluster_gcs_block_dedup_pending_x_deny_next(int worker_id, const BufferTag *tag, + GcsBlockDedupEntry *denied_out); +extern GcsBlockPendingXDenyResult +cluster_gcs_block_dedup_pending_x_deny_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, uint8 transition_id, + GcsBlockDedupEntry *denied_out); extern bool cluster_gcs_block_dedup_set_request_flags_exact( int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, uint8 transition_id, uint8 request_flags); @@ -491,8 +503,8 @@ cluster_gcs_block_dedup_pcm_x_reserve(int worker_id, const GcsBlockDedupKey *key const GcsBlockPcmXImageBinding *reserved_binding); extern GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_materialize( int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, - const GcsBlockPcmXImageBinding *ready_binding, const GcsBlockReplyHeader *reply_header, - const char *block_data); + const GcsBlockPcmXImageBinding *ready_binding, uint64 reservation_token, uint8 source_pcm_state, + const GcsBlockReplyHeader *reply_header, const char *block_data); extern GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_publish_ready_exact(int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, @@ -501,9 +513,19 @@ extern GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_lookup( int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, const GcsBlockPcmXImageBinding *expected_binding, GcsBlockDedupEntry *cached_reply_out); extern GcsBlockPcmXImageResult -cluster_gcs_block_dedup_pcm_x_release_exact(int worker_id, const GcsBlockDedupKey *key, - const BufferTag *tag, - const GcsBlockPcmXImageBinding *binding); +cluster_gcs_block_dedup_pcm_x_drain_status_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *binding); +extern GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_release_exact( + int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, + const GcsBlockPcmXImageBinding *binding, int32 drained_master_node); +extern bool cluster_gcs_block_dedup_pcm_x_retire_up_to(uint64 cluster_epoch, + int32 authenticated_master_node, + uint64 authenticated_master_session, + uint64 retire_through_ticket_id); +extern GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_preserve_finish_error_exact( + int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, + const GcsBlockPcmXImageBinding *binding, uint64 reservation_token, uint8 source_pcm_state); extern GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_next_work(int worker_id, GcsBlockPcmXImageWork *work_out); extern GcsBlockPcmXImageResult diff --git a/src/include/cluster/cluster_ic.h b/src/include/cluster/cluster_ic.h index ed626ff601..c8ca15c96a 100644 --- a/src/include/cluster/cluster_ic.h +++ b/src/include/cluster/cluster_ic.h @@ -378,6 +378,9 @@ typedef enum ClusterICPlane { * verdict) and every INSTALL_READY stays the V1 104-byte exact frame, so an * old master never sees a length its byte-exact table would refuse. */ #define PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1 ((uint32)0x00000400U) +/* P0-20: this binary accepts the 104-byte PCM-X REVOKE V2 frame and binds its + * trailing required_page_scn to the immutable source-image lifecycle. */ +#define PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1 ((uint32)0x00000800U) /* * PGRAC: spec-7.2 D2 — plane + connection-epoch ride the documented-zero * pad region (capabilities precedent: occupy pad bytes, do not resize V1). diff --git a/src/include/cluster/cluster_inject.h b/src/include/cluster/cluster_inject.h index 614ab14d91..b6e258fe62 100644 --- a/src/include/cluster/cluster_inject.h +++ b/src/include/cluster/cluster_inject.h @@ -115,6 +115,9 @@ extern void cluster_injection_run(const char *name); extern bool cluster_injection_should_skip(const char *name); +/* Non-consuming armed-state probe for deterministic test setup. */ +extern bool cluster_injection_is_armed(const char *name); + /* * cluster_cr_injection_armed -- spec-3.9 D7 CR-specific armed-state peek. * Returns true iff the named CR injection point is armed; sets *out_param diff --git a/src/include/cluster/cluster_lms.h b/src/include/cluster/cluster_lms.h index 7277c4dca9..b79242c125 100644 --- a/src/include/cluster/cluster_lms.h +++ b/src/include/cluster/cluster_lms.h @@ -321,6 +321,9 @@ typedef struct ClusterLmsSharedState { */ pg_atomic_uint64 worker_outbound_not_admitted_count[CLUSTER_LMS_MAX_WORKERS]; pg_atomic_uint64 worker_outbound_requeue_drop_count[CLUSTER_LMS_MAX_WORKERS]; + /* Capability-bound wire frames consumed before send because the peer's + * current HELLO record no longer matches the staged connection generation. */ + pg_atomic_uint64 worker_outbound_cap_guard_drop_count[CLUSTER_LMS_MAX_WORKERS]; } ClusterLmsSharedState; @@ -401,11 +404,19 @@ extern void cluster_lms_outbound_shmem_register(void); extern void cluster_lms_outbound_request_lwlocks(void); extern bool cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, const void *payload, uint16 payload_len); +extern bool cluster_lms_outbound_enqueue_cap_bound(int worker_id, uint8 msg_type, + uint32 dest_node_id, const void *payload, + uint16 payload_len, uint32 required_capability, + uint32 connection_generation); struct GcsBlockReplyHeader; extern bool cluster_lms_outbound_enqueue_zero_block_reply( int worker_id, uint32 dest_node_id, const struct GcsBlockReplyHeader *header, bool direct_land); extern int cluster_lms_outbound_drain_send(int worker_id); extern uint32 cluster_lms_outbound_depth(int worker_id); +extern void cluster_lms_note_pcm_x_image_ready_boundary( + uint8 msg_type, const char *boundary, int result, int runtime_state, bool fence_enforcing, + bool fence_allowed, uint32 dest_node_id, uint64 request_id, uint64 ticket_id, uint64 grant_generation, + uint64 image_id); /* * Read-only accessors for SQL view + diagnostics. @@ -476,8 +487,10 @@ extern uint64 cluster_lms_obs_serve_hist_bound_us(int bucket); * requeue-drop; get_* readers follow the -1 = pool aggregate idiom). */ extern void cluster_lms_obs_note_outbound_not_admitted(int worker_id); extern void cluster_lms_obs_note_outbound_requeue_drop(int worker_id); +extern void cluster_lms_obs_note_outbound_cap_guard_drop(int worker_id); extern uint64 cluster_lms_obs_get_outbound_not_admitted(int worker_id); extern uint64 cluster_lms_obs_get_outbound_requeue_drop(int worker_id); +extern uint64 cluster_lms_obs_get_outbound_cap_guard_drop(int worker_id); /* spec-7.3 D8 (Q8) — apply cluster.lms_nice to the calling LMS process * (setpriority, best-effort; 0 = leave the inherited priority alone). */ diff --git a/src/include/cluster/cluster_pcm_lock.h b/src/include/cluster/cluster_pcm_lock.h index ee2d8935ae..0301a2f933 100644 --- a/src/include/cluster/cluster_pcm_lock.h +++ b/src/include/cluster/cluster_pcm_lock.h @@ -104,6 +104,16 @@ typedef enum PcmLockTransition { } PcmLockTransition; #define PCM_TRANSITION_COUNT 9 +/* Exact master-side apply verdict. PENDING_X is distinct from a structural + * state incompatibility so the block data plane can return its retryable + * DENIED_PENDING_X status after the final, entry-lock-serialized admission + * check closes a preflight-to-apply race. */ +typedef enum PcmGcsTransitionApplyResult { + PCM_GCS_TRANSITION_APPLIED = 0, + PCM_GCS_TRANSITION_INCOMPATIBLE = 1, + PCM_GCS_TRANSITION_PENDING_X = 2 +} PcmGcsTransitionApplyResult; + /* * GrdEntry -- opaque per-block global lock state (master node). @@ -171,6 +181,13 @@ typedef enum PcmXGrdHandoffResult { PCM_X_GRD_HANDOFF_BAD_STATE, PCM_X_GRD_HANDOFF_INVALID } PcmXGrdHandoffResult; +typedef enum PcmXTransferCommitResult { + PCM_X_TRANSFER_COMMIT_OK = 0, + PCM_X_TRANSFER_COMMIT_NOT_FOUND, + PCM_X_TRANSFER_COMMIT_STALE, + PCM_X_TRANSFER_COMMIT_BAD_STATE +} PcmXTransferCommitResult; + typedef enum PcmPendingXReserveResult { PCM_PENDING_X_RESERVE_INVALID = -2, @@ -374,13 +391,13 @@ extern bool cluster_pcm_clean_page_xfer_consume(void); extern void cluster_pcm_lock_unlock_content_buffer(BufferDesc *buf, PcmLockMode mode); extern void cluster_pcm_lock_release_saved_tag_for_eviction(BufferTag tag, PcmLockMode mode); extern void cluster_pcm_lock_release_buffer_for_eviction(BufferDesc *buf, PcmLockMode mode); -/* PGRAC: spec-5.2 D11 — local master records self as new X holder after a - * writer-transfer (the remote holder shipped + released its X). S3 forensics - * step 1b: the shipping holder + wire request_id/epoch ride along so the - * watermark-advance provenance table records WHO produced the advance. */ -extern void cluster_pcm_lock_master_take_x_after_transfer(BufferTag tag, XLogRecPtr page_lsn, - SCN page_scn, int32 holder_node, - uint64 request_id, uint64 epoch); +/* PGRAC: spec-5.2 D11 / P0-26 — commit a local-master X transfer only if the + * full remote-X authority sampled before the wire round is unchanged. The + * exact source identity/generation/session closes late-reply and concurrent + * queue-handoff races; STALE is a fresh-request retry boundary. */ +extern PcmXTransferCommitResult cluster_pcm_lock_master_take_x_after_transfer( + BufferTag tag, const PcmAuthoritySnapshot *expected, XLogRecPtr page_lsn, SCN page_scn, + int32 holder_node, uint32 requester_procno, uint64 request_id, uint64 epoch); /* PGRAC: spec-5.2 D11 path B — master==holder==self ships its X image to a * remote requester and records the requester as the new X holder (single-phase * writer-transfer-revoke; caller drops self's copy no-wire before calling). @@ -422,6 +439,7 @@ extern void cluster_pcm_lock_downgrade(BufferTag tag, PcmLockMode target_mode, b */ extern PcmLockMode cluster_pcm_lock_query(BufferTag tag); extern bool cluster_pcm_lock_authority_snapshot(BufferTag tag, PcmAuthoritySnapshot *out); +extern bool cluster_pcm_lock_authority_matches(BufferTag tag, const PcmAuthoritySnapshot *expected); extern PcmXGrdHandoffResult cluster_pcm_lock_queue_handoff_x_exact(const PcmXGrdHandoffToken *token); extern int cluster_pcm_grd_count(void); @@ -451,6 +469,9 @@ extern void cluster_pcm_grd_init(void); extern bool cluster_pcm_transition_legal(PcmState from, PcmState to, PcmLockTransition trans); extern void cluster_pcm_transition_apply(struct GrdEntry *entry, PcmLockTransition trans, int holder_node_id); +extern PcmGcsTransitionApplyResult +cluster_pcm_lock_apply_gcs_transition_result(BufferTag tag, PcmLockTransition trans, + int holder_node_id); extern bool cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, int holder_node_id); diff --git a/src/include/cluster/cluster_pcm_own.h b/src/include/cluster/cluster_pcm_own.h index 158d6477c3..cfe8bbc14c 100644 --- a/src/include/cluster/cluster_pcm_own.h +++ b/src/include/cluster/cluster_pcm_own.h @@ -63,11 +63,14 @@ typedef struct ClusterPcmOwnEntry { pg_atomic_uint64 generation; /* monotone; bumped on every committed transition */ pg_atomic_uint64 reservation_token; /* monotone; active iff a transient flag is set */ + pg_atomic_uint64 writer_activation_token; /* committed X not yet activated under content X */ pg_atomic_uint32 flags; /* PCM_OWN_FLAG_* */ - uint32 _pad; /* keep 24B aligned */ + uint32 _pad; /* keep 32B aligned */ } ClusterPcmOwnEntry; -StaticAssertDecl(sizeof(ClusterPcmOwnEntry) == 24, "ClusterPcmOwnEntry must remain 24 bytes"); +StaticAssertDecl(sizeof(ClusterPcmOwnEntry) == 32, "ClusterPcmOwnEntry must remain 32 bytes"); +StaticAssertDecl(offsetof(ClusterPcmOwnEntry, writer_activation_token) == 16, + "writer activation fence offset must remain stable"); typedef enum ClusterPcmOwnResult { CLUSTER_PCM_OWN_OK = 0, @@ -130,6 +133,16 @@ extern ClusterPcmOwnResult cluster_pcm_own_revoke_commit_exact(int buf_id, uint64 expected_generation, uint64 reservation_token, uint64 *out_committed_generation); +/* X-only combined linearization: clear the exact GRANT_PENDING lifecycle, + * bump its ownership generation, and publish the writer activation fence + * before the caller releases the BufferDesc header lock. */ +extern ClusterPcmOwnResult cluster_pcm_own_writer_grant_commit_exact( + int buf_id, uint64 expected_generation, uint64 reservation_token, + uint64 *out_committed_generation); +/* Clear only the exact committed generation/token after content-EXCLUSIVE + * activation proof or after exact queue-claim cleanup. */ +extern ClusterPcmOwnResult cluster_pcm_own_writer_activation_clear_exact( + int buf_id, uint64 expected_generation, uint64 reservation_token); /* PCM-X retained-image revoke: commit advances the ownership generation but * deliberately leaves the exact REVOKING token live until DRAIN proves the * immutable image record is no longer needed. Release clears only that @@ -175,6 +188,14 @@ cluster_pcm_own_reservation_token_get(int buf_id) return pg_atomic_read_u64(&ClusterPcmOwnArray[buf_id].reservation_token); } +static inline uint64 +cluster_pcm_own_writer_activation_token_get(int buf_id) +{ + if (ClusterPcmOwnArray == NULL || buf_id < 0) + return 0; + return pg_atomic_read_u64(&ClusterPcmOwnArray[buf_id].writer_activation_token); +} + static inline void cluster_pcm_own_flags_apply(int buf_id, uint32 set, uint32 clear) { diff --git a/src/include/cluster/cluster_pcm_x_bufmgr.h b/src/include/cluster/cluster_pcm_x_bufmgr.h index 9c0a58e330..15e544e526 100644 --- a/src/include/cluster/cluster_pcm_x_bufmgr.h +++ b/src/include/cluster/cluster_pcm_x_bufmgr.h @@ -21,6 +21,7 @@ #ifndef CLUSTER_PCM_X_BUFMGR_H #define CLUSTER_PCM_X_BUFMGR_H +#include "access/transam.h" #include "access/xlogdefs.h" #include "cluster/cluster_pcm_own.h" #include "cluster/cluster_pcm_x_convert.h" @@ -33,6 +34,17 @@ * becomes a transaction ERROR merely because RETIRE owns the short gate. */ #define CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS 5 +/* FSM pages are advisory and may be consumed without a content lock, so they + * are outside PCM/PCM-X ownership. Every other fork keeps the relation-level + * user/shared-catalog tracking policy. */ +static inline bool +cluster_pcm_x_buffer_tag_tracked(const BufferTag *tag, bool shared_catalog) +{ + if (tag == NULL || tag->forkNum == FSM_FORKNUM) + return false; + return shared_catalog || tag->relNumber >= (RelFileNumber)FirstNormalObjectId; +} + typedef enum ClusterPcmXHolderRetryAction { CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE = 0, CLUSTER_PCM_X_HOLDER_RETRY_WAIT, @@ -74,7 +86,8 @@ cluster_pcm_x_holder_register_retry_action(PcmXQueueResult result, bool runtime_ if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) return CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE; if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BARRIER_CLOSED - || (result == PCM_X_QUEUE_NOT_READY && runtime_active)) + || (runtime_active + && (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY))) return CLUSTER_PCM_X_HOLDER_RETRY_WAIT; return CLUSTER_PCM_X_HOLDER_RETRY_FAIL; } @@ -113,12 +126,50 @@ typedef struct ClusterPcmOwnSnapshot { BufferTag tag; uint64 generation; uint64 reservation_token; + /* Diagnostic projection of the grant->content activation fence. It is + * sampled under the same BufferDesc header lock as the authoritative + * ownership tuple, but is not part of snapshot matching or wire state. */ + uint64 writer_activation_token; uint32 flags; uint8 pcm_state; uint8 _reserved[3]; } ClusterPcmOwnSnapshot; -StaticAssertDecl(sizeof(ClusterPcmOwnSnapshot) == 48, "ClusterPcmOwnSnapshot must remain 48 bytes"); +StaticAssertDecl(sizeof(ClusterPcmOwnSnapshot) == 56, "ClusterPcmOwnSnapshot must remain 56 bytes"); + +/* + * Process-local evidence for a reversible finish-revoke refusal. The DATA + * worker fills this from the same header-lock sample that chose BUSY, so GCS + * can correlate the exact ticket/image with the actual VM/FSM pin, I/O, or + * ownership-lifecycle branch. This is diagnostics only; it is never sent on + * wire or persisted. + */ +typedef enum ClusterPcmOwnFinishRefusalReason { + CLUSTER_PCM_OWN_FINISH_REFUSAL_NONE = 0, + CLUSTER_PCM_OWN_FINISH_REFUSAL_VM_FSM_PINNED, + CLUSTER_PCM_OWN_FINISH_REFUSAL_IO_IN_PROGRESS, + CLUSTER_PCM_OWN_FINISH_REFUSAL_LIVE_FLAGS +} ClusterPcmOwnFinishRefusalReason; + +typedef struct ClusterPcmOwnFinishRefusal { + uint64 live_token; + uint32 shared_refcount; + uint32 live_flags; + ClusterPcmOwnFinishRefusalReason reason; + bool bm_io_in_progress; +} ClusterPcmOwnFinishRefusal; + +/* Process-local reason for a retryable S-source image preparation refusal. + * DIRTY_FLUSHED records forward progress; the other values identify the + * exact nonblocking gate that asked the image pump to retry. */ +typedef enum ClusterPcmOwnSourcePrepareRefusal { + CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_NONE = 0, + CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_BEGIN_REVOKE, + CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_CONTENT_LOCK, + CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_DIRTY_FLUSHED, + CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_DIRTY_RACED, + CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_IO_IN_PROGRESS +} ClusterPcmOwnSourcePrepareRefusal; /* The queue returns execution authority before bufmgr takes content EXCLUSIVE. * Bind that claim to the one committed ownership generation and require the @@ -428,12 +479,20 @@ cluster_bufmgr_pcm_own_abort_n_revoke(BufferDesc *buf, extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_begin_s_revoke(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_s, ClusterPcmOwnSnapshot *out_revoking); +/* Freeze one exact S source and choose the newest safe bytes from its clean + * current image and verified shared storage. A valid required_page_scn is a + * hard floor; V1's zero floor still prefers the newer observed copy. */ +extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_prepare_s_source_image( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_s, SCN required_page_scn, + ClusterPcmOwnSnapshot *out_revoking, char block_data[BLCKSZ], + XLogRecPtr *out_page_lsn, uint64 *out_page_scn, + ClusterPcmOwnSourcePrepareRefusal *out_refusal); extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_abort_s_revoke(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking); extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_finish_revoke_retain( BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, XLogRecPtr expected_lsn, - ClusterPcmOwnSnapshot *out_retained); + ClusterPcmOwnSnapshot *out_retained, ClusterPcmOwnFinishRefusal *out_refusal); extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, uint64 source_generation); /* Process-local descriptor evidence captured by the DRAIN-time probe below diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h index 39a1efd3e7..7f4d836324 100644 --- a/src/include/cluster/cluster_pcm_x_convert.h +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -33,7 +33,7 @@ #define PCM_X_SHMEM_REGION_NAME "pgrac cluster pcm convert queue" #define PCM_X_SHMEM_MAGIC ((uint32)0x50435851) /* "PCXQ" */ /* 13: A' rebase slot growth (ticket 392 / local tag 760 / header stats). */ -#define PCM_X_SHMEM_LAYOUT_VERSION ((uint32)13) +#define PCM_X_SHMEM_LAYOUT_VERSION ((uint32)14) #define PCM_X_INVALID_SLOT_INDEX ((Size) - 1) #define PCM_X_LOCK_PARTITIONS NUM_BUFFER_PARTITIONS #define PCM_X_LWLOCK_COUNT (1 + 2 * PCM_X_LOCK_PARTITIONS) @@ -194,6 +194,13 @@ typedef struct PcmXRevokePayload { uint64 image_id; } PcmXRevokePayload; +/* P0-20 source-floor extension. V1 remains the byte-exact 96-byte prefix; + * V2 is sent only to a connection that advertised SOURCE_FLOOR_V1. */ +typedef struct PcmXRevokePayloadV2 { + PcmXRevokePayload v1; + uint64 required_page_scn; +} PcmXRevokePayloadV2; + typedef struct PcmXGrantPayload { PcmXTicketRef ref; PcmXImageToken image; @@ -259,6 +266,9 @@ StaticAssertDecl(sizeof(PcmXPrehandleCancelPayload) == 80, "PCM-X pre-handle CAN StaticAssertDecl(sizeof(PcmXAdmitAckPayload) == 112, "PCM-X ADMIT_ACK ABI"); StaticAssertDecl(sizeof(PcmXPhasePayload) == 96, "PCM-X phase ABI"); StaticAssertDecl(sizeof(PcmXRevokePayload) == 96, "PCM-X revoke ABI"); +StaticAssertDecl(sizeof(PcmXRevokePayloadV2) == 104, "PCM-X revoke V2 ABI"); +StaticAssertDecl(offsetof(PcmXRevokePayloadV2, required_page_scn) == sizeof(PcmXRevokePayload), + "PCM-X revoke V2 preserves the complete V1 prefix"); StaticAssertDecl(sizeof(PcmXGrantPayload) == 128, "PCM-X grant ABI"); StaticAssertDecl(sizeof(PcmXInstallReadyPayload) == 112, "PCM-X INSTALL_READY ABI"); StaticAssertDecl(offsetof(PcmXInstallReadyPayload, rebased_own_generation) @@ -440,6 +450,25 @@ typedef enum PcmXQueueResult { PCM_X_QUEUE_BARRIER_CLOSED } PcmXQueueResult; +/* Process-local classification for the holder-side IMAGE_READY arm. This is + * diagnostic evidence only: the queue result remains the protocol verdict and + * no value from this enum is persisted in shared memory or sent on the wire. */ +typedef enum PcmXLocalImageReadyRefusal { + PCM_X_LOCAL_IMAGE_READY_REFUSAL_NONE = 0, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_INVALID, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_RUNTIME, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_DIRECTORY, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_TAG_SLOT, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_LOCAL_GATE, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_RUNTIME_RECHECK, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_IDENTITY, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_ACTIVE_HOLDER, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_RELIABLE_LEG, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_BAD_PHASE, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_COUNTER, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_GATE_RELEASE +} PcmXLocalImageReadyRefusal; + /* A generation-exact type-47 replay can prove the blocker set DUPLICATE * after the ticket has already advanced beyond ACTIVE_PROBE. Its type-48 * ACK is still safe to replay; only structural corruption warrants closing @@ -613,6 +642,7 @@ StaticAssertDecl(sizeof(PcmXLocalProgress) == 248, "PCM-X local progress ABI"); typedef struct PcmXLocalHolderProgress { PcmXTicketRef ref; PcmXImageToken image; + uint64 required_page_scn; uint64 reliable_state_sequence; uint16 pending_opcode; uint16 last_response_opcode; @@ -624,7 +654,7 @@ typedef struct PcmXLocalHolderProgress { uint32 reserved; } PcmXLocalHolderProgress; -StaticAssertDecl(sizeof(PcmXLocalHolderProgress) == 160, "PCM-X local holder progress ABI"); +StaticAssertDecl(sizeof(PcmXLocalHolderProgress) == 168, "PCM-X local holder progress ABI"); typedef struct PcmXLocalCutoff { PcmXSlotRef tag_slot; @@ -992,6 +1022,7 @@ typedef struct PcmXLocalTagSlot { uint64 committed_own_generation; PcmXTicketRef holder_ref; PcmXImageToken holder_image; + uint64 holder_required_page_scn; PcmXReliableLegState holder_reliable; uint64 holder_terminal_drain_generation; PcmXTicketRef blocker_snapshot_ref; @@ -1036,8 +1067,8 @@ StaticAssertDecl(sizeof(PcmXMasterTicketSlot) == 392, "PCM-X master ticket slot StaticAssertDecl(offsetof(PcmXMasterTicketSlot, grant_base_own_generation) == 384, "PCM-X master ticket grant-base offset"); StaticAssertDecl(sizeof(PcmXBlockerSlot) == 128, "PCM-X blocker slot ABI"); -StaticAssertDecl(sizeof(PcmXLocalTagSlot) == 760, "PCM-X local tag slot ABI"); -StaticAssertDecl(offsetof(PcmXLocalTagSlot, grant_base_own_generation) == 752, +StaticAssertDecl(sizeof(PcmXLocalTagSlot) == 768, "PCM-X local tag slot ABI"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, grant_base_own_generation) == 760, "PCM-X local tag grant-base offset"); StaticAssertDecl(offsetof(PcmXLocalTagSlot, membership_count) == 384, "PCM-X local membership count offset"); @@ -1050,13 +1081,15 @@ StaticAssertDecl(offsetof(PcmXLocalTagSlot, committed_own_generation) == 408, StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_ref) == 416, "PCM-X local holder ticket offset"); StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_image) == 504, "PCM-X local holder image offset"); -StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_reliable) == 544, +StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_required_page_scn) == 544, + "PCM-X local holder source-floor offset"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_reliable) == 552, "PCM-X local holder reliable-leg offset"); -StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_terminal_drain_generation) == 600, +StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_terminal_drain_generation) == 608, "PCM-X local holder terminal drain generation offset"); -StaticAssertDecl(offsetof(PcmXLocalTagSlot, blocker_snapshot_ref) == 608, +StaticAssertDecl(offsetof(PcmXLocalTagSlot, blocker_snapshot_ref) == 616, "PCM-X local blocker snapshot ticket offset"); -StaticAssertDecl(offsetof(PcmXLocalTagSlot, blocker_snapshot_reliable) == 696, +StaticAssertDecl(offsetof(PcmXLocalTagSlot, blocker_snapshot_reliable) == 704, "PCM-X local blocker snapshot reliable-leg offset"); StaticAssertDecl(sizeof(PcmXLocalMembershipSlot) == 168, "PCM-X local membership slot ABI"); StaticAssertDecl(offsetof(PcmXLocalMembershipSlot, admitted_round) == 160, @@ -1432,6 +1465,9 @@ extern PcmXQueueResult cluster_pcm_x_master_commit_retry_exact(const PcmXMasterDriveSnapshot *expected, uint64 now_ms, uint64 next_retry_deadline_ms, PcmXPhasePayload *commit_out, PcmXMasterDriveSnapshot *snapshot_out); +extern PcmXQueueResult cluster_pcm_x_master_invalidate_busy_backoff_exact( + const PcmXMasterDriveSnapshot *expected, int32 busy_node, uint64 now_ms, + uint64 retry_delay_ms, PcmXMasterDriveSnapshot *snapshot_out); typedef bool (*PcmXStageFrameCallback)(uint8 msg_type, int32 dest_node_id, const void *payload, Size payload_len, void *callback_arg); extern PcmXQueueResult @@ -1603,9 +1639,18 @@ extern PcmXQueueResult cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, int32 authenticated_master_node, uint64 authenticated_master_session); +extern PcmXQueueResult +cluster_pcm_x_local_holder_revoke_apply_floor_exact(const PcmXRevokePayload *revoke, + uint64 required_page_scn, + int32 authenticated_master_node, + uint64 authenticated_master_session); extern PcmXQueueResult cluster_pcm_x_local_holder_image_ready_arm_exact( const PcmXGrantPayload *image_ready, int32 authenticated_master_node, uint64 authenticated_master_session, PcmXGrantPayload *replay_out); +extern PcmXQueueResult cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( + const PcmXGrantPayload *image_ready, int32 authenticated_master_node, + uint64 authenticated_master_session, PcmXGrantPayload *replay_out, + PcmXLocalImageReadyRefusal *refusal_out); extern PcmXQueueResult cluster_pcm_x_local_enqueue_arm_exact(const PcmXLocalHandle *leader, PcmXEnqueuePayload *payload_out, PcmXLocalReliableToken *token_out); diff --git a/src/include/cluster/cluster_pcm_x_image_fetch.h b/src/include/cluster/cluster_pcm_x_image_fetch.h index d86b18f170..c7ada654b7 100644 --- a/src/include/cluster/cluster_pcm_x_image_fetch.h +++ b/src/include/cluster/cluster_pcm_x_image_fetch.h @@ -24,6 +24,18 @@ #ifdef USE_PGRAC_CLUSTER +typedef enum PcmXImageFetchRequestRefusal { + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_NONE = 0, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_ARGUMENT, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_ENVELOPE, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_REQUEST, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_MASTER, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_REF, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_LEG, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_IMAGE, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_REQUESTER_ID +} PcmXImageFetchRequestRefusal; + extern bool cluster_pcm_x_image_fetch_build_request(const PcmXLocalProgress *progress, int32 requester_node, int32 requester_backend_id, @@ -33,6 +45,10 @@ extern bool cluster_pcm_x_image_fetch_request_exact(const ClusterICEnvelope *env const PcmXLocalHolderProgress *holder, int32 holder_node, int32 current_master_node, uint64 current_epoch); +extern bool cluster_pcm_x_image_fetch_request_exact_diagnosed( + const ClusterICEnvelope *env, const GcsBlockRequestPayload *request, + const PcmXLocalHolderProgress *holder, int32 holder_node, int32 current_master_node, + uint64 current_epoch, PcmXImageFetchRequestRefusal *refusal_out); extern bool cluster_pcm_x_image_fetch_reply_exact(const GcsBlockReplyHeader *reply, const char *block_data, const PcmXLocalProgress *progress, diff --git a/src/include/cluster/cluster_sf_dep.h b/src/include/cluster/cluster_sf_dep.h index 5d0f61d4a0..286bd470fa 100644 --- a/src/include/cluster/cluster_sf_dep.h +++ b/src/include/cluster/cluster_sf_dep.h @@ -124,6 +124,19 @@ cluster_sf_peer_cap_family_sample(const ClusterSfPeerCap *cap, uint32 required_b return true; } +/* Exact pre-send fence for a capability-bound frame. Both the required + * bits and generation are read from one immutable record snapshot by the + * caller holding the capability-store lock. */ +static inline bool +cluster_sf_peer_cap_generation_matches_exact(const ClusterSfPeerCap *cap, uint32 required_bits, + uint32 expected_generation) +{ + uint32 observed_generation; + + return cluster_sf_peer_cap_generation_for_bits(cap, required_bits, &observed_generation) + && observed_generation == expected_generation; +} + /* Returns true iff the record was live for exactly this generation and got * cleared; a mismatch (older/newer generation, already invalid) is a no-op. */ static inline bool @@ -258,6 +271,12 @@ extern bool cluster_sf_peer_supports_xid_authority_flock(int32 peer_id); extern bool cluster_sf_peer_supports_gcs_inval_busy(int32 peer_id); extern bool cluster_sf_peer_supports_pcm_x_convert(int32 peer_id); extern bool cluster_sf_peer_supports_pcm_x_rebase(int32 peer_id); +extern bool cluster_sf_peer_supports_pcm_x_source_floor(int32 peer_id); +extern bool cluster_sf_peer_pcm_x_source_floor_sample(int32 peer_id, bool *source_floor_out, + uint32 *generation_out); +extern bool cluster_sf_peer_capability_generation_matches(int32 peer_id, + uint32 required_capabilities, + uint32 expected_generation); /* review P0-2: lock-coherent (CONVERT supported, REBASE bit, record * generation) triple for the formation collector's double sample. */ extern bool cluster_sf_peer_pcm_x_capability_sample(int32 peer_id, bool *rebase_out, diff --git a/src/include/cluster/cluster_tt_local.h b/src/include/cluster/cluster_tt_local.h index ffd051ec5e..fc3286eb67 100644 --- a/src/include/cluster/cluster_tt_local.h +++ b/src/include/cluster/cluster_tt_local.h @@ -39,6 +39,7 @@ #include "c.h" #include "access/transam.h" +#include "cluster/cluster_itl_slot.h" /* UBA */ #include "cluster/cluster_scn.h" /* SCN */ #include "cluster/cluster_tt_status.h" /* ClusterTTStatus */ @@ -107,6 +108,15 @@ extern void cluster_tt_local_record_abort(TransactionId xid); */ extern void cluster_tt_local_record_active(TransactionId xid); +/* + * Publish ACTIVE for an ordinary heap data write using the exact UBA carried + * by the page's ITL slot. Record-extent rollover can make the UBA's record + * segment differ from the transaction binding's canonical TT segment; this + * entry point registers that exact-key alias so commit/abort later publish the + * same terminal status to it. `uba` must carry the binding's slot offset. + */ +extern void cluster_tt_local_record_data_active(TransactionId xid, UBA uba); + /* * Counters / introspection used by test_cluster_tt_status (D9 T17). */ diff --git a/src/include/cluster/cluster_visibility_resolve.h b/src/include/cluster/cluster_visibility_resolve.h index 8efec94a65..50dfb0bbd8 100644 --- a/src/include/cluster/cluster_visibility_resolve.h +++ b/src/include/cluster/cluster_visibility_resolve.h @@ -184,6 +184,10 @@ extern ClusterVisVerdict cluster_vis_self_verdict(ClusterTTStatus status); extern ClusterVisVerdict cluster_vis_toast_verdict(ClusterTTStatus status); /* OBS-2 Update: xmin gate then xmax outcome. */ +extern bool cluster_vis_xmin_needs_resolution(uint16 infomask); + +/* P0-28: local GlobalVis is not a shared-storage pruning horizon. */ +extern bool cluster_vis_prune_must_defer(bool storage_mode, bool cluster_horizon_available); extern ClusterVisVerdict cluster_vis_update_xmin_verdict(ClusterTTStatus status); extern ClusterVisVerdict cluster_vis_update_xmax_verdict(ClusterTTStatus status, bool is_delete); diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index 0651600fe3..d8f96d0211 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -57,7 +57,7 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '183', 'pg_stat_cluster_injections returns 183 rows (branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-writer-cached-x-stall + cluster-pcm-restore-aba-window + cluster-pcm-grant-finalize-window + cluster-pcm-grant-finalize-deliver-invalidate + cluster-pcm-drop-prepin-window + cluster-pcm-restore-aba-force-round; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 horizon report-drop/epoch-fence; spec-5.22d D4-8 +1 authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '184', 'pg_stat_cluster_injections returns 184 rows (P0-20 +1 cluster-pcm-x-retain-flush-error; branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-writer-cached-x-stall + cluster-pcm-restore-aba-window + cluster-pcm-grant-finalize-window + cluster-pcm-grant-finalize-deliver-invalidate + cluster-pcm-drop-prepin-window + cluster-pcm-restore-aba-force-round; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 horizon report-drop/epoch-fence; spec-5.22d D4-8 +1 authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- @@ -85,8 +85,8 @@ 'postgres', 'SELECT string_agg(name, \',\' ORDER BY name) FROM pg_stat_cluster_injections' ), - 'cluster-boc-event-publish,cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-done-drop,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-duplicate-grant-reply,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-fallback-refresh-stale,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-master-direct-fallback-storage-stale,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-stale-ship-resident,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-gcs-block-yield-notify-drop,cluster-gcs-xfer-copy-drop-window,cluster-ges-master-work-queue-full,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-conn-reset,cluster-lms-cr-construct,cluster-lms-cr-fence-recheck,cluster-lms-cr-fence-refuse,cluster-lms-data-dispatch,cluster-lms-undo-fetch,cluster-mxid-halfspace-hard-limit,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-drop-prepin-window,cluster-pcm-grant-finalize-deliver-invalidate,cluster-pcm-grant-finalize-window,cluster-pcm-release-pre,cluster-pcm-restore-aba-force-round,cluster-pcm-restore-aba-window,cluster-pcm-writer-cached-x-stall,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-undo-authority-block0-prove,cluster-undo-authority-scan,cluster-undo-horizon-epoch-fence,cluster-undo-horizon-report-drop,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', - '180 injection point names match the full registry (S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 cluster-undo-horizon-report-drop + cluster-undo-horizon-epoch-fence; spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + 'cluster-boc-event-publish,cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-done-drop,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-duplicate-grant-reply,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-fallback-refresh-stale,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-master-direct-fallback-storage-stale,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-stale-ship-resident,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-gcs-block-yield-notify-drop,cluster-gcs-xfer-copy-drop-window,cluster-ges-master-work-queue-full,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-conn-reset,cluster-lms-cr-construct,cluster-lms-cr-fence-recheck,cluster-lms-cr-fence-refuse,cluster-lms-data-dispatch,cluster-lms-undo-fetch,cluster-mxid-halfspace-hard-limit,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-drop-prepin-window,cluster-pcm-grant-finalize-deliver-invalidate,cluster-pcm-grant-finalize-window,cluster-pcm-release-pre,cluster-pcm-restore-aba-force-round,cluster-pcm-restore-aba-window,cluster-pcm-writer-cached-x-stall,cluster-pcm-x-retain-flush-error,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-undo-authority-block0-prove,cluster-undo-authority-scan,cluster-undo-horizon-epoch-fence,cluster-undo-horizon-report-drop,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', + '184 injection point names match the full registry (P0-20 +1 cluster-pcm-x-retain-flush-error; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 cluster-undo-horizon-report-drop + cluster-undo-horizon-epoch-fence; spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/017_debug.pl b/src/test/cluster_tap/t/017_debug.pl index 1f217dabb4..3f8fa9189c 100644 --- a/src/test/cluster_tap/t/017_debug.pl +++ b/src/test/cluster_tap/t/017_debug.pl @@ -128,15 +128,15 @@ 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.fault_type'}), - '183', - 'all 183 injection points have a .fault_type entry (branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '184', + 'all 184 injection points have a .fault_type entry (P0-20 +1 cluster-pcm-x-retain-flush-error; branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is( $node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.hits'}), - '183', - 'all 183 injection points have a .hits entry (branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '184', + 'all 184 injection points have a .hits entry (P0-20 +1 cluster-pcm-x-retain-flush-error; branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/030_acceptance.pl b/src/test/cluster_tap/t/030_acceptance.pl index 28f1736087..78788b9e8b 100644 --- a/src/test/cluster_tap/t/030_acceptance.pl +++ b/src/test/cluster_tap/t/030_acceptance.pl @@ -325,12 +325,12 @@ # ============================================================ -# §M error injection 112 注入点 + 5 fault types (6 tests) +# §M error injection 184 注入点 + 7 fault types (6 tests) # ============================================================ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '183', 'M1 183 injection points (branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '184', 'M1 184 injection points (P0-20 +1 cluster-pcm-x-retain-flush-error; branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql('postgres', q{SELECT string_agg(name, ',' ORDER BY name) FROM pg_stat_cluster_injections WHERE name LIKE 'cluster-init-%'}), @@ -360,8 +360,8 @@ 'postgres', q{SELECT count(DISTINCT key) FROM pg_cluster_state WHERE category='inject' AND (key LIKE '%.fault_type' OR key LIKE '%.hits')} - ) eq '366', - 'M5 inject category has 183×2 = 366 sub-keys (.fault_type + .hits; branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + ) eq '368', + 'M5 inject category has 184×2 = 368 sub-keys (.fault_type + .hits; P0-20 +1 cluster-pcm-x-retain-flush-error; branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->get_cluster_state_value('inject', 'armed_count'), '0', 'M6 inject.armed_count starts at 0 in fresh backend'); diff --git a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl index e2152aec0b..56014ac1ba 100644 --- a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl +++ b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl @@ -66,6 +66,7 @@ sub state_int ); my @pcm_x_lmd_keys = qw( + wait_edge_count pcm_convert_wfg_replace_count pcm_convert_wfg_remove_count pcm_convert_wfg_replace_fail_count @@ -188,6 +189,28 @@ sub wait_for_pcm_gauges_zero return (0, \@snapshots); } +sub wait_for_node_state_gt +{ + my ($node, $category, $key, $before, $timeout_seconds) = @_; + my $deadline = time() + $timeout_seconds; + my $last_value; + + do + { + my ($rc, $out, $err) = $node->psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='$category' AND key='$key'}, + timeout => 3); + if (defined($rc) && $rc == 0 && defined($out) && $out =~ /\A\d+\z/) + { + $last_value = $out + 0; + return (1, $last_value) if $last_value > $before; + } + usleep(100_000); + } while (time() < $deadline); + + return (0, $last_value); +} + my $warmup_error_count = 0; sub write_retry @@ -210,6 +233,30 @@ sub write_retry return 0; } +sub wait_for_lms_finish_flush_reload +{ + my ($node, $log_offset, $expected_workers, $expected_armed, + $expected_value, $timeout_seconds) = @_; + my $deadline = time() + $timeout_seconds; + my $last_log = ''; + + do + { + $last_log = substr(slurp_file($node->logfile), $log_offset); + my %ready_workers; + while ($last_log =~ /cluster_lms: DATA worker=(\d+) applied PCM-X finish Flush injection config: pid=\d+ armed=(true|false) value="([^"]*)"/g) + { + my ($worker_id, $armed, $value) = ($1, $2, $3); + next unless $armed eq $expected_armed && $value eq $expected_value; + $ready_workers{$worker_id} = 1; + } + return (1, $last_log) if scalar(keys %ready_workers) == $expected_workers; + usleep(100_000); + } while (time() < $deadline); + + return (0, $last_log); +} + sub write_file { my ($path, $contents) = @_; @@ -231,6 +278,7 @@ sub write_file 'cluster.xid_striping = on', 'cluster.crossnode_runtime_visibility = on', 'cluster.page_scn_shortcut = on', + 'cluster.past_image = on', 'cluster.crossnode_write_write = on', 'cluster.undo_gcs_coherence = on', 'cluster.crossnode_cr_data_plane = on', @@ -283,6 +331,14 @@ sub write_file CREATE TABLE pcm_xq_self ( id integer, v bigint NOT NULL + ) WITH (fillfactor = 100); + CREATE TABLE pcm_xq_dirty_retain ( + id integer, + v bigint NOT NULL + ) WITH (fillfactor = 100); + CREATE TABLE pcm_xq_flush_error ( + id integer, + v bigint NOT NULL ) WITH (fillfactor = 100) }); } @@ -310,7 +366,58 @@ sub write_file ok(write_retry($quad->node0, q{SELECT count(*) FROM pcm_xq_hot WHERE id BETWEEN 1 AND 4}), 'L2 seed owner installed committed visibility hints'); +my @l2s_checkpoint_log_offsets = map { (-s $quad->node($_)->logfile) // 0 } (0 .. 3); +# The hot main page is guaranteed dirty by seed/VACUUM and hashes to remote +# master node1 on shard1. That single consumed note deterministically proves +# the P0-24 DATA-plane route. The VM page also hashes to node3, but its +# checkpoint write-note is conditional on still holding PCM S/X at flush; N +# is legal, so waiting for node3 would assert a non-contractual side effect. +my %l2s_pi_durable_applied_before = map { + $_ => state_int($quad->node($_), 'gcs', 'pi_durable_note_apply_count') +} (1); +my @l2s_pi_route_before_by_node = map { + { + misroute => state_int($quad->node($_), 'gcs', + 'dedup_misroute_failclosed_count'), + requeue_drop => state_int($quad->node($_), 'lms', + 'lms_outbound_requeue_drop_count'), + } +} (0 .. 3); ok(write_retry($quad->node0, 'CHECKPOINT'), 'L2 seed checkpointed'); +for my $target (1) +{ + my ($applied, $after) = wait_for_node_state_gt( + $quad->node($target), 'gcs', 'pi_durable_note_apply_count', + $l2s_pi_durable_applied_before{$target}, 15); + ok($applied, + "L2S target node$target DATA worker consumed its checkpoint status-3 PI durable note") + or diag("L2S node$target pi_durable_note_apply_count before=" + . $l2s_pi_durable_applied_before{$target} . ' after=' + . (defined($after) ? $after : '')); +} +for my $i (0 .. 3) +{ + my ($checkpoint_probe_rc, $checkpoint_probe_out, $checkpoint_probe_err) + = $quad->node($i)->psql('postgres', 'SELECT 1', timeout => 5); + ok(defined($checkpoint_probe_rc) && $checkpoint_probe_rc == 0 + && defined($checkpoint_probe_out) && $checkpoint_probe_out eq '1', + "L2S node$i survived checkpoint PI durable-note routing") + or diag("L2S node$i checkpoint probe rc=" + . (defined($checkpoint_probe_rc) ? $checkpoint_probe_rc : '') + . " stdout=[" . ($checkpoint_probe_out // '') + . "] stderr=[" . ($checkpoint_probe_err // '') . "]"); + my $checkpoint_log = substr(slurp_file($quad->node($i)->logfile), + $l2s_checkpoint_log_offsets[$i]); + unlike($checkpoint_log, + qr/gcs block invalidate-ack misrouted|failed Assert\("tag_shard == recv_worker"\)/, + "L2S node$i checkpoint emitted no invalidate-ack shard violation"); + is(state_int($quad->node($i), 'gcs', 'dedup_misroute_failclosed_count'), + $l2s_pi_route_before_by_node[$i]{misroute}, + "L2S node$i checkpoint misroute counter stayed exact"); + is(state_int($quad->node($i), 'lms', 'lms_outbound_requeue_drop_count'), + $l2s_pi_route_before_by_node[$i]{requeue_drop}, + "L2S node$i checkpoint outbound requeue-drop counter stayed exact"); +} # Build a deterministic sole-requester S source before the four-writer leg. # The cache-off INSERT above releases node0's X to N at content-lock unlock. @@ -327,6 +434,14 @@ sub write_file 'pcm_x_self_handoff_count'); my $self_handoff_drain_before = state_int($quad->node0, 'gcs', 'pcm_x_self_handoff_drain_count'); +my @self_runtime_before_by_node = map { + { + runtime_state => state_int($quad->node($_), 'pcm', + 'pcm_x_runtime_state'), + recovery_blocked => state_int($quad->node($_), 'pcm', + 'pcm_x_queue_recovery_blocked_count'), + } +} (0 .. 3); my ($self_write_rc, $self_write_out, $self_write_err) = $quad->node0->psql( 'postgres', q{UPDATE pcm_xq_self SET v = v + 1 WHERE id = 1}, timeout => 30); @@ -353,10 +468,116 @@ sub write_file 'L2S sole-requester S source exercised the fused revoke-to-grant handoff'); cmp_ok($self_handoff_drain_after - $self_handoff_drain_before, '>', 0, 'L2S sole-requester S source released its immutable record at DRAIN'); +for my $i (0 .. 3) +{ + is(state_int($quad->node($i), 'pcm', 'pcm_x_runtime_state'), 1, + "L2S node$i PCM-X runtime remained ACTIVE across DRAIN/RETIRE"); + is(state_int($quad->node($i), 'pcm', + 'pcm_x_queue_recovery_blocked_count'), + $self_runtime_before_by_node[$i]{recovery_blocked}, + "L2S node$i recovery-blocked count did not advance at RETIRE"); +} is($quad->node0->safe_psql('postgres', q{SELECT v FROM pcm_xq_self WHERE id = 1}), '1', 'L2S sole-requester conversion preserved exact page contents'); +# Seed and vacuum one target page so node1's later INSERT obtains that exact +# heap page from the FSM and asks for X directly. An UPDATE would first read +# the row into a requester-local S mirror, making the authority selector take +# the intentional self-source path instead of the remote finish path tested +# here. The node0 UPDATE after this checkpoint is the sole dirty source. +ok(write_retry($quad->node0, + q{INSERT INTO pcm_xq_dirty_retain(id, v) VALUES (1, 0)}), + 'L2F seeded the direct-X target page'); +ok(write_retry($quad->node0, 'VACUUM pcm_xq_dirty_retain'), + 'L2F published target-page free space before the direct-X request'); +ok(write_retry($quad->node0, 'CHECKPOINT'), + 'L2F baseline checkpointed before the dirty retain leg'); + +# The source INSERT leaves an uncheckpointed dirty X image on node0. A +# different node must materialize that exact image and retain it only after +# FlushBuffer's caller-owned pin contract is satisfied. This is a behavioral +# gate: the immutable staging counter proves the DATA worker reached the dirty +# remote-source leg, while h_pi_write_note_count increments only after +# FlushBuffer's smgrwrite returned, proving the real flush completed. +my $dirty_stage_before = state_int($quad->node0, 'gcs', + 'dedup_pcm_x_stage_count'); +my $dirty_flush_before = state_int($quad->node0, 'xnode_lever', + 'h_pi_write_note_count'); +my $dirty_flush_log_offset = (-s $quad->node0->logfile) // 0; +my $dirty_requester_log_offset = (-s $quad->node1->logfile) // 0; +my $dirty_lms_workers = $quad->node0->safe_psql('postgres', + 'SHOW cluster.lms_workers') + 0; +my $dirty_requester_lms_workers = $quad->node1->safe_psql('postgres', + 'SHOW cluster.lms_workers') + 0; +$quad->node0->safe_psql('postgres', q{ + ALTER SYSTEM SET cluster.injection_points = 'cluster-pcm-x-retain-flush-error'; + SELECT pg_reload_conf() +}); +$quad->node1->safe_psql('postgres', q{ + ALTER SYSTEM SET cluster.injection_points = 'cluster-pcm-x-retain-flush-error'; + SELECT pg_reload_conf() +}); +my ($dirty_reload_ready, $dirty_reload_log) = wait_for_lms_finish_flush_reload( + $quad->node0, $dirty_flush_log_offset, $dirty_lms_workers, 'true', + 'cluster-pcm-x-retain-flush-error', 15); +ok($dirty_reload_ready, + 'L2F every node0 DATA worker applied the finish-Flush injection arm') + or diag("L2F DATA-worker reload log=[$dirty_reload_log]"); +my ($dirty_requester_reload_ready, $dirty_requester_reload_log) + = wait_for_lms_finish_flush_reload( + $quad->node1, $dirty_requester_log_offset, $dirty_requester_lms_workers, 'true', + 'cluster-pcm-x-retain-flush-error', 15); +ok($dirty_requester_reload_ready, + 'L2F every node1 DATA worker applied the transfer-boundary diagnostic arm') + or diag("L2F requester DATA-worker reload log=[$dirty_requester_reload_log]"); +my ($dirty_seed_rc, $dirty_seed_out, $dirty_seed_err) = $quad->node0->psql( + 'postgres', q{UPDATE pcm_xq_dirty_retain SET v = 1 WHERE id = 1}, timeout => 30); +is($dirty_seed_rc, 0, 'L2F node0 created an uncheckpointed dirty X source') + or diag("L2F dirty seed stdout=[$dirty_seed_out] stderr=[$dirty_seed_err]"); +my ($dirty_retain_rc, $dirty_retain_out, $dirty_retain_err) = $quad->node1->psql( + 'postgres', q{INSERT INTO pcm_xq_dirty_retain(id, v) VALUES (2, 1)}, timeout => 30); +is($dirty_retain_rc, 0, + 'L2F remote writer completed the dirty retain/flush lifecycle') + or diag("L2F dirty retain stdout=[$dirty_retain_out] stderr=[$dirty_retain_err]"); +cmp_ok(state_int($quad->node0, 'gcs', 'dedup_pcm_x_stage_count') + - $dirty_stage_before, '>', 0, + 'L2F source DATA worker materialized the immutable dirty image'); +cmp_ok(state_int($quad->node0, 'xnode_lever', 'h_pi_write_note_count') + - $dirty_flush_before, '>', 0, + 'L2F source FlushBuffer completed smgrwrite before ownership commit'); +my $dirty_flush_log = substr(slurp_file($quad->node0->logfile), + $dirty_flush_log_offset); +like($dirty_flush_log, + qr/cluster injection point "cluster-pcm-x-retain-flush-error" armed with WARNING/, + 'L2F DATA worker reached the finish-exclusive FlushBuffer injection seam'); +like($dirty_flush_log, + qr/cluster PCM-X retained-image finish FlushBuffer succeeded:/, + 'L2F finish-exclusive FlushBuffer returned after the physical write'); +is($quad->node1->safe_psql('postgres', + q{SELECT string_agg(id::text || ':' || v::text, ',' ORDER BY id) + FROM pcm_xq_dirty_retain}), '1:1,2:1', + 'L2F dirty retain preserved the exact page contents'); +$quad->node0->safe_psql('postgres', q{ + ALTER SYSTEM RESET cluster.injection_points; + SELECT pg_reload_conf() +}); +$quad->node1->safe_psql('postgres', q{ + ALTER SYSTEM RESET cluster.injection_points; + SELECT pg_reload_conf() +}); +my ($dirty_reset_ready, $dirty_reset_log) = wait_for_lms_finish_flush_reload( + $quad->node0, $dirty_flush_log_offset, $dirty_lms_workers, 'false', '', 15); +ok($dirty_reset_ready, + 'L2F every node0 DATA worker applied the finish-Flush injection disarm') + or diag("L2F DATA-worker reset log=[$dirty_reset_log]"); +my ($dirty_requester_reset_ready, $dirty_requester_reset_log) + = wait_for_lms_finish_flush_reload( + $quad->node1, $dirty_requester_log_offset, $dirty_requester_lms_workers, 'false', '', 15); +ok($dirty_requester_reset_ready, + 'L2F every node1 DATA worker applied the transfer-boundary diagnostic disarm') + or diag("L2F requester DATA-worker reset log=[$dirty_requester_reset_log]"); + for my $i (0 .. 3) { ok(write_retry($quad->node($i), @@ -379,6 +600,7 @@ sub write_file }); diag("L2 fixed hot-block tuple map: rel=$paths[0] tuples=$tuple_map"); +my @workload_log_offsets = map { (-s $quad->node($_)->logfile) // 0 } (0 .. 3); my @pcm_before_by_node = map { state_snapshot($quad->node($_), 'pcm', \@pcm_x_pcm_keys) } (0 .. 3); @@ -394,10 +616,14 @@ sub write_file my $passive_s_before = 0; $passive_s_before += state_int($quad->node($_), 'gcs', 'invalidate_passive_s_release_count') for (0 .. 3); +my @holder_evicted_before_by_node = map { + state_int($quad->node($_), 'gcs', 'block_forward_holder_evicted_count') +} (0 .. 3); my $start_at = $quad->node0->safe_psql('postgres', q{SELECT (clock_timestamp() + interval '5 seconds')::text}); my $script_dir = PostgreSQL::Test::Utils::tempdir(); my @runs; +my $data_worker_buffer_content_wait_samples = 0; for my $i (0 .. 3) { @@ -453,6 +679,15 @@ sub write_file timeout => 10); } // 'probe-failed'; $aux =~ s/\n/ | /g; + my $data_content_waits = eval { + $quad->node($i)->safe_psql('postgres', + q{SELECT count(*) FROM pg_stat_activity + WHERE backend_type = 'lms worker' + AND wait_event = 'BufferContent'}, + timeout => 10); + } // 0; + $data_worker_buffer_content_wait_samples += $data_content_waits + if $data_content_waits =~ /\A\d+\z/; my $wire = eval { $quad->node($i)->safe_psql('postgres', q{SELECT string_agg('peer' || node_id || ':s' || msg_send_count @@ -476,6 +711,8 @@ sub write_file diag("L3 mid-leg t+$offset node$i waits=[$waits]" . ($samples[$offset][$i] ? '' : ' snapshot-failed')); diag("L3 mid-leg t+$offset node$i aux=[$aux]"); + diag("L3 mid-leg t+$offset node$i DATA BufferContent waits=" + . $data_content_waits); diag("L3 mid-leg t+$offset node$i wire=[$wire]"); diag("L3 mid-leg t+$offset node$i slots=[$slots]"); } @@ -540,12 +777,21 @@ sub write_file my $passive_s_after = 0; $passive_s_after += state_int($quad->node($_), 'gcs', 'invalidate_passive_s_release_count') for (0 .. 3); +my @holder_evicted_after_by_node = map { + state_int($quad->node($_), 'gcs', 'block_forward_holder_evicted_count') +} (0 .. 3); diag('L3 path probes: pcm_x_queue_enqueue_delta=' . ($queue_after - $queue_before) . ' legacy_denied_pending_x_delta=' . ($denied_after - $denied_before) . ' passive_s_release_delta=' . ($passive_s_after - $passive_s_before)); +for my $i (0 .. 3) +{ + diag("L3 node$i holder copy refusal baseline=$holder_evicted_before_by_node[$i]" + . " final=$holder_evicted_after_by_node[$i] delta=" + . ($holder_evicted_after_by_node[$i] - $holder_evicted_before_by_node[$i])); +} # Per-node runtime probe: the aggregate sums above cannot distinguish which # node fused. Name the fused node and its fail-closed arm (file:line). @@ -564,6 +810,20 @@ sub write_file . ($pcm_after_by_node[$i]{pcm_x_queue_recovery_blocked_count} - $pcm_before_by_node[$i]{pcm_x_queue_recovery_blocked_count}) . " fail_closed_site=[$site]"); + is($pcm_after_by_node[$i]{pcm_x_runtime_state}, 1, + "L3 node$i PCM-X runtime remained ACTIVE"); + is($pcm_after_by_node[$i]{pcm_x_runtime_generation}, + $pcm_before_by_node[$i]{pcm_x_runtime_generation}, + "L3 node$i PCM-X runtime generation stayed exact"); + is($site, '(none)', "L3 node$i PCM-X fail-closed site stayed empty"); + is($lmd_after_by_node[$i]{wait_edge_count}, + $lmd_before_by_node[$i]{wait_edge_count}, + "L3 node$i WFG live edge count returned to baseline"); + for my $key (@pcm_x_final_gauge_keys) + { + is($pcm_after_by_node[$i]{$key}, 0, + "L3 node$i final PCM-X gauge $key is zero"); + } } for my $key (@pcm_x_pcm_keys) { @@ -583,6 +843,7 @@ sub write_file WHERE category = 'gcs' AND key IN ( 'block_master_not_holder_count', + 'block_forward_holder_evicted_count', 'block_x_self_ship_count', 'drop_pinned_deny_count', 'x_vs_s_no_carrier_denied_count', @@ -661,6 +922,9 @@ sub write_file is($runs[$i]->{errors}, 0, "L3 node$i writer surfaced zero client errors"); cmp_ok($runs[$i]->{transactions}, '>', 0, "L3 node$i writer made progress"); + cmp_ok($pcm_after_by_node[$i]{pcm_x_queue_wait_count} + - $pcm_before_by_node[$i]{pcm_x_queue_wait_count}, '>', 0, + "L3 node$i requester identity entered the queue wait lifecycle"); } # Queue counters live on the static tag master, not on each requester. The @@ -668,7 +932,7 @@ sub write_file # four requesters traversed the protocol; requiring a local enqueue delta on # non-master nodes is a false topology assumption. cmp_ok($queue_after - $queue_before, '>=', 4, - 'L3 all four node writers entered the PCM-X queue protocol'); + 'L3 aggregate PCM-X queue admission floor reached four'); cmp_ok($denied_after - $denied_before, '>', 0, 'L3 Shape-B arbitration denied an in-flight legacy reader without a client error'); cmp_ok($passive_s_after - $passive_s_before, '>', 0, @@ -701,12 +965,32 @@ sub write_file is($lmd_after{pcm_convert_wfg_exact_remove_stale_count} - $lmd_before{pcm_convert_wfg_exact_remove_stale_count}, 0, 'L3 PCM-X WFG exact-remove stale identities stayed zero'); +is($data_worker_buffer_content_wait_samples, 0, + 'L3 DATA workers never waited on BufferContent while receive progress depended on them'); ok($gauges_drained, 'L3 PCM-X terminal gauges drained within 30 seconds'); for my $key (@pcm_x_final_gauge_keys) { is($pcm_after{$key}, 0, "L3 final aggregate PCM-X gauge $key is zero"); } +my $successful_transactions = 0; +$successful_transactions += $runs[$_]->{transactions} for (0 .. 3); +my $stale_miss_delta + = ($pcm_after{pcm_x_queue_stale_count} - $pcm_before{pcm_x_queue_stale_count}) + + ($pcm_after{pcm_x_queue_miss_count} - $pcm_before{pcm_x_queue_miss_count}); +cmp_ok($stale_miss_delta, '<=', 4 * $successful_transactions + 16, + 'L3 retryable stale/miss churn stayed within the per-transaction budget'); + +for my $i (0 .. 3) +{ + my $log = slurp_file($quad->node($i)->logfile); + my $workload_log = substr($log, $workload_log_offsets[$i]); + unlike($workload_log, qr/\b(?:FATAL|PANIC):/, + "L3 node$i log contains no FATAL or PANIC"); + unlike($workload_log, qr/owner-plane[^\n]*(?:violation|corrupt|fail(?:ed|ure)?)/i, + "L3 node$i log contains no owner-plane violation"); +} + my ($advanced_rc, $advanced, $advanced_err) = $quad->node0->psql('postgres', q{ SELECT string_agg(id::text || ':' || row_count::text || ':' || total_v::text, ',' ORDER BY id) @@ -734,5 +1018,94 @@ sub write_file 'L4 aggregate value equals total committed pgbench transactions') or diag("L4 expected_sum=$expected_sum stderr=[$sum_err]"); +# The destructive leg is deliberately last: its expected outcome is a +# fail-closed runtime, so no later assertion may depend on ACTIVE or drained +# gauges. GUC+reload is required because injection state is process-local; +# the DATA worker, not this SQL backend, executes the finish boundary. +ok(write_retry($quad->node0, + q{INSERT INTO pcm_xq_flush_error(id, v) VALUES (1, 0)}), + 'L5F seeded the direct-X target page'); +ok(write_retry($quad->node0, 'VACUUM pcm_xq_flush_error'), + 'L5F published target-page free space before the direct-X request'); +ok(write_retry($quad->node0, 'CHECKPOINT'), + 'L5F checkpointed before the destructive finish-Flush test'); +my $flush_error_relfilenode = $quad->node0->safe_psql('postgres', + q{SELECT pg_relation_filenode('pcm_xq_flush_error'::regclass)}) + 0; +my $flush_error_stage_before = state_int($quad->node0, 'gcs', + 'dedup_pcm_x_stage_count'); +my $flush_error_release_before = state_int($quad->node0, 'gcs', + 'dedup_pcm_x_release_count'); +my $flush_error_blocked_before = state_int($quad->node0, 'pcm', + 'pcm_x_queue_recovery_blocked_count'); +my $flush_error_log_offset = (-s $quad->node0->logfile) // 0; + +$quad->node0->safe_psql('postgres', q{ + ALTER SYSTEM SET cluster.injection_points = + 'cluster-pcm-x-retain-flush-error:skipn:1'; + SELECT pg_reload_conf() +}); +my ($flush_error_reload_ready, $flush_error_reload_log) + = wait_for_lms_finish_flush_reload( + $quad->node0, $flush_error_log_offset, $dirty_lms_workers, 'true', + 'cluster-pcm-x-retain-flush-error:skipn:1', 15); +ok($flush_error_reload_ready, + 'L5F every node0 DATA worker applied the one-shot finish-Flush fault arm') + or diag("L5F DATA-worker reload log=[$flush_error_reload_log]"); + +my ($flush_seed_rc, $flush_seed_out, $flush_seed_err) = $quad->node0->psql( + 'postgres', q{UPDATE pcm_xq_flush_error SET v = 1 WHERE id = 1}, timeout => 30); +is($flush_seed_rc, 0, 'L5F node0 created the destructive-leg X source') + or diag("L5F seed stdout=[$flush_seed_out] stderr=[$flush_seed_err]"); +my ($flush_error_rc, $flush_error_out, $flush_error_err) = $quad->node1->psql( + 'postgres', q{ + SET statement_timeout = '5s'; + INSERT INTO pcm_xq_flush_error(id, v) VALUES (2, 1) + }, timeout => 30); +isnt($flush_error_rc, 0, + 'L5F remote writer failed when finish FlushBuffer raised the injected ERROR') + or diag("L5F unexpected success stdout=[$flush_error_out] stderr=[$flush_error_err]"); + +my ($flush_error_runtime_after, $flush_error_blocked_after); +my $flush_error_deadline = time() + 15; +while (1) +{ + $flush_error_runtime_after = state_int($quad->node0, 'pcm', + 'pcm_x_runtime_state'); + $flush_error_blocked_after = state_int($quad->node0, 'pcm', + 'pcm_x_queue_recovery_blocked_count'); + last if $flush_error_runtime_after == 0 + && $flush_error_blocked_after > $flush_error_blocked_before; + last if time() >= $flush_error_deadline; + usleep(100_000); +} + +is($quad->node0->safe_psql('postgres', 'SELECT 1'), '1', + 'L5F node0 postmaster remained alive after the DATA-worker ERROR'); +is($flush_error_runtime_after, 0, + 'L5F node0 PCM-X runtime moved to RECOVERY_BLOCKED'); +cmp_ok($flush_error_blocked_after - $flush_error_blocked_before, '>', 0, + 'L5F recovery-blocked counter recorded the finish error'); +cmp_ok(state_int($quad->node0, 'gcs', 'dedup_pcm_x_stage_count') + - $flush_error_stage_before, '>', 0, + 'L5F immutable A-record reached MATERIALIZED_UNCOMMITTED before ERROR'); +my $flush_error_log = substr(slurp_file($quad->node0->logfile), + $flush_error_log_offset); +like($flush_error_log, + qr/PCM-X finish-error evidence exact.*?preserve_result=12 retained=true worker=\d+ tag=\d+\/\d+\/\Q$flush_error_relfilenode\E\/0\/0 requester=1 backend=\d+ request_id=\d+ ticket=\d+ queue_generation=\d+ grant_generation=\d+ image_id=\d+ reservation_token=\d+ source_state=\d+/s, + 'L5F exact immutable A-record remains retained after ERROR'); +is(state_int($quad->node0, 'gcs', 'dedup_pcm_x_release_count'), + $flush_error_release_before, + 'L5F ERROR did not release immutable evidence or its REVOKING fence'); +like($flush_error_log, + qr/injected PCM-X retained-image FlushBuffer failure/, + 'L5F exact pre-smgrwrite finish FlushBuffer ERROR reached the DATA worker'); +like($flush_error_log, + qr/cluster PCM-X runtime fail-closed \(recovery blocked\)/, + 'L5F GCS finish catch preserved evidence and fused the runtime'); + $quad->stop_quad; +my $flush_error_shutdown_log = substr(slurp_file($quad->node0->logfile), + $flush_error_log_offset); +unlike($flush_error_shutdown_log, qr/lost track of buffer IO/, + 'L5F absorbed FlushBuffer ERROR left no ResourceOwner BufferIO at shutdown'); done_testing(); diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index f9f6c1f97d..b5a42a1762 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -1353,7 +1353,9 @@ test_cluster_advisory: test_cluster_advisory.c unit_test.h \ # acquire/release/upgrade/downgrade/query paths are exercised standalone. test_cluster_pcm_lock: test_cluster_pcm_lock.c unit_test.h \ $(CLUSTER_VERSION_O) $(CLUSTER_PCM_LOCK_O) - $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CC) $(CFLAGS) $(CPPFLAGS) \ + -DGCS_BLOCK_SOURCE_PATH='"$(top_srcdir)/src/backend/cluster/cluster_gcs_block.c"' \ + $< \ $(CLUSTER_VERSION_O) $(CLUSTER_PCM_LOCK_O) \ $(top_builddir)/src/common/libpgcommon_srv.a \ $(top_builddir)/src/port/libpgport_srv.a -o $@ @@ -1379,6 +1381,7 @@ test_cluster_pcm_direct_init: test_cluster_pcm_direct_init.c unit_test.h \ -DVM_SOURCE_PATH='"$(top_srcdir)/src/backend/access/heap/visibilitymap.c"' \ -DFSM_SOURCE_PATH='"$(top_srcdir)/src/backend/storage/freespace/freespace.c"' \ -DHEAPAM_SOURCE_PATH='"$(top_srcdir)/src/backend/access/heap/heapam.c"' \ + -DHIO_SOURCE_PATH='"$(top_srcdir)/src/backend/access/heap/hio.c"' \ $< \ $(CLUSTER_PCM_DIRECT_INIT_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ @@ -1450,7 +1453,10 @@ test_cluster_gcs_dispatch: test_cluster_gcs_dispatch.c unit_test.h \ test_cluster_gcs_block: test_cluster_gcs_block.c unit_test.h $(CLUSTER_VERSION_O) $(CC) $(CFLAGS) $(CPPFLAGS) \ -DGCS_BLOCK_SOURCE_PATH='"$(top_srcdir)/src/backend/cluster/cluster_gcs_block.c"' \ + -DBUFMGR_SOURCE_PATH='"$(top_srcdir)/src/backend/storage/buffer/bufmgr.c"' \ -DLMS_SOURCE_PATH='"$(top_srcdir)/src/backend/cluster/cluster_lms.c"' \ + -DLMS_OUTBOUND_SOURCE_PATH='"$(top_srcdir)/src/backend/cluster/cluster_lms_outbound.c"' \ + -DT400_SOURCE_PATH='"$(top_srcdir)/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl"' \ $< \ $(CLUSTER_VERSION_O) \ $(top_builddir)/src/common/libpgcommon_srv.a \ @@ -1613,9 +1619,12 @@ test_cluster_tt_status_hint: test_cluster_tt_status_hint.c unit_test.h \ # HeapTupleSatisfiesMVCC behavioral testing (requires real PG backend; # behavioral coverage in cluster_tap t/204). test_cluster_visibility_fork: test_cluster_visibility_fork.c unit_test.h \ - $(CLUSTER_VERSION_O) - $(CC) $(CFLAGS) $(CPPFLAGS) $< \ - $(CLUSTER_VERSION_O) \ + $(CLUSTER_VERSION_O) $(CLUSTER_VIS_VERDICT_O) + $(CC) $(CFLAGS) $(CPPFLAGS) \ + -DHEAPAM_SOURCE_PATH='"$(top_srcdir)/src/backend/access/heap/heapam.c"' \ + -DTT_LOCAL_SOURCE_PATH='"$(top_srcdir)/src/backend/cluster/cluster_tt_local.c"' \ + $< \ + $(CLUSTER_VERSION_O) $(CLUSTER_VIS_VERDICT_O) \ $(top_builddir)/src/common/libpgcommon_srv.a \ $(top_builddir)/src/port/libpgport_srv.a -o $@ diff --git a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c index d7b60618e1..62959db442 100644 --- a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c +++ b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c @@ -379,16 +379,18 @@ cluster_gcs_send_block_request_and_wait(struct BufferDesc *buf pg_attribute_unus /* spec-5.2 D2 sub-case B stub: local-master read-image forward unreachable. */ bool cluster_gcs_local_master_read_image_and_wait(struct BufferDesc *buf pg_attribute_unused(), - int32 holder_node pg_attribute_unused()) + const PcmAuthoritySnapshot *expected pg_attribute_unused(), + bool *out_retry_denied pg_attribute_unused()) { abort(); } /* spec-5.2 D11 stub: local-master X-transfer forward unreachable in this unit. */ bool -cluster_gcs_local_master_x_transfer_and_wait(struct BufferDesc *buf pg_attribute_unused(), - int32 holder_node pg_attribute_unused(), - bool clean_eligible pg_attribute_unused()) +cluster_gcs_local_master_x_transfer_and_wait( + struct BufferDesc *buf pg_attribute_unused(), + const PcmAuthoritySnapshot *expected pg_attribute_unused(), + bool clean_eligible pg_attribute_unused(), bool *out_retry_denied pg_attribute_unused()) { abort(); } diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 2d0a59d3d6..5560c6e735 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -1555,6 +1555,11 @@ cluster_gcs_get_pi_watermark_retire_count(void) return 0; } uint64 +cluster_gcs_get_pi_durable_note_apply_count(void) +{ + return 0; +} +uint64 cluster_gcs_get_lost_write_detected_count(void) { return 0; @@ -4315,6 +4320,7 @@ UT_TEST(test_debug_dump_exposes_exact_pcm_x_lmd_and_gcs_key_sets) "invalidate_passive_s_release_count", "pcm_x_self_handoff_count", "pcm_x_self_handoff_drain_count", + "pi_durable_note_apply_count", }; LOCAL_FCINFO(fcinfo, 0); ReturnSetInfo rsinfo; @@ -4330,7 +4336,7 @@ UT_TEST(test_debug_dump_exposes_exact_pcm_x_lmd_and_gcs_key_sets) UT_ASSERT_EQ(captured_dump_count("pcm", NULL), 60); UT_ASSERT_EQ(captured_dump_count("lmd", NULL), 51); - UT_ASSERT_EQ(captured_dump_count("gcs", NULL), 119); + UT_ASSERT_EQ(captured_dump_count("gcs", NULL), 120); for (i = 0; i < (int)lengthof(pcm_keys); i++) UT_ASSERT_EQ(captured_dump_count("pcm", pcm_keys[i]), 1); for (i = 0; i < (int)lengthof(lmd_keys); i++) @@ -4537,6 +4543,12 @@ cluster_lms_obs_get_outbound_requeue_drop(int worker_id pg_attribute_unused()) { return 0; } + +uint64 +cluster_lms_obs_get_outbound_cap_guard_drop(int worker_id pg_attribute_unused()) +{ + return 0; +} uint64 cluster_lms_obs_get_serve_hist(int worker_id pg_attribute_unused(), int bucket pg_attribute_unused()) diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 0506d5dec9..e7261acec3 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -2011,6 +2011,315 @@ UT_TEST(test_pcm_x_invalidate_ack_matches_only_exact_unacked_holder) } +UT_TEST(test_pcm_x_invalidate_busy_routes_to_exact_ticket_backoff) +{ + char *source = read_gcs_block_source(); + const char *handler; + const char *queue_busy; + const char *backoff; + const char *received; + const char *queue_return; + const char *legacy_busy; + const char *transfer; + const char *deadline; + const char *revoke; + const char *delay_helper; + const char *delay_end; + const char *leg_retry_delay; + const char *execute; + const char *self_capable_first; + const char *self_capable_second; + const char *execute_end; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + handler = strstr(source, "\ncluster_gcs_handle_block_invalidate_ack_envelope("); + queue_busy = handler != NULL ? strstr(handler, "queue_result == PCM_X_QUEUE_BUSY") : NULL; + backoff = queue_busy != NULL + ? strstr(queue_busy, "cluster_pcm_x_master_invalidate_busy_backoff_exact(") + : NULL; + received = backoff != NULL ? strstr(backoff, "invalidate_busy_received_count") : NULL; + queue_return = received != NULL ? strstr(received, "if (!queue_positive)") : NULL; + legacy_busy = queue_return != NULL + ? strstr(queue_return, + "ack->ack_status == GCS_BLOCK_INVALIDATE_ACK_STATUS_RETRYABLE_BUSY") + : NULL; + UT_ASSERT_NOT_NULL(handler); + UT_ASSERT_NOT_NULL(queue_busy); + UT_ASSERT_NOT_NULL(backoff); + UT_ASSERT_NOT_NULL(received); + UT_ASSERT_NOT_NULL(queue_return); + UT_ASSERT_NOT_NULL(legacy_busy); + if (handler != NULL && queue_busy != NULL && backoff != NULL && received != NULL + && queue_return != NULL && legacy_busy != NULL) + UT_ASSERT(handler < queue_busy && queue_busy < backoff && backoff < received + && received < queue_return && queue_return < legacy_busy); + + transfer = strstr(source, "\ngcs_block_pcm_x_master_drive_transfer("); + deadline = transfer != NULL ? strstr(transfer, "snapshot->retry_deadline_ms") : NULL; + revoke = transfer != NULL ? strstr(transfer, "cluster_pcm_x_master_revoke_arm_exact(") : NULL; + UT_ASSERT_NOT_NULL(transfer); + UT_ASSERT_NOT_NULL(deadline); + UT_ASSERT_NOT_NULL(revoke); + if (transfer != NULL && deadline != NULL && revoke != NULL) + UT_ASSERT(transfer < deadline && deadline < revoke); + + /* reliable.retry_count counts repeated REVOKE arming while an earlier + * holder INVALIDATE is still missing; it is not an INVALIDATE-BUSY retry + * counter. Feeding it into this delay can saturate the very first BUSY + * retry at 25s and phase-lock the denied reader's GRANT_PENDING window. */ + delay_helper = strstr(source, "\ngcs_block_pcm_x_invalidate_busy_retry_delay_ms("); + delay_end = delay_helper != NULL ? strstr(delay_helper, "\n}") : NULL; + leg_retry_delay + = delay_helper != NULL ? strstr(delay_helper, "snapshot->retry_count") : NULL; + UT_ASSERT_NOT_NULL(delay_helper); + UT_ASSERT_NOT_NULL(delay_end); + if (delay_helper != NULL && delay_end != NULL) + UT_ASSERT(leg_retry_delay == NULL || leg_retry_delay > delay_end); + + /* Self has no HELLO capability record. The local binary is nevertheless + * BUSY-capable on both GRANT_PENDING and pinned-S refusal arms. */ + execute = strstr(source, "\ngcs_block_invalidate_execute("); + execute_end = execute != NULL ? strstr(execute, "\n}\n") : NULL; + self_capable_first + = execute != NULL ? strstr(execute, "inv->master_node == cluster_node_id") : NULL; + self_capable_second = self_capable_first != NULL + ? strstr(self_capable_first + 1, "inv->master_node == cluster_node_id") + : NULL; + UT_ASSERT_NOT_NULL(execute); + UT_ASSERT_NOT_NULL(execute_end); + UT_ASSERT_NOT_NULL(self_capable_first); + UT_ASSERT_NOT_NULL(self_capable_second); + if (execute != NULL && self_capable_first != NULL && self_capable_second != NULL + && execute_end != NULL) + UT_ASSERT(execute < self_capable_first && self_capable_first < self_capable_second + && self_capable_second < execute_end); + free(source); +} + + +UT_TEST(test_pcm_x_local_pending_s_denial_match_is_attempt_exact) +{ + BufferTag slot_tag = { 0 }; + BufferTag inv_tag; + + slot_tag.spcOid = 31; + slot_tag.dbOid = 32; + slot_tag.relNumber = 33; + slot_tag.forkNum = MAIN_FORKNUM; + slot_tag.blockNum = 34; + inv_tag = slot_tag; + + UT_ASSERT(GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + UT_ASSERT(GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_ABORTED, false, &inv_tag, UINT64_C(41), 2)); + + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + false, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, true, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, false, true, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_X, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + inv_tag.blockNum++; + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + inv_tag = slot_tag; + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(40), 2, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 1, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_ARMED, false, &inv_tag, UINT64_C(41), 2)); + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_UNARMED, true, &inv_tag, UINT64_C(41), 2)); +} + + +UT_TEST(test_pcm_x_grant_pending_invalidate_wakes_local_s_before_busy) +{ + char *source = read_gcs_block_source(); + const char *direct_prepare; + const char *direct_prepare_end; + const char *direct_first_reply_check; + const char *direct_target_prepare; + const char *direct_second_reply_check; + const char *direct_target_cleanup; + const char *helper; + const char *execute; + const char *pending; + const char *wake; + const char *busy; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + direct_prepare = strstr(source, "\ngcs_block_direct_prepare_attempt("); + direct_prepare_end + = direct_prepare != NULL ? strstr(direct_prepare, "\n}\n\n\n") : NULL; + direct_first_reply_check + = direct_prepare != NULL ? strstr(direct_prepare, "slot->reply_received") : NULL; + direct_target_prepare = direct_first_reply_check != NULL + ? strstr(direct_first_reply_check, + "cluster_bufmgr_prepare_direct_land_target_for_gcs(") + : NULL; + direct_second_reply_check = direct_target_prepare != NULL + ? strstr(direct_target_prepare, "slot->reply_received") + : NULL; + direct_target_cleanup = direct_second_reply_check != NULL + ? strstr(direct_second_reply_check, + "gcs_block_direct_finish_target(buf, true, false") + : NULL; + helper = strstr(source, "\ngcs_block_wake_local_pending_s_request("); + execute = strstr(source, "\ngcs_block_invalidate_execute("); + pending = execute != NULL ? strstr(execute, "cluster_bufmgr_block_grant_pending(inv->tag)") + : NULL; + wake = pending != NULL + ? strstr(pending, "gcs_block_wake_local_pending_s_request(inv)") + : NULL; + busy = wake != NULL ? strstr(wake, "invalidate_busy_sent_count") : NULL; + UT_ASSERT_NOT_NULL(helper); + UT_ASSERT_NOT_NULL(direct_prepare); + UT_ASSERT_NOT_NULL(direct_prepare_end); + UT_ASSERT_NOT_NULL(direct_first_reply_check); + UT_ASSERT_NOT_NULL(direct_target_prepare); + UT_ASSERT_NOT_NULL(direct_second_reply_check); + UT_ASSERT_NOT_NULL(direct_target_cleanup); + UT_ASSERT_NOT_NULL(execute); + UT_ASSERT_NOT_NULL(pending); + UT_ASSERT_NOT_NULL(wake); + UT_ASSERT_NOT_NULL(busy); + if (direct_prepare != NULL && direct_prepare_end != NULL + && direct_first_reply_check != NULL && direct_target_prepare != NULL + && direct_second_reply_check != NULL && direct_target_cleanup != NULL) + UT_ASSERT(direct_prepare < direct_first_reply_check + && direct_first_reply_check < direct_target_prepare + && direct_target_prepare < direct_second_reply_check + && direct_second_reply_check < direct_target_cleanup + && direct_target_cleanup < direct_prepare_end); + if (helper != NULL && execute != NULL && pending != NULL && wake != NULL && busy != NULL) + UT_ASSERT(helper < execute && execute < pending && pending < wake && wake < busy); + free(source); +} + + +UT_TEST(test_pcm_x_grant_pending_orphan_observation_is_identity_exact) +{ + char *gcs_source = read_gcs_block_source(); + char *bufmgr_source = read_source_path(BUFMGR_SOURCE_PATH); + const char *execute; + const char *pending; + const char *wake; + const char *observe; + const char *busy; + const char *retry; + const char *abort; + const char *abort_observe; + const char *release; + const char *release_end; + const char *release_live; + const char *direct_fail; + const char *direct_fail_end; + const char *direct_finish; + const char *direct_abort_observe; + + UT_ASSERT_NOT_NULL(gcs_source); + UT_ASSERT_NOT_NULL(bufmgr_source); + if (gcs_source == NULL || bufmgr_source == NULL) { + free(gcs_source); + free(bufmgr_source); + return; + } + + execute = strstr(gcs_source, "\ngcs_block_invalidate_execute("); + pending = execute != NULL + ? strstr(execute, "cluster_bufmgr_block_grant_pending(inv->tag)") + : NULL; + wake = pending != NULL + ? strstr(pending, "woke_local = gcs_block_wake_local_pending_s_request(inv)") + : NULL; + observe = wake != NULL + ? strstr(wake, "gcs_block_observe_grant_pending_invalidate(inv, woke_local)") + : NULL; + busy = observe != NULL ? strstr(observe, "invalidate_busy_sent_count") : NULL; + UT_ASSERT_NOT_NULL(execute); + UT_ASSERT_NOT_NULL(pending); + UT_ASSERT_NOT_NULL(wake); + UT_ASSERT_NOT_NULL(observe); + UT_ASSERT_NOT_NULL(busy); + if (execute != NULL && pending != NULL && wake != NULL && observe != NULL && busy != NULL) + UT_ASSERT(execute < pending && pending < wake && wake < observe && observe < busy); + + /* The diagnostic must report why a candidate was rejected without replacing + * or weakening the established attempt-exact protocol predicate. */ + UT_ASSERT_NOT_NULL(strstr(gcs_source, "GcsBlockLocalPendingSDenialMatches(")); + UT_ASSERT_NOT_NULL(strstr(gcs_source, "grant-pending invalidate observation:")); + UT_ASSERT_NOT_NULL(strstr(gcs_source, "slot_backend=%d slot_index=%d slot_request_id=")); + UT_ASSERT_NOT_NULL(strstr(gcs_source, "direct_state=%d direct_prepared=%d reject_mask=0x%x")); + + retry = strstr(bufmgr_source, "\ncluster_bufmgr_pcm_retry_denied_rearm("); + abort = retry != NULL + ? strstr(retry, "cluster_pcm_own_abort_grant_reservation(buf, base, *reservation_token)") + : NULL; + abort_observe = abort != NULL + ? strstr(abort, "cluster PCM pending-X exact abort observation:") + : NULL; + UT_ASSERT_NOT_NULL(retry); + UT_ASSERT_NOT_NULL(abort); + UT_ASSERT_NOT_NULL(abort_observe); + if (retry != NULL && abort != NULL && abort_observe != NULL) + UT_ASSERT(retry < abort && abort < abort_observe); + + release = strstr(gcs_source, "\ngcs_block_release_slot("); + release_end = release != NULL ? strstr(release, "\n}\n") : NULL; + release_live = release != NULL + ? strstr(release, + "cluster GCS block slot released with live direct target observation:") + : NULL; + UT_ASSERT_NOT_NULL(release); + UT_ASSERT_NOT_NULL(release_end); + UT_ASSERT_NOT_NULL(release_live); + if (release != NULL && release_end != NULL && release_live != NULL) + UT_ASSERT(release < release_live && release_live < release_end); + + direct_fail = strstr(gcs_source, "\ngcs_block_direct_fail_slot("); + direct_fail_end = direct_fail != NULL ? strstr(direct_fail, "\n}\n") : NULL; + direct_finish = direct_fail != NULL + ? strstr(direct_fail, + "gcs_block_direct_finish_target(target_buf, prepared, false") + : NULL; + direct_abort_observe = direct_finish != NULL + ? strstr(direct_finish, + "cluster GCS block direct abort observation:") + : NULL; + UT_ASSERT_NOT_NULL(direct_fail); + UT_ASSERT_NOT_NULL(direct_fail_end); + UT_ASSERT_NOT_NULL(direct_finish); + UT_ASSERT_NOT_NULL(direct_abort_observe); + if (direct_fail != NULL && direct_fail_end != NULL && direct_finish != NULL + && direct_abort_observe != NULL) + UT_ASSERT(direct_fail < direct_finish && direct_finish < direct_abort_observe + && direct_abort_observe < direct_fail_end); + + free(gcs_source); + free(bufmgr_source); +} + + UT_TEST(test_pcm_x_final_ack_builds_exact_grd_handoff_token) { PcmAuthoritySnapshot authority; @@ -2056,6 +2365,60 @@ UT_TEST(test_pcm_x_final_ack_builds_exact_grd_handoff_token) } +UT_TEST(test_pcm_x_final_ack_fail_closed_names_exact_handoff_stage) +{ + char *source = read_gcs_block_source(); + const char *handler + = source != NULL ? strstr(source, "\ncluster_gcs_handle_pcm_x_final_ack_envelope(") : NULL; + const char *handler_end + = handler != NULL + ? strstr(handler, "\ncluster_gcs_handle_pcm_x_final_commit_ack_envelope(") + : NULL; + const char *stage_log + = handler != NULL ? strstr(handler, "PCM-X FINAL_ACK fail-closed at %s") : NULL; + const char *canonical + = handler != NULL ? strstr(handler, "cluster_pcm_x_runtime_fail_closed()") : NULL; + const char *direct + = handler != NULL ? strstr(handler, "cluster_pcm_x_runtime_transition(") : NULL; + const char *master_holder + = handler != NULL ? strstr(handler, "master_holder=%u") : NULL; + const char *image_page_scn + = handler != NULL ? strstr(handler, "image_page_scn=%llu") : NULL; + const char *watermark_scn + = handler != NULL ? strstr(handler, "watermark_scn=%llu") : NULL; + const char *watermark_source + = handler != NULL ? strstr(handler, "wm_src=%s") : NULL; + const char *watermark_sender + = handler != NULL ? strstr(handler, "wm_sender=%d") : NULL; + const char *watermark_request + = handler != NULL ? strstr(handler, "wm_request_id=%llu") : NULL; + const char *watermark_old + = handler != NULL ? strstr(handler, "wm_old_scn=%llu") : NULL; + const char *watermark_new + = handler != NULL ? strstr(handler, "wm_new_scn=%llu") : NULL; + + UT_ASSERT_NOT_NULL(handler); + UT_ASSERT_NOT_NULL(handler_end); + UT_ASSERT_NOT_NULL(stage_log); + UT_ASSERT_NOT_NULL(canonical); + UT_ASSERT_NOT_NULL(master_holder); + UT_ASSERT_NOT_NULL(image_page_scn); + UT_ASSERT_NOT_NULL(watermark_scn); + UT_ASSERT_NOT_NULL(watermark_source); + UT_ASSERT_NOT_NULL(watermark_sender); + UT_ASSERT_NOT_NULL(watermark_request); + UT_ASSERT_NOT_NULL(watermark_old); + UT_ASSERT_NOT_NULL(watermark_new); + UT_ASSERT_NULL(source != NULL ? strstr(source, "cluster_pcm_x_runtime_transition(") : NULL); + /* A blocked runtime must always publish the canonical counter and file:line arm. */ + if (handler != NULL && handler_end != NULL && stage_log != NULL && canonical != NULL) + UT_ASSERT(handler < stage_log && stage_log < canonical && canonical < handler_end); + if (handler != NULL && handler_end != NULL) + UT_ASSERT(direct == NULL || direct >= handler_end); + free(source); +} + + UT_TEST(test_pcm_x_holder_image_evidence_never_uses_generation_as_presence) { PcmXLocalHolderProgress progress; @@ -2109,6 +2472,15 @@ UT_TEST(test_pcm_x_ready_publication_follows_exact_retained_commit) const char *publish; const char *send; const char *end; + const char *wrapper; + const char *wrapper_end; + const char *wrapper_catch; + const char *copy_error; + const char *flush_error; + const char *preserve; + const char *fail_closed; + const char *return_corrupt; + const char *rollback; if (source == NULL) return; @@ -2120,7 +2492,7 @@ UT_TEST(test_pcm_x_ready_publication_follows_exact_retained_commit) } end = strstr(begin + 1, "\n}\n\n\n"); materialize = strstr(begin, "cluster_gcs_block_dedup_pcm_x_materialize("); - finish = strstr(begin, "cluster_bufmgr_pcm_own_finish_revoke_retain("); + finish = strstr(begin, "gcs_block_pcm_x_finish_revoke_retain("); publish = strstr(begin, "cluster_gcs_block_dedup_pcm_x_publish_ready_exact("); send = strstr(begin, "gcs_block_pcm_x_stage_ready_work("); UT_ASSERT_NOT_NULL(end); @@ -2130,6 +2502,45 @@ UT_TEST(test_pcm_x_ready_publication_follows_exact_retained_commit) UT_ASSERT_NOT_NULL(send); if (end != NULL && materialize != NULL && finish != NULL && publish != NULL && send != NULL) UT_ASSERT(materialize < finish && finish < publish && publish < send && send < end); + wrapper = strstr(source, "\ngcs_block_pcm_x_finish_revoke_retain("); + wrapper_end = wrapper != NULL ? strstr(wrapper, "\n}\n") : NULL; + wrapper_catch = wrapper != NULL ? strstr(wrapper, "PG_CATCH();") : NULL; + copy_error = wrapper_catch != NULL ? strstr(wrapper_catch, "CopyErrorData();") : NULL; + flush_error = copy_error != NULL ? strstr(copy_error, "FlushErrorState();") : NULL; + preserve + = flush_error != NULL + ? strstr(flush_error, "cluster_gcs_block_dedup_pcm_x_preserve_finish_error_exact(") + : NULL; + fail_closed + = preserve != NULL ? strstr(preserve, "cluster_pcm_x_runtime_fail_closed();") : NULL; + return_corrupt + = fail_closed != NULL ? strstr(fail_closed, "result = CLUSTER_PCM_OWN_CORRUPT;") : NULL; + rollback = wrapper_catch != NULL + ? strstr(wrapper_catch, "gcs_block_pcm_x_abort_image_before_finish(") + : NULL; + UT_ASSERT_NOT_NULL(wrapper); + UT_ASSERT_NOT_NULL(wrapper_end); + UT_ASSERT_NOT_NULL(wrapper_catch); + UT_ASSERT_NOT_NULL(copy_error); + UT_ASSERT_NOT_NULL(flush_error); + UT_ASSERT_NOT_NULL(preserve); + UT_ASSERT_NOT_NULL(strstr(wrapper_catch, "PCM-X finish-error evidence exact")); + UT_ASSERT_NOT_NULL(strstr(wrapper_catch, "preserve_result")); + UT_ASSERT_NOT_NULL(strstr(wrapper_catch, "work->key")); + UT_ASSERT_NOT_NULL(strstr(wrapper_catch, "work->binding.identity.ref")); + UT_ASSERT_NOT_NULL(strstr(wrapper_catch, "revoking->reservation_token")); + UT_ASSERT_NOT_NULL(fail_closed); + UT_ASSERT_NOT_NULL(return_corrupt); + if (wrapper != NULL && wrapper_end != NULL && wrapper_catch != NULL && copy_error != NULL + && flush_error != NULL && preserve != NULL && fail_closed != NULL + && return_corrupt != NULL) { + UT_ASSERT(wrapper < wrapper_catch && wrapper_catch < copy_error && copy_error < flush_error + && flush_error < preserve && preserve < fail_closed + && fail_closed < return_corrupt && return_corrupt < wrapper_end); + UT_ASSERT(rollback == NULL || rollback >= wrapper_end); + UT_ASSERT(strstr(wrapper_catch, "PG_RE_THROW();") == NULL + || strstr(wrapper_catch, "PG_RE_THROW();") >= wrapper_end); + } free(source); } @@ -2170,13 +2581,23 @@ UT_TEST(test_pcm_x_ready_materializes_exact_n_s_or_x_source_without_wire_change) UT_ASSERT_NOT_NULL(strstr(begin, "current.pcm_state == (uint8)PCM_STATE_S")); UT_ASSERT_NOT_NULL(strstr(begin, "current.pcm_state == (uint8)PCM_STATE_X")); UT_ASSERT_NOT_NULL(strstr(begin, "cluster_bufmgr_pcm_own_prepare_n_source_image(")); - UT_ASSERT_NOT_NULL(strstr(begin, "cluster_bufmgr_pcm_own_begin_s_revoke(")); + UT_ASSERT_NOT_NULL(strstr(begin, "cluster_bufmgr_pcm_own_prepare_s_source_image(")); + UT_ASSERT_NOT_NULL(strstr(begin, "binding.required_page_scn")); + UT_ASSERT_NOT_NULL(strstr(begin, "&source_prepare_refusal")); + UT_ASSERT_NOT_NULL( + strstr(source, "materialize-begin-s-content-lock")); + UT_ASSERT_NOT_NULL( + strstr(source, "materialize-begin-s-dirty-flushed")); + UT_ASSERT_NOT_NULL( + strstr(source, "materialize-begin-s-dirty-raced")); + UT_ASSERT_NOT_NULL( + strstr(source, "materialize-begin-s-io-in-progress")); UT_ASSERT_NOT_NULL(strstr(begin, "cluster_bufmgr_pcm_own_begin_x_revoke(")); UT_ASSERT_NOT_NULL(strstr(begin, "cluster_pcm_x_revoke_finish_mode(")); UT_ASSERT_NOT_NULL(strstr(begin, "CLUSTER_PCM_X_REVOKE_FINISH_DROP")); copy = strstr(begin, "cluster_bufmgr_copy_block_for_gcs("); materialize = strstr(begin, "cluster_gcs_block_dedup_pcm_x_materialize("); - finish = strstr(begin, "cluster_bufmgr_pcm_own_finish_revoke_retain("); + finish = strstr(begin, "gcs_block_pcm_x_finish_revoke_retain("); publish = strstr(begin, "cluster_gcs_block_dedup_pcm_x_publish_ready_exact("); UT_ASSERT_NOT_NULL(copy); UT_ASSERT_NOT_NULL(materialize); @@ -2194,14 +2615,23 @@ UT_TEST(test_pcm_x_ready_materializes_exact_n_s_or_x_source_without_wire_change) const char *local_drain = strstr(drain, "cluster_pcm_x_local_drain_poll_certificate_exact("); const char *duplicate_guard - = local_drain != NULL ? strstr(local_drain, "if (result != PCM_X_QUEUE_OK)") : NULL; + = local_drain != NULL + ? strstr(local_drain, + "if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE)") + : NULL; + const char *drain_status + = duplicate_guard != NULL + ? strstr(duplicate_guard, "cluster_gcs_block_dedup_pcm_x_drain_status_exact(") + : NULL; + const char *drained_replay + = drain_status != NULL ? strstr(drain_status, "GCS_BLOCK_PCM_X_IMAGE_DUPLICATE") : NULL; /* The self-source release authority is the exact completion - * certificate; an inexact ledger preserves the record without a - * node fuse (the descriptor probe alone may report corruption). */ + * certificate. Once local DRAIN is durable, an inexact ledger cannot + * be retried from a fabricated certificate and must preserve evidence + * under the runtime fuse . */ const char *certificate_policy = strstr(drain, "source_own_generation + 1"); - const char *certificate_refusal = certificate_policy != NULL - ? strstr(certificate_policy, "PCM_X_QUEUE_NOT_READY") - : NULL; + const char *certificate_refusal + = certificate_policy != NULL ? strstr(certificate_policy, "PCM_X_QUEUE_CORRUPT") : NULL; const char *release_record = strstr(drain, "cluster_gcs_block_dedup_pcm_x_release_exact("); const char *finish_mode_gate = strstr(drain, "cluster_pcm_x_revoke_finish_mode("); const char *drop_arm = strstr(drain, "CLUSTER_PCM_X_REVOKE_FINISH_DROP"); @@ -2210,19 +2640,24 @@ UT_TEST(test_pcm_x_ready_materializes_exact_n_s_or_x_source_without_wire_change) UT_ASSERT_NOT_NULL(local_drain); UT_ASSERT_NOT_NULL(duplicate_guard); + UT_ASSERT_NOT_NULL(drain_status); + UT_ASSERT_NOT_NULL(drained_replay); UT_ASSERT_NOT_NULL(certificate_policy); UT_ASSERT_NOT_NULL(certificate_refusal); UT_ASSERT_NOT_NULL(release_record); UT_ASSERT_NOT_NULL(finish_mode_gate); UT_ASSERT_NOT_NULL(drop_arm); UT_ASSERT_NOT_NULL(release_retained); - if (local_drain != NULL && duplicate_guard != NULL && certificate_policy != NULL - && certificate_refusal != NULL && release_record != NULL && finish_mode_gate != NULL - && drop_arm != NULL && release_retained != NULL) - UT_ASSERT(local_drain < duplicate_guard && duplicate_guard < certificate_policy + if (local_drain != NULL && duplicate_guard != NULL && drain_status != NULL + && drained_replay != NULL && certificate_policy != NULL && certificate_refusal != NULL + && release_record != NULL && finish_mode_gate != NULL && drop_arm != NULL + && release_retained != NULL) + UT_ASSERT(local_drain < duplicate_guard && duplicate_guard < drain_status + && drain_status < drained_replay && drained_replay < certificate_policy && certificate_policy < certificate_refusal - && certificate_refusal < release_record && release_record < finish_mode_gate - && finish_mode_gate < drop_arm && drop_arm < release_retained); + && certificate_refusal < finish_mode_gate + && finish_mode_gate < release_retained && release_retained < drop_arm + && drop_arm < release_record); } if (generic_install != NULL) { const char *content = strstr(generic_install, "LWLockAcquire(content_lock, LW_EXCLUSIVE)"); @@ -2240,6 +2675,61 @@ UT_TEST(test_pcm_x_ready_materializes_exact_n_s_or_x_source_without_wire_change) } +UT_TEST(test_pcm_x_s_source_hard_failure_observation_is_reason_exact) +{ + char *source = read_source_path(BUFMGR_SOURCE_PATH); + const char *observe; + const char *prepare; + const char *prepare_end; + static const char *const reasons[] = { + "CLUSTER_PCM_S_SOURCE_HARD_INITIAL_CURRENT_IMAGE", + "CLUSTER_PCM_S_SOURCE_HARD_INITIAL_IO_ERROR", + "CLUSTER_PCM_S_SOURCE_HARD_BEGIN_REVOKE_CORRUPT", + "CLUSTER_PCM_S_SOURCE_HARD_STORAGE_VERIFY", + "CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_CURRENT_IMAGE", + "CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_IO_ERROR", + "CLUSTER_PCM_S_SOURCE_HARD_NO_COVER", + "CLUSTER_PCM_S_SOURCE_HARD_ABORT_FAILURE" + }; + int i; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + observe = strstr(source, "\ncluster_bufmgr_pcm_own_observe_s_source_hard_failure("); + prepare = strstr(source, "\ncluster_bufmgr_pcm_own_prepare_s_source_image("); + prepare_end = prepare != NULL + ? strstr(prepare, "\n/* Abort only the matching S-source staging reservation. */") + : NULL; + UT_ASSERT_NOT_NULL(observe); + UT_ASSERT_NOT_NULL(prepare); + UT_ASSERT_NOT_NULL(prepare_end); + for (i = 0; i < lengthof(reasons); i++) + UT_ASSERT_NOT_NULL(strstr(source, reasons[i])); + + /* Hard-failure evidence is local-only and state-change suppressed. It + * identifies the exact ownership tuple and records both sides of the SCN + * floor comparison without changing the prepare function's public API. */ + UT_ASSERT_NOT_NULL(strstr(source, "ClusterPcmOwnSSourceHardFailureObservation cache[8]")); + UT_ASSERT_NOT_NULL(strstr(source, "memcmp(&cache[cache_idx], &obs, sizeof(obs)) == 0")); + UT_ASSERT_NOT_NULL(strstr(source, "cluster PCM S-source hard failure observation:")); + UT_ASSERT_NOT_NULL(strstr(source, "reason=%s cause=%s result=%d abort_result=%d")); + UT_ASSERT_NOT_NULL(strstr(source, "buffer=%d spc=%u db=%u rel=%u fork=%d blk=%u")); + UT_ASSERT_NOT_NULL(strstr(source, "state=%u generation=%llu token=%llu flags=0x%x")); + UT_ASSERT_NOT_NULL(strstr(source, "required_scn=%llu local_scn=%llu storage_scn=%llu")); + UT_ASSERT_NOT_NULL(strstr(source, "buffer_state=0x%x buffer_type=%u")); + if (observe != NULL && prepare != NULL) + UT_ASSERT(observe < prepare); + if (prepare != NULL && prepare_end != NULL) { + UT_ASSERT_NOT_NULL(strstr(prepare, "hard_failure_reason =")); + UT_ASSERT_NOT_NULL(strstr(prepare, "cluster_bufmgr_pcm_own_observe_s_source_hard_failure(")); + UT_ASSERT(strstr(prepare, "cluster_bufmgr_pcm_own_observe_s_source_hard_failure(") + < prepare_end); + } + free(source); +} + + UT_TEST(test_pcm_x_self_and_remote_drain_share_full_image_release_wrapper) { char *source = read_gcs_block_source(); @@ -2294,14 +2784,24 @@ UT_TEST(test_pcm_x_self_and_remote_drain_share_full_image_release_wrapper) UT_TEST(test_pcm_x_ready_admission_marks_before_send_and_rolls_back_refusal) { char *source = read_gcs_block_source(); + char *outbound_source = read_source_path(LMS_OUTBOUND_SOURCE_PATH); const char *begin; const char *end; + const char *arm; + const char *arm_refusal; const char *mark; const char *send; const char *rollback; + const char *handler; + const char *handler_end; + const char *prepare_handler; + const char *prepare_handler_end; - if (source == NULL) + if (source == NULL || outbound_source == NULL) { + free(source); + free(outbound_source); return; + } begin = strstr(source, "\ngcs_block_pcm_x_stage_ready_work("); UT_ASSERT_NOT_NULL(begin); if (begin == NULL) { @@ -2309,16 +2809,60 @@ UT_TEST(test_pcm_x_ready_admission_marks_before_send_and_rolls_back_refusal) return; } end = strstr(begin + 1, "\n}\n\n\n"); + arm = strstr(begin, "cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed("); + arm_refusal = strstr(begin, "gcs_block_pcm_x_image_ready_arm_refusal_note_work("); mark = strstr(begin, "cluster_gcs_block_dedup_pcm_x_mark_staged_exact("); send = strstr(begin, "cluster_gcs_pcm_x_stage_frame("); rollback = strstr(begin, "cluster_gcs_block_dedup_pcm_x_unmark_staged_exact("); UT_ASSERT_NOT_NULL(end); + UT_ASSERT_NOT_NULL(arm); + UT_ASSERT_NOT_NULL(arm_refusal); UT_ASSERT_NOT_NULL(mark); UT_ASSERT_NOT_NULL(send); UT_ASSERT_NOT_NULL(rollback); - if (end != NULL && mark != NULL && send != NULL && rollback != NULL) - UT_ASSERT(mark < send && send < rollback && rollback < end); + if (end != NULL && arm != NULL && arm_refusal != NULL && mark != NULL && send != NULL + && rollback != NULL) + UT_ASSERT(arm < arm_refusal && arm_refusal < mark && mark < send && send < rollback + && rollback < end); + UT_ASSERT_NOT_NULL(strstr(begin, "PCM_X_LOCAL_IMAGE_READY_REFUSAL_ACTIVE_HOLDER")); + UT_ASSERT_NOT_NULL(strstr(begin, "PCM_X_LOCAL_IMAGE_READY_REFUSAL_RELIABLE_LEG")); + UT_ASSERT_NOT_NULL(strstr(begin, "PCM-X IMAGE_READY stage boundary")); + UT_ASSERT_NOT_NULL(strstr(begin, "mark_result")); + UT_ASSERT_NOT_NULL(strstr(begin, "stage_result")); + UT_ASSERT_NOT_NULL( + strstr(outbound_source, "cluster_lms_note_pcm_x_image_ready_boundary(")); + handler = strstr(source, "\ncluster_gcs_handle_pcm_x_image_ready_envelope("); + UT_ASSERT_NOT_NULL(handler); + if (handler != NULL) { + handler_end = strstr(handler + 1, "\n}\n\n\n"); + UT_ASSERT_NOT_NULL(handler_end); + UT_ASSERT_NOT_NULL(strstr(handler, "image-ready-master-consume")); + UT_ASSERT_NOT_NULL(strstr(handler, "PCM-X IMAGE_READY master boundary: ingress")); + UT_ASSERT_NOT_NULL(strstr(handler, "PCM-X IMAGE_READY master boundary: consume")); + UT_ASSERT_NOT_NULL(strstr(handler, "wire_valid")); + UT_ASSERT_NOT_NULL(strstr(handler, "authorized")); + UT_ASSERT_NOT_NULL(strstr(handler, "prepare_stage_result")); + if (handler_end != NULL) + UT_ASSERT(strstr(handler, "image-ready-master-consume") < handler_end); + } + prepare_handler = strstr(source, "\ncluster_gcs_handle_pcm_x_prepare_grant_envelope("); + UT_ASSERT_NOT_NULL(prepare_handler); + if (prepare_handler != NULL) { + prepare_handler_end = strstr(prepare_handler + 1, "\n}\n\n\n"); + UT_ASSERT_NOT_NULL(prepare_handler_end); + UT_ASSERT_NOT_NULL( + strstr(prepare_handler, "PCM-X PREPARE_GRANT requester boundary: ingress")); + UT_ASSERT_NOT_NULL( + strstr(prepare_handler, "PCM-X PREPARE_GRANT requester boundary: apply")); + UT_ASSERT_NOT_NULL(strstr(prepare_handler, "wire_valid")); + UT_ASSERT_NOT_NULL(strstr(prepare_handler, "authorized")); + UT_ASSERT_NOT_NULL(strstr(prepare_handler, "lookup_result")); + if (prepare_handler_end != NULL) + UT_ASSERT(strstr(prepare_handler, "PCM-X PREPARE_GRANT requester boundary: apply") + < prepare_handler_end); + } free(source); + free(outbound_source); } @@ -2356,6 +2900,7 @@ UT_TEST(test_pcm_x_lms_owner_death_and_restart_audit_fail_closed) worker_start = strstr(lms_source, "\nLmsWorkerMain(int worker_id)"); UT_ASSERT_NOT_NULL(main_start); UT_ASSERT_NOT_NULL(worker_start); + UT_ASSERT_NOT_NULL(strstr(lms_source, "PCM-X IMAGE_READY transport boundary")); main_call = main_start != NULL ? strstr(main_start, "cluster_gcs_block_pcm_x_owner_start(0)") : NULL; worker_call = worker_start != NULL @@ -2373,6 +2918,98 @@ UT_TEST(test_pcm_x_lms_owner_death_and_restart_audit_fail_closed) } +UT_TEST(test_pcm_x_lms_reload_acknowledges_finish_flush_injection) +{ + char *lms_source = read_source_path(LMS_SOURCE_PATH); + const char *helper; + const char *main_start; + const char *main_reload; + const char *main_ack; + const char *worker_start; + const char *worker_reload; + const char *worker_ack; + + if (lms_source == NULL) + return; + helper = strstr(lms_source, "\nlms_note_pcm_x_finish_flush_injection_reload("); + main_start = strstr(lms_source, "\nLmsMain(void)"); + worker_start = strstr(lms_source, "\nLmsWorkerMain(int worker_id)"); + UT_ASSERT_NOT_NULL(helper); + UT_ASSERT_NOT_NULL(main_start); + UT_ASSERT_NOT_NULL(worker_start); + if (helper != NULL) { + UT_ASSERT_NOT_NULL( + strstr(helper, "cluster_injection_is_armed(\"cluster-pcm-x-retain-flush-error\")")); + UT_ASSERT_NOT_NULL(strstr(helper, "MyProcPid")); + UT_ASSERT_NOT_NULL(strstr(helper, "cluster_injection_points")); + } + main_reload = main_start != NULL ? strstr(main_start, "ProcessConfigFile(PGC_SIGHUP);") : NULL; + main_ack = main_reload != NULL + ? strstr(main_reload, "lms_note_pcm_x_finish_flush_injection_reload(0);") + : NULL; + worker_reload + = worker_start != NULL ? strstr(worker_start, "ProcessConfigFile(PGC_SIGHUP);") : NULL; + worker_ack + = worker_reload != NULL + ? strstr(worker_reload, "lms_note_pcm_x_finish_flush_injection_reload(worker_id);") + : NULL; + UT_ASSERT_NOT_NULL(main_reload); + UT_ASSERT_NOT_NULL(main_ack); + UT_ASSERT_NOT_NULL(worker_reload); + UT_ASSERT_NOT_NULL(worker_ack); + if (main_reload != NULL && main_ack != NULL && worker_start != NULL) + UT_ASSERT(main_reload < main_ack && main_ack < worker_start); + if (worker_reload != NULL && worker_ack != NULL) + UT_ASSERT(worker_reload < worker_ack); + free(lms_source); +} + + +UT_TEST(test_pcm_x_destructive_finish_fault_times_out_in_sql_before_harness) +{ + char *source = read_source_path(T400_SOURCE_PATH); + const char *leg; + const char *statement_timeout; + const char *insert; + const char *harness_timeout; + const char *assertion; + const char *destructive_leg; + const char *exact_oracle; + const char *global_occupancy; + const char *relfilenode; + + if (source == NULL) + return; + leg = strstr(source, "my ($flush_error_rc, $flush_error_out, $flush_error_err)"); + destructive_leg = strstr(source, "# The destructive leg is deliberately last"); + UT_ASSERT_NOT_NULL(leg); + UT_ASSERT_NOT_NULL(destructive_leg); + statement_timeout = leg != NULL ? strstr(leg, "statement_timeout") : NULL; + insert = leg != NULL + ? strstr(leg, "INSERT INTO pcm_xq_flush_error(id, v) VALUES (2, 1)") + : NULL; + harness_timeout = leg != NULL ? strstr(leg, "timeout => 30") : NULL; + assertion = leg != NULL ? strstr(leg, "L5F remote writer failed") : NULL; + relfilenode = destructive_leg != NULL ? strstr(destructive_leg, "flush_error_relfilenode") : NULL; + exact_oracle + = destructive_leg != NULL ? strstr(destructive_leg, "PCM-X finish-error evidence exact") + : NULL; + global_occupancy + = destructive_leg != NULL ? strstr(destructive_leg, "dedup_entry_count") : NULL; + UT_ASSERT_NOT_NULL(statement_timeout); + UT_ASSERT_NOT_NULL(insert); + UT_ASSERT_NOT_NULL(harness_timeout); + UT_ASSERT_NOT_NULL(assertion); + UT_ASSERT_NOT_NULL(relfilenode); + UT_ASSERT_NOT_NULL(exact_oracle); + UT_ASSERT_NULL(global_occupancy); + if (statement_timeout != NULL && insert != NULL && harness_timeout != NULL && assertion != NULL) + UT_ASSERT(statement_timeout < insert && insert < harness_timeout + && harness_timeout < assertion); + free(source); +} + + UT_TEST(test_pcm_x_image_fetch_intercepts_canonical_id_before_generic_dedup) { char *source = read_gcs_block_source(); @@ -2397,6 +3034,10 @@ UT_TEST(test_pcm_x_image_fetch_intercepts_canonical_id_before_generic_dedup) UT_ASSERT(intercept < generic_lookup); } if (serve != NULL) { + UT_ASSERT_NOT_NULL(strstr(serve, "PCM-X image fetch source boundary: ingress")); + UT_ASSERT_NOT_NULL(strstr(serve, "PCM-X image fetch source boundary: validate")); + UT_ASSERT_NOT_NULL(strstr(serve, "PCM-X image fetch source boundary: lookup")); + UT_ASSERT_NOT_NULL(strstr(serve, "PCM-X image fetch source boundary: reply")); UT_ASSERT_NOT_NULL(strstr(serve, "cluster_pcm_x_image_fetch_request_exact(")); UT_ASSERT(count_occurrences(serve, "cluster_pcm_x_local_holder_progress_exact(") >= 2); UT_ASSERT_NOT_NULL(strstr(serve, "gcs_block_pcm_x_authenticated_session(")); @@ -2440,6 +3081,15 @@ UT_TEST(test_pcm_x_requester_fetch_revalidates_queue_and_reservation_before_inst UT_ASSERT_NOT_NULL(fetch); UT_ASSERT_NOT_NULL(fetch_end); if (fetch != NULL && fetch_end != NULL) { + UT_ASSERT_NOT_NULL(strstr(fetch, "PCM-X image fetch requester boundary: entry")); + UT_ASSERT_NOT_NULL(strstr(fetch, "PCM-X image fetch requester boundary: pre-slot reject")); + UT_ASSERT_NOT_NULL(strstr(fetch, "branch=identity-base")); + UT_ASSERT_NOT_NULL(strstr(fetch, "branch=reservation-live")); + UT_ASSERT_NOT_NULL(strstr(fetch, "progress_grant_base=%llu effective_grant_base=%llu")); + UT_ASSERT_NOT_NULL(strstr(fetch, "live_writer_activation_token=%llu")); + UT_ASSERT_NOT_NULL(strstr(fetch, "PCM-X image fetch requester boundary: send")); + UT_ASSERT_NOT_NULL(strstr(fetch, "PCM-X image fetch requester boundary: receive")); + UT_ASSERT_NOT_NULL(strstr(fetch, "PCM-X image fetch requester boundary: complete")); UT_ASSERT_NOT_NULL(strstr(fetch, "gcs_block_try_reserve_exact_slot(")); UT_ASSERT_NOT_NULL(strstr(fetch, "cluster_pcm_x_image_fetch_build_request(")); UT_ASSERT_NOT_NULL(strstr(fetch, "const PcmXRuntimeSnapshot *request_runtime")); @@ -2638,7 +3288,7 @@ UT_TEST(test_pcm_x_self_source_handoff_is_no_copy_and_drain_preserves_x) && finish_catch_rethrow < finish_end_try && finish_end_try < finish_end); } self_arm = strstr(materialize, "if (self_source_handoff)"); - remote_finish = strstr(materialize, "cluster_bufmgr_pcm_own_finish_revoke_retain("); + remote_finish = strstr(materialize, "gcs_block_pcm_x_finish_revoke_retain("); UT_ASSERT_NOT_NULL(self_arm); UT_ASSERT_NOT_NULL(remote_finish); if (self_arm != NULL && remote_finish != NULL && materialize_end != NULL) @@ -2655,8 +3305,8 @@ UT_TEST(test_pcm_x_self_source_handoff_is_no_copy_and_drain_preserves_x) UT_ASSERT_NOT_NULL(release_retained); if (verify_x != NULL && release_record != NULL && self_return != NULL && release_retained != NULL && drain_end != NULL) - UT_ASSERT(verify_x < release_record && release_record < self_return - && self_return < release_retained && release_retained < drain_end); + UT_ASSERT(verify_x < release_retained && release_retained < release_record + && release_record < self_return && self_return < drain_end); free(source); } @@ -3474,6 +4124,7 @@ UT_TEST(test_pcm_x_retire_commit_wakes_exact_waiters_before_ack_or_resolve) const char *allocate; const char *allocate_after; const char *collect; + const char *dedup_retire; const char *exact_wake; const char *wake; const char *wake_end; @@ -3578,14 +4229,19 @@ UT_TEST(test_pcm_x_retire_commit_wakes_exact_waiters_before_ack_or_resolve) collect = allocate != NULL ? strstr(allocate, "cluster_pcm_x_local_retire_up_to_collect_exact(") : NULL; + dedup_retire = collect != NULL + ? strstr(collect, "cluster_gcs_block_dedup_pcm_x_retire_up_to(") + : NULL; exact_wake = collect != NULL ? strstr(collect, "gcs_block_pcm_x_wake_requester_exact(") : NULL; allocate_after = allocate != NULL ? strstr(allocate + 1, "palloc0(") : NULL; UT_ASSERT_NOT_NULL(allocate); UT_ASSERT_NOT_NULL(collect); + UT_ASSERT_NOT_NULL(dedup_retire); UT_ASSERT_NOT_NULL(exact_wake); - if (allocate != NULL && collect != NULL && exact_wake != NULL) - UT_ASSERT(allocate < collect && collect < exact_wake && exact_wake < common_end); + if (allocate != NULL && collect != NULL && dedup_retire != NULL && exact_wake != NULL) + UT_ASSERT(allocate < collect && collect < dedup_retire && dedup_retire < exact_wake + && exact_wake < common_end); UT_ASSERT(allocate_after == NULL || allocate_after > common_end); } remote = strstr(source, "\ncluster_gcs_handle_pcm_x_retire_up_to_envelope("); @@ -3740,7 +4396,7 @@ UT_TEST(test_revoke_handler_silent_refusal_arms_all_note) * the streak resets only after exact READY publication. */ UT_ASSERT_NOT_NULL(handler); if (handler != NULL) - UT_ASSERT(count_occurrences(handler, "gcs_block_pcm_x_revoke_refusal_note(") >= 6); + UT_ASSERT(count_occurrences(handler, "gcs_block_pcm_x_revoke_refusal_note_exact(") >= 6); UT_ASSERT_NOT_NULL(materialize); if (materialize != NULL) { UT_ASSERT_NOT_NULL(strstr(materialize, "\"materialize-copy\"")); @@ -3751,8 +4407,844 @@ UT_TEST(test_revoke_handler_silent_refusal_arms_all_note) : NULL; reset = publish != NULL ? strstr(publish, "gcs_block_pcm_x_revoke_refusal_note(NULL, 0, NULL)") : NULL; + if (reset == NULL && publish != NULL) + reset = strstr(publish, + "gcs_block_pcm_x_revoke_refusal_note_exact(NULL, 0, NULL, NULL, 0, 0)"); UT_ASSERT_NOT_NULL(publish); UT_ASSERT_NOT_NULL(reset); + UT_ASSERT_NOT_NULL(strstr(source, "request_id=%llu wait_seq=%llu")); + UT_ASSERT_NOT_NULL(strstr(source, "image_id=%llu source_generation=%llu")); + UT_ASSERT_NOT_NULL(strstr(source, "own_generation=%llu token=%llu")); + UT_ASSERT_NOT_NULL(strstr(source, "materialized-finish-vm-fsm-pinned")); + UT_ASSERT_NOT_NULL(strstr(source, "materialized-finish-io-in-progress")); + UT_ASSERT_NOT_NULL(strstr(source, "materialized-finish-live-flags")); + UT_ASSERT_NOT_NULL(strstr(source, "finish_refusal=%u shared_refcount=%u")); + UT_ASSERT_NOT_NULL(strstr(source, "bm_io_in_progress=%s live_flags=%u live_token=%llu")); + free(source); +} + + +UT_TEST(test_local_master_read_image_retries_holder_busy_with_fresh_identity) +{ + char *source = read_gcs_block_source(); + const char *read_image + = source != NULL ? strstr(source, "\ncluster_gcs_local_master_read_image_and_wait(") : NULL; + const char *read_image_end = read_image != NULL ? strstr(read_image, "\n}\n") : NULL; + const char *budget; + const char *loop; + const char *backoff; + const char *fresh_id; + const char *slot_id; + const char *forward_id; + const char *send; + const char *retryable_deny; + const char *retry; + const char *terminal_error; + + /* P0-21 completion: a DATA worker must never wait behind BufferContent. + * Its conditional-copy refusal is HC105 transient, so the local-master + * requester retries with bounded backoff. Each re-arm mints a new request + * id, preventing a delayed denial from an earlier attempt from winning the + * next attempt's slot. */ + UT_ASSERT_NOT_NULL(read_image); + UT_ASSERT_NOT_NULL(read_image_end); + if (read_image == NULL || read_image_end == NULL) { + free(source); + return; + } + budget = strstr(read_image, "cluster_gcs_block_retransmit_max_retries"); + loop = budget != NULL ? strstr(budget, "for (retry_attempt = 0;") : NULL; + backoff = loop != NULL ? strstr(loop, "gcs_block_backoff_ms_for_retry(retry_attempt)") : NULL; + fresh_id = backoff != NULL ? strstr(backoff, "gcs_block_pcm_x_next_request_id(&request_id)") : NULL; + slot_id = fresh_id != NULL ? strstr(fresh_id, "slot->request_id = request_id") : NULL; + forward_id = slot_id != NULL ? strstr(slot_id, "fwd.request_id = request_id") : NULL; + send = forward_id != NULL + ? strstr(forward_id, "cluster_grd_outbound_enqueue_backend_msg(") + : NULL; + retryable_deny + = send != NULL ? strstr(send, "GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER") : NULL; + retry = retryable_deny != NULL ? strstr(retryable_deny, "continue;") : NULL; + terminal_error = retry != NULL ? strstr(retry, "could not obtain read image from X holder") : NULL; + UT_ASSERT_NOT_NULL(budget); + UT_ASSERT_NOT_NULL(loop); + UT_ASSERT_NOT_NULL(backoff); + UT_ASSERT_NOT_NULL(fresh_id); + UT_ASSERT_NOT_NULL(slot_id); + UT_ASSERT_NOT_NULL(forward_id); + UT_ASSERT_NOT_NULL(send); + UT_ASSERT_NOT_NULL(retryable_deny); + UT_ASSERT_NOT_NULL(retry); + UT_ASSERT_NOT_NULL(terminal_error); + if (budget != NULL && loop != NULL && backoff != NULL && fresh_id != NULL && slot_id != NULL + && forward_id != NULL && send != NULL && retryable_deny != NULL && retry != NULL + && terminal_error != NULL) + UT_ASSERT(read_image < budget && budget < loop && loop < backoff && backoff < fresh_id + && fresh_id < slot_id && slot_id < forward_id && forward_id < send + && send < retryable_deny && retryable_deny < retry && retry < terminal_error + && terminal_error < read_image_end); + free(source); +} + +UT_TEST(test_local_master_read_image_stops_retrying_displaced_holder_exactly) +{ + char *source = read_gcs_block_source(); + const char *read_image + = source != NULL ? strstr(source, "\ncluster_gcs_local_master_read_image_and_wait(") : NULL; + const char *read_image_end = read_image != NULL ? strstr(read_image, "\n}\n") : NULL; + const char *expected_arg; + const char *retry_arg; + const char *precheck; + const char *reserve; + const char *backoff; + const char *backoff_check; + const char *fresh_id; + const char *denial; + const char *denial_check; + const char *drift_retry; + const char *release; + const char *drift_return; + const char *terminal_error; + + /* P0-21 residual: a conditional refusal is retryable only while the + * complete remote-X authority token still names the same holder. Once a + * queue handoff displaces that token, this helper must stop spending its + * bounded budget on the old node and return the retry boundary to bufmgr; + * the outer GRANT_PENDING abort/rearm owns fresh authority selection. */ + UT_ASSERT_NOT_NULL(read_image); + UT_ASSERT_NOT_NULL(read_image_end); + if (read_image == NULL || read_image_end == NULL) { + free(source); + return; + } + expected_arg = strstr(read_image, "const PcmAuthoritySnapshot *expected"); + retry_arg = expected_arg != NULL ? strstr(expected_arg, "bool *out_retry_denied") : NULL; + precheck = retry_arg != NULL + ? strstr(retry_arg, "if (!cluster_pcm_lock_authority_matches(tag, expected))") + : NULL; + reserve = precheck != NULL ? strstr(precheck, "gcs_block_reserve_slot(") : NULL; + backoff = reserve != NULL ? strstr(reserve, "gcs_block_backoff_ms_for_retry(retry_attempt)") + : NULL; + backoff_check = backoff != NULL + ? strstr(backoff, "if (!cluster_pcm_lock_authority_matches(tag, expected))") + : NULL; + fresh_id = backoff_check != NULL + ? strstr(backoff_check, "gcs_block_pcm_x_next_request_id(&request_id)") + : NULL; + denial = fresh_id != NULL + ? strstr(fresh_id, "GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER") + : NULL; + denial_check = denial != NULL + ? strstr(denial, "if (!cluster_pcm_lock_authority_matches(tag, expected))") + : NULL; + drift_retry = denial_check != NULL ? strstr(denial_check, "*out_retry_denied = true") : NULL; + release = drift_retry != NULL ? strstr(drift_retry, "gcs_block_release_slot(slot)") : NULL; + drift_return = release != NULL ? strstr(release, "if (*out_retry_denied)") : NULL; + terminal_error = drift_return != NULL + ? strstr(drift_return, "could not obtain read image from X holder") + : NULL; + + UT_ASSERT_NOT_NULL(expected_arg); + UT_ASSERT_NOT_NULL(retry_arg); + UT_ASSERT_NOT_NULL(precheck); + UT_ASSERT_NOT_NULL(reserve); + UT_ASSERT_NOT_NULL(backoff); + UT_ASSERT_NOT_NULL(backoff_check); + UT_ASSERT_NOT_NULL(fresh_id); + UT_ASSERT_NOT_NULL(denial); + UT_ASSERT_NOT_NULL(denial_check); + UT_ASSERT_NOT_NULL(drift_retry); + UT_ASSERT_NOT_NULL(release); + UT_ASSERT_NOT_NULL(drift_return); + UT_ASSERT_NOT_NULL(terminal_error); + if (expected_arg != NULL && retry_arg != NULL && precheck != NULL && reserve != NULL + && backoff != NULL && backoff_check != NULL && fresh_id != NULL && denial != NULL + && denial_check != NULL && drift_retry != NULL && release != NULL + && drift_return != NULL && terminal_error != NULL) + UT_ASSERT(read_image < expected_arg && expected_arg < retry_arg && retry_arg < precheck + && precheck < reserve && reserve < backoff && backoff < backoff_check + && backoff_check < fresh_id && fresh_id < denial && denial < denial_check + && denial_check < drift_retry && drift_retry < release && release < drift_return + && drift_return < terminal_error && terminal_error < read_image_end); + free(source); +} + +UT_TEST(test_local_master_read_image_refusal_evidence_is_attempt_exact) +{ + char *source = read_gcs_block_source(); + char *t400 = read_source_path(T400_SOURCE_PATH); + const char *read_image + = source != NULL ? strstr(source, "\ncluster_gcs_local_master_read_image_and_wait(") : NULL; + const char *read_image_end = read_image != NULL ? strstr(read_image, "\n}\n") : NULL; + const char *attempts = read_image != NULL ? strstr(read_image, "attempts=%d") : NULL; + const char *last_status = attempts != NULL ? strstr(attempts, "last_status=%d") : NULL; + const char *forward + = source != NULL ? strstr(source, "\ncluster_gcs_handle_block_forward_envelope(") : NULL; + const char *forward_end = forward != NULL ? strstr(forward, "\n}\n") : NULL; + const char *refusal_name + = forward != NULL ? strstr(forward, "cluster_bufmgr_gcs_copy_refusal_name") : NULL; + const char *refusal_log + = refusal_name != NULL ? strstr(refusal_name, "holder ship image refused") : NULL; + const char *request_id + = refusal_log != NULL ? strstr(refusal_log, "request_id=") : NULL; + + /* P0-21 observation only: the requester terminal error reports its exact + * bounded-attempt result, while the holder process log binds each copy + * refusal reason to the forwarded request identity. */ + UT_ASSERT_NOT_NULL(read_image); + UT_ASSERT_NOT_NULL(read_image_end); + UT_ASSERT_NOT_NULL(attempts); + UT_ASSERT_NOT_NULL(last_status); + if (read_image_end != NULL && attempts != NULL && last_status != NULL) + UT_ASSERT(attempts < last_status && last_status < read_image_end); + UT_ASSERT_NOT_NULL(forward); + UT_ASSERT_NOT_NULL(forward_end); + UT_ASSERT_NOT_NULL(refusal_name); + UT_ASSERT_NOT_NULL(refusal_log); + UT_ASSERT_NOT_NULL(request_id); + if (forward_end != NULL && refusal_name != NULL && refusal_log != NULL && request_id != NULL) + UT_ASSERT(refusal_name < refusal_log && refusal_log < request_id && request_id < forward_end); + UT_ASSERT_NOT_NULL(t400); + if (t400 != NULL) { + const char *holder_before = strstr(t400, "holder_evicted_before_by_node"); + const char *holder_after + = holder_before != NULL ? strstr(holder_before, "holder_evicted_after_by_node") : NULL; + const char *holder_diag + = holder_after != NULL ? strstr(holder_after, "holder copy refusal baseline") : NULL; + const char *holder_final + = holder_diag != NULL ? strstr(holder_diag, "'block_forward_holder_evicted_count'") + : NULL; + + UT_ASSERT_NOT_NULL(holder_before); + UT_ASSERT_NOT_NULL(holder_after); + UT_ASSERT_NOT_NULL(holder_diag); + UT_ASSERT_NOT_NULL(holder_final); + if (holder_before != NULL && holder_after != NULL && holder_diag != NULL + && holder_final != NULL) + UT_ASSERT(holder_before < holder_after && holder_after < holder_diag + && holder_diag < holder_final); + } + free(source); + free(t400); +} + +UT_TEST(test_local_master_x_transfer_revalidates_exact_authority_and_retries_stale) +{ + char *source = read_gcs_block_source(); + const char *transfer + = source != NULL ? strstr(source, "\ncluster_gcs_local_master_x_transfer_and_wait(") : NULL; + const char *transfer_end = transfer != NULL ? strstr(transfer, "\n}\n") : NULL; + const char *precheck; + const char *reserve; + const char *forward_id; + const char *send; + const char *installed; + const char *commit; + const char *stale; + const char *not_found; + const char *commit_retry; + const char *postcheck; + const char *post_retry; + const char *legacy_terminal; + + /* P0-26: carry one complete authority token from the local-master + * decision through the wire request. Reject drift before sending; after + * an installed reply, atomically commit only that token. A displaced + * token or a denial/timeout after authority drift returns retryable so + * bufmgr aborts GRANT_PENDING and mints a fresh token/request identity. */ + UT_ASSERT_NOT_NULL(transfer); + UT_ASSERT_NOT_NULL(transfer_end); + if (transfer == NULL || transfer_end == NULL) { + free(source); + return; + } + precheck = strstr(transfer, "if (!cluster_pcm_lock_authority_matches(tag, expected))"); + reserve = precheck != NULL ? strstr(precheck, "gcs_block_reserve_slot(") : NULL; + forward_id = reserve != NULL ? strstr(reserve, "fwd.request_id = request_id") : NULL; + send = forward_id != NULL ? strstr(forward_id, "cluster_grd_outbound_enqueue_backend_msg(") + : NULL; + installed = send != NULL ? strstr(send, "if (installed)") : NULL; + commit = installed != NULL ? strstr(installed, "cluster_pcm_lock_master_take_x_after_transfer(") + : NULL; + stale = commit != NULL ? strstr(commit, "PCM_X_TRANSFER_COMMIT_STALE") : NULL; + not_found = stale != NULL ? strstr(stale, "PCM_X_TRANSFER_COMMIT_NOT_FOUND") : NULL; + commit_retry = not_found != NULL ? strstr(not_found, "*out_retry_denied = true") : NULL; + postcheck + = commit_retry != NULL + ? strstr(commit_retry, "if (!cluster_pcm_lock_authority_matches(tag, expected))") + : NULL; + post_retry = postcheck != NULL ? strstr(postcheck, "*out_retry_denied = true") : NULL; + legacy_terminal = post_retry != NULL ? strstr(post_retry, "if (read_image)") : NULL; + + UT_ASSERT_NOT_NULL(precheck); + UT_ASSERT_NOT_NULL(reserve); + UT_ASSERT_NOT_NULL(forward_id); + UT_ASSERT_NOT_NULL(send); + UT_ASSERT_NOT_NULL(installed); + UT_ASSERT_NOT_NULL(commit); + UT_ASSERT_NOT_NULL(stale); + UT_ASSERT_NOT_NULL(not_found); + UT_ASSERT_NOT_NULL(commit_retry); + UT_ASSERT_NOT_NULL(postcheck); + UT_ASSERT_NOT_NULL(post_retry); + UT_ASSERT_NOT_NULL(legacy_terminal); + if (precheck != NULL && reserve != NULL && forward_id != NULL && send != NULL + && installed != NULL && commit != NULL && stale != NULL && not_found != NULL + && commit_retry != NULL && postcheck != NULL && post_retry != NULL + && legacy_terminal != NULL) + UT_ASSERT(transfer < precheck && precheck < reserve && reserve < forward_id + && forward_id < send && send < installed && installed < commit && commit < stale + && stale < not_found && not_found < commit_retry && commit_retry < postcheck + && postcheck < post_retry && post_retry < legacy_terminal + && legacy_terminal < transfer_end); + free(source); +} + +UT_TEST(test_remote_downgrade_prepares_exact_image_before_notify_and_reply) +{ + char *gcs_source = read_gcs_block_source(); + char *bufmgr_source = read_source_path("../../backend/storage/buffer/bufmgr.c"); + const char *forward = gcs_source != NULL + ? strstr(gcs_source, "\ncluster_gcs_handle_block_forward_envelope(") : NULL; + const char *forward_end = forward != NULL ? strstr(forward, "\n}\n") : NULL; + const char *inject = forward != NULL + ? strstr(forward, "cluster-gcs-block-evict-holder-before-ship") : NULL; + const char *downgrade = inject != NULL + ? strstr(inject, "cluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image(") : NULL; + const char *reuse = downgrade != NULL ? strstr(downgrade, "holder_ship_ok = remote_downgraded") + : NULL; + const char *post_notify_guard = downgrade != NULL + ? strstr(downgrade, + "remote_downgrade_outcome == CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY") + : NULL; + const char *post_notify_return + = post_notify_guard != NULL ? strstr(post_notify_guard, "return;") : NULL; + const char *copy = reuse != NULL ? strstr(reuse, "gcs_block_get_ship_image(") : NULL; + const char *helper = bufmgr_source != NULL + ? strstr(bufmgr_source, "\ncluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image(") + : NULL; + const char *helper_end = helper != NULL ? strstr(helper, "\n}\n") : NULL; + const char *reserve = helper != NULL ? strstr(helper, "cluster_bufmgr_pcm_own_begin_x_revoke(") + : NULL; + const char *precopy = reserve != NULL ? strstr(reserve, "memcpy(dst,") : NULL; + const char *notify = precopy != NULL + ? strstr(precopy, "cluster_gcs_send_transition_nowait(") : NULL; + const char *commit = notify != NULL + ? strstr(notify, "cluster_bufmgr_pcm_own_finish_x_to_s_downgrade(") : NULL; + const char *abort = helper != NULL + ? strstr(helper, "cluster_bufmgr_pcm_own_abort_x_revoke(") : NULL; + const char *local_helper = bufmgr_source != NULL + ? strstr(bufmgr_source, "\ncluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image(") + : NULL; + const char *local_helper_end = local_helper != NULL ? strstr(local_helper, "\n}\n") : NULL; + const char *local_reserve = local_helper != NULL + ? strstr(local_helper, "cluster_bufmgr_pcm_own_begin_x_revoke(") : NULL; + const char *local_copy = local_reserve != NULL ? strstr(local_reserve, "memcpy(dst,") : NULL; + const char *local_master = local_copy != NULL + ? strstr(local_copy, "cluster_pcm_lock_apply_gcs_transition(") : NULL; + const char *local_commit = local_master != NULL + ? strstr(local_master, "cluster_bufmgr_pcm_own_finish_x_to_s_downgrade(") : NULL; + const char *local_call = gcs_source != NULL + ? strstr(gcs_source, "cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image(") : NULL; + const char *prepared = local_call != NULL ? strstr(local_call, "scache_image_prepared = true") + : NULL; + const char *produce = prepared != NULL + ? strstr(prepared, "gcs_block_produce_reply(req, block_buf, scache_image_prepared") : NULL; + const char *produce_def = gcs_source != NULL ? strstr(gcs_source, "\ngcs_block_produce_reply(") + : NULL; + const char *skip_second_copy = produce_def != NULL + ? strstr(produce_def, "if (!preprepared_image\n\t\t&& !gcs_block_get_ship_image(") : NULL; + + /* P0-32: injection is decided before any irreversible downgrade. The + * successful arm reuses bytes proven under the same content EXCLUSIVE + * interval; it must not take a second conditional copy. */ + UT_ASSERT_NOT_NULL(forward); + UT_ASSERT_NOT_NULL(forward_end); + UT_ASSERT_NOT_NULL(inject); + UT_ASSERT_NOT_NULL(downgrade); + UT_ASSERT_NOT_NULL(reuse); + UT_ASSERT_NOT_NULL(post_notify_guard); + UT_ASSERT_NOT_NULL(post_notify_return); + UT_ASSERT_NOT_NULL(copy); + if (forward_end != NULL && inject != NULL && downgrade != NULL && reuse != NULL && copy != NULL) + UT_ASSERT(inject < downgrade && downgrade < reuse && reuse < copy && copy < forward_end); + if (post_notify_guard != NULL && post_notify_return != NULL && copy != NULL) + UT_ASSERT(downgrade < post_notify_guard && post_notify_guard < post_notify_return + && post_notify_return < copy); + + /* The remote holder's only irreversible order is exact reserve, flush + + * pre-copy/LSN proof, nowait notify, then exact X+REVOKING -> S commit. + * Pre-notify refusal has an exact abort; post-notify commit failure is + * fail-closed and therefore cannot produce S_GRANTED. */ + UT_ASSERT_NOT_NULL(helper); + UT_ASSERT_NOT_NULL(helper_end); + UT_ASSERT_NOT_NULL(reserve); + UT_ASSERT_NOT_NULL(precopy); + UT_ASSERT_NOT_NULL(notify); + UT_ASSERT_NOT_NULL(commit); + UT_ASSERT_NOT_NULL(abort); + if (helper_end != NULL && reserve != NULL && precopy != NULL && notify != NULL && commit != NULL) + UT_ASSERT(reserve < precopy && precopy < notify && notify < commit && commit < helper_end); + if (helper_end != NULL && abort != NULL) + UT_ASSERT(abort < notify && abort < helper_end); + UT_ASSERT_NOT_NULL(helper != NULL ? strstr(helper, "cluster_pcm_x_runtime_fail_closed()") : NULL); + UT_ASSERT_NOT_NULL(local_helper); + UT_ASSERT_NOT_NULL(local_helper_end); + UT_ASSERT_NOT_NULL(local_reserve); + UT_ASSERT_NOT_NULL(local_copy); + UT_ASSERT_NOT_NULL(local_master); + UT_ASSERT_NOT_NULL(local_commit); + if (local_helper_end != NULL && local_reserve != NULL && local_copy != NULL + && local_master != NULL && local_commit != NULL) + UT_ASSERT(local_reserve < local_copy && local_copy < local_master + && local_master < local_commit && local_commit < local_helper_end); + UT_ASSERT_NOT_NULL(local_call); + UT_ASSERT_NOT_NULL(prepared); + UT_ASSERT_NOT_NULL(produce); + UT_ASSERT_NOT_NULL(skip_second_copy); + if (local_call != NULL && prepared != NULL && produce != NULL) + UT_ASSERT(local_call < prepared && prepared < produce); + free(gcs_source); + free(bufmgr_source); +} + + +UT_TEST(test_pending_x_apply_race_maps_to_retryable_block_denial) +{ + char *source = read_gcs_block_source(); + const char *produce = source != NULL ? strstr(source, "\ngcs_block_produce_reply(") : NULL; + const char *produce_end + = produce != NULL + ? strstr(produce, "\n}\n\nstatic bool\ngcs_block_queue_pending_x_authoritative(") + : NULL; + const char *apply1; + const char *deny1; + const char *apply2; + const char *deny2; + const char *requester_retry; + + /* P0-25: the entry-lock final admission distinguishes a raced pending-X + * from structural incompatibility. Both direct-storage and image-present + * N->S apply sites must return DENIED_PENDING_X, whose requester path exits + * through the existing fresh-identity retry boundary instead of a client + * terminal ERROR. */ + UT_ASSERT_NOT_NULL(produce); + UT_ASSERT_NOT_NULL(produce_end); + if (produce == NULL || produce_end == NULL) { + free(source); + return; + } + apply1 = strstr(produce, "cluster_pcm_lock_apply_gcs_transition_result("); + deny1 = apply1 != NULL ? strstr(apply1, "GcsBlockApplyRefusalStatus(") : NULL; + apply2 = deny1 != NULL ? strstr(deny1, "cluster_pcm_lock_apply_gcs_transition_result(") : NULL; + deny2 = apply2 != NULL ? strstr(apply2, "GcsBlockApplyRefusalStatus(") : NULL; + requester_retry = strstr(source, "if (final_status == GCS_BLOCK_REPLY_DENIED_PENDING_X)"); + UT_ASSERT_NOT_NULL(apply1); + UT_ASSERT_NOT_NULL(deny1); + UT_ASSERT_NOT_NULL(apply2); + UT_ASSERT_NOT_NULL(deny2); + UT_ASSERT_NOT_NULL(requester_retry); + if (apply1 != NULL && deny1 != NULL && apply2 != NULL && deny2 != NULL) + UT_ASSERT(produce < apply1 && apply1 < deny1 && deny1 < apply2 && apply2 < deny2 + && deny2 < produce_end); + free(source); +} + + +UT_TEST(test_gcs_apply_state_drift_restarts_with_fresh_request_identity) +{ + char *source = read_gcs_block_source(); + const char *produce = source != NULL ? strstr(source, "\ngcs_block_produce_reply(") : NULL; + const char *produce_end + = produce != NULL + ? strstr(produce, "\n}\n\nstatic bool\ngcs_block_queue_pending_x_authoritative(") + : NULL; + const char *apply1; + const char *map1; + const char *apply2; + const char *map2; + + /* The request transition was already wire-validated. If authority moves + * after the master's outer S/X decision but before its entry-lock apply, + * N->S/N->X must abandon this attempt and re-enter bufmgr with a fresh + * request/token identity. Other transition incompatibilities remain + * structural terminal denials. */ + UT_ASSERT_EQ((int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_PENDING_X, + PCM_TRANS_N_TO_S), + (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ((int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_INCOMPATIBLE, + PCM_TRANS_N_TO_S), + (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ((int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_INCOMPATIBLE, + PCM_TRANS_N_TO_X), + (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ((int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_INCOMPATIBLE, + PCM_TRANS_S_TO_X_UPGRADE), + (int)GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE); + + UT_ASSERT_NOT_NULL(produce); + UT_ASSERT_NOT_NULL(produce_end); + if (produce == NULL || produce_end == NULL) { + free(source); + return; + } + apply1 = strstr(produce, "cluster_pcm_lock_apply_gcs_transition_result("); + map1 = apply1 != NULL ? strstr(apply1, "GcsBlockApplyRefusalStatus(") : NULL; + apply2 = map1 != NULL ? strstr(map1, "cluster_pcm_lock_apply_gcs_transition_result(") : NULL; + map2 = apply2 != NULL ? strstr(apply2, "GcsBlockApplyRefusalStatus(") : NULL; + UT_ASSERT_NOT_NULL(apply1); + UT_ASSERT_NOT_NULL(map1); + UT_ASSERT_NOT_NULL(apply2); + UT_ASSERT_NOT_NULL(map2); + if (apply1 != NULL && map1 != NULL && apply2 != NULL && map2 != NULL) + UT_ASSERT(produce < apply1 && apply1 < map1 && map1 < apply2 && apply2 < map2 + && map2 < produce_end); + free(source); +} + + +UT_TEST(test_master_direct_copy_busy_uses_only_fresh_identity_retry_boundary) +{ + struct CopyRefusalCase { + ClusterBufmgrGcsCopyRefusal refusal; + GcsBlockReplyStatus expected; + }; + static const struct CopyRefusalCase cases[] = { + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE, + GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INVALID_ARGUMENT, + GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT, + GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID, + GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST, + GCS_BLOCK_REPLY_DENIED_PENDING_X }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_SECOND, + GCS_BLOCK_REPLY_DENIED_PENDING_X }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY, + GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_HC89_LSN_DRIFT, + GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_SMART_FUSION_UNCLASSIFIED, + GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INJECTED_EVICT, + GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + }; + char *source = read_gcs_block_source(); + const char *produce = source != NULL ? strstr(source, "\ngcs_block_produce_reply(") : NULL; + const char *produce_end + = produce != NULL + ? strstr(produce, "\n}\n\nstatic bool\ngcs_block_queue_pending_x_authoritative(") + : NULL; + const char *copy_refusal; + const char *copy_refusal_end; + const char *mapping; + const char *terminal_literal; + const char *requester_retry; + const char *retry_flag; + const char *retry_break; + const char *same_id_continue; + size_t i; + + /* P0-21 residual: a DATA worker's conditional BufferContent refusal is + * transient by construction (the lock owner may itself be waiting for that + * worker), so only the two conditional stages enter the existing + * DENIED_PENDING_X fresh-request/token retry boundary. Structural + * residency/current-image failures and HC89's bounded second LSN drift stay + * terminal status 6; no authority or safety gate is weakened. */ + for (i = 0; i < lengthof(cases); i++) + UT_ASSERT_EQ((int)GcsBlockMasterDirectCopyRefusalStatus(cases[i].refusal), + (int)cases[i].expected); + + UT_ASSERT_NOT_NULL(produce); + UT_ASSERT_NOT_NULL(produce_end); + if (produce == NULL || produce_end == NULL) { + free(source); + return; + } + copy_refusal = strstr(produce, "produce-copy-refused:%s"); + copy_refusal_end = copy_refusal != NULL ? strstr(copy_refusal, "return true;") : NULL; + mapping = copy_refusal != NULL + ? strstr(copy_refusal, "GcsBlockMasterDirectCopyRefusalStatus(copy_refusal)") + : NULL; + terminal_literal = copy_refusal != NULL + ? strstr(copy_refusal, + "*out_status = GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER") + : NULL; + UT_ASSERT_NOT_NULL(copy_refusal); + UT_ASSERT_NOT_NULL(copy_refusal_end); + UT_ASSERT_NOT_NULL(mapping); + if (copy_refusal != NULL && copy_refusal_end != NULL && mapping != NULL) + UT_ASSERT(copy_refusal < mapping && mapping < copy_refusal_end); + UT_ASSERT(terminal_literal == NULL || copy_refusal_end == NULL + || terminal_literal > copy_refusal_end); + + /* Status 10 must leave the per-call retransmit loop rather than retrying + * the same cached request id. Bufmgr then aborts the exact old reservation, + * waits, and rearms a fresh token/request identity (covered by + * test_pending_x_denied_retry_leaves_master_invalidate_gap). */ + requester_retry + = strstr(source, "if (final_status == GCS_BLOCK_REPLY_DENIED_PENDING_X)"); + retry_flag = requester_retry != NULL ? strstr(requester_retry, "retry_denied = true;") : NULL; + retry_break = retry_flag != NULL ? strstr(retry_flag, "break;") : NULL; + same_id_continue = retry_flag != NULL ? strstr(retry_flag, "continue;") : NULL; + UT_ASSERT_NOT_NULL(requester_retry); + UT_ASSERT_NOT_NULL(retry_flag); + UT_ASSERT_NOT_NULL(retry_break); + if (requester_retry != NULL && retry_flag != NULL && retry_break != NULL) + UT_ASSERT(requester_retry < retry_flag && retry_flag < retry_break + && retry_break < produce); + UT_ASSERT(same_id_continue == NULL || retry_break == NULL || same_id_continue > retry_break); + free(source); +} + + +UT_TEST(test_master_not_holder_producers_log_one_coherent_authority_snapshot) +{ + char *source = read_gcs_block_source(); + const char *helper + = source != NULL ? strstr(source, "\ngcs_block_log_master_not_holder_producer(") : NULL; + const char *helper_end = helper != NULL ? strstr(helper, "\n}\n") : NULL; + const char *produce = source != NULL ? strstr(source, "\ngcs_block_produce_reply(") : NULL; + const char *produce_end + = produce != NULL + ? strstr(produce, "\n}\n\nstatic bool\ngcs_block_queue_pending_x_authoritative(") + : NULL; + static const char *const required_fields[] = { + "cluster_pcm_lock_authority_snapshot(tag, &authority)", + "authority.state", + "authority.x_holder_node", + "authority.s_holders_bitmap", + "authority.master_holder.node_id", + "authority.master_holder.procno", + "authority.master_holder.cluster_epoch", + "authority.master_holder.request_id", + "authority.pending_x_requester_node", + "authority.pending_x_since_lsn", + "authority.transition_count", + }; + static const char *const required_reasons[] = { + "direct-land-nonsendable", + "direct-land-forward-rearm", + "pcm-x-image-not-ready", + "produce-no-resident-authority", + "produce-copy-refused", + "clean-third-party-master", + "live-x-other-holder", + "pending-x-reserve-failed", + "x-forward-send-failed", + "x-state-holder-unroutable", + "holder-immediate-deny", + "holder-drop-pinned", + "holder-drop-stale", + "holder-copy-refused", + }; + size_t i; + + /* A status=6 requester error is not enough to identify its producer: the + * aggregate counter is also incremented at requester consumption. Every + * producer therefore logs a stable reason plus one entry-lock-coherent + * authority snapshot, correlated by the unchanged request identity. */ + UT_ASSERT_NOT_NULL(helper); + UT_ASSERT_NOT_NULL(helper_end); + if (helper != NULL && helper_end != NULL) { + const char *snapshot; + const char *duplicate_snapshot; + const char *forbidden; + + for (i = 0; i < lengthof(required_fields); i++) { + const char *field = strstr(helper, required_fields[i]); + + UT_ASSERT_NOT_NULL(field); + if (field != NULL) + UT_ASSERT(field < helper_end); + } + snapshot = strstr(helper, "cluster_pcm_lock_authority_snapshot("); + duplicate_snapshot + = snapshot != NULL ? strstr(snapshot + 1, "cluster_pcm_lock_authority_snapshot(") : NULL; + UT_ASSERT_NOT_NULL(snapshot); + UT_ASSERT(snapshot != NULL && snapshot < helper_end); + UT_ASSERT(duplicate_snapshot == NULL || duplicate_snapshot >= helper_end); + forbidden = strstr(helper, "cluster_pcm_lock_query("); + UT_ASSERT(forbidden == NULL || forbidden >= helper_end); + forbidden = strstr(helper, "cluster_pcm_master_holder_node_by_tag("); + UT_ASSERT(forbidden == NULL || forbidden >= helper_end); + forbidden = strstr(helper, "cluster_pcm_lock_query_s_holders_bitmap("); + UT_ASSERT(forbidden == NULL || forbidden >= helper_end); + } + for (i = 0; i < lengthof(required_reasons); i++) + UT_ASSERT_NOT_NULL(source != NULL ? strstr(source, required_reasons[i]) : NULL); + /* The final master-direct copy refusal must retain the bufmgr's exact + * nonblocking stage in the producer reason; no status/retry change here. */ + UT_ASSERT_NOT_NULL(produce); + UT_ASSERT_NOT_NULL(produce_end); + if (produce != NULL && produce_end != NULL) { + const char *capture = strstr(produce, "ClusterBufmgrGcsCopyRefusal copy_refusal"); + const char *pass = strstr(produce, "©_refusal"); + const char *name = strstr(produce, "cluster_bufmgr_gcs_copy_refusal_name(copy_refusal)"); + + UT_ASSERT_NOT_NULL(capture); + UT_ASSERT_NOT_NULL(pass); + UT_ASSERT_NOT_NULL(name); + if (capture != NULL && pass != NULL && name != NULL) + UT_ASSERT(produce < capture && capture < pass && pass < name && name < produce_end); + } + free(source); +} + + +UT_TEST(test_pi_durable_note_drain_stages_before_consuming_on_data_plane) +{ + char *source = read_gcs_block_source(); + const char *drain = strstr(source, "\ncluster_gcs_block_pi_discard_drain("); + const char *drain_end = drain != NULL ? strstr(drain, "\n}\n") : NULL; + const char *data_plane; + const char *status3; + const char *shard; + const char *enqueue; + const char *enqueue_refusal; + const char *advance; + const char *data_continue; + const char *control_apply; + const char *control_send; + + /* + * Checkpoint-confirmed PI durable notes are produced in shared memory, + * but the DATA-plane drain runs only in LMS worker 0. It must therefore + * stage both local- and remote-master status-3 ACKs onto tag->worker, + * and may consume the source note only after that staging succeeds. The + * old CONTROL-plane direct apply/send path remains available. + */ + UT_ASSERT_NOT_NULL(drain); + UT_ASSERT_NOT_NULL(drain_end); + if (drain == NULL || drain_end == NULL) { + free(source); + return; + } + data_plane = strstr(drain, "cluster_gcs_block_family_on_data_plane()"); + status3 = strstr(drain, "GCS_BLOCK_INVALIDATE_ACK_STATUS_PI_DURABLE_NOTE"); + shard = strstr(drain, "cluster_lms_shard_for_tag(&tag, cluster_lms_workers)"); + enqueue = strstr(drain, "cluster_lms_outbound_enqueue("); + enqueue_refusal = enqueue != NULL ? strstr(enqueue, "break;") : NULL; + advance = enqueue != NULL ? strstr(enqueue, "ClusterGcsBlock->pi_note_drain_seq++") : NULL; + data_continue = advance != NULL ? strstr(advance, "continue;") : NULL; + control_apply = strstr(drain, "gcs_block_pi_discard_master_apply(tag, page_scn)"); + control_send = strstr(drain, "cluster_ic_send_envelope(PGRAC_IC_MSG_GCS_BLOCK_INVALIDATE_ACK"); + UT_ASSERT_NOT_NULL(data_plane); + UT_ASSERT_NOT_NULL(status3); + UT_ASSERT_NOT_NULL(shard); + UT_ASSERT_NOT_NULL(enqueue); + UT_ASSERT_NOT_NULL(enqueue_refusal); + UT_ASSERT_NOT_NULL(advance); + UT_ASSERT_NOT_NULL(data_continue); + UT_ASSERT_NOT_NULL(control_apply); + UT_ASSERT_NOT_NULL(control_send); + if (data_plane != NULL && status3 != NULL && shard != NULL && enqueue != NULL + && enqueue_refusal != NULL && advance != NULL && data_continue != NULL) + UT_ASSERT(data_plane < status3 && status3 < shard && shard < enqueue + && enqueue < enqueue_refusal && enqueue_refusal < advance + && advance < data_continue && data_continue < drain_end); + if (control_apply != NULL) + UT_ASSERT(data_continue < control_apply && control_apply < drain_end); + if (control_send != NULL) + UT_ASSERT(data_continue < control_send && control_send < drain_end); + free(source); +} + + +UT_TEST(test_pi_durable_note_receive_is_observable_before_apply) +{ + char *source = read_gcs_block_source(); + const char *handler + = source != NULL ? strstr(source, "\ncluster_gcs_handle_block_invalidate_ack_envelope(") + : NULL; + const char *status3 = handler != NULL + ? strstr(handler, "GCS_BLOCK_INVALIDATE_ACK_STATUS_PI_DURABLE_NOTE") + : NULL; + const char *epoch + = status3 != NULL ? strstr(status3, "ack->epoch == cluster_epoch_get_current()") : NULL; + const char *accepted = epoch != NULL ? strstr(epoch, "pi_durable_note_apply_count") : NULL; + const char *apply + = accepted != NULL ? strstr(accepted, "gcs_block_pi_discard_master_apply(") : NULL; + + UT_ASSERT_NOT_NULL(handler); + UT_ASSERT_NOT_NULL(status3); + UT_ASSERT_NOT_NULL(epoch); + UT_ASSERT_NOT_NULL(accepted); + UT_ASSERT_NOT_NULL(apply); + if (handler != NULL && status3 != NULL && epoch != NULL && accepted != NULL && apply != NULL) + UT_ASSERT(handler < status3 && status3 < epoch && epoch < accepted && accepted < apply); + free(source); +} + +/* P0-20 source-floor V2 must remain bound to the HELLO connection sampled + * around source authority. The reliable REVOKE leg is armed first; a later + * generation drift returns retryable without ACKing it, while the exact-match + * V2 enters a guarded LMS slot. A current peer without the additive bit gets + * the byte-compatible V1 fallback. */ +UT_TEST(test_pcm_x_source_floor_v2_is_connection_bound_until_lms_drain) +{ + char *source = read_gcs_block_source(); + const char *stage_bound; + const char *stage_bound_end; + const char *shard; + const char *enqueue; + const char *transfer; + const char *transfer_end; + const char *auth; + const char *arm; + const char *sample; + const char *generation; + const char *guarded_stage; + const char *v1_fallback; + const char *stray_simple_query; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + stage_bound = strstr(source, "\ngcs_block_pcm_x_stage_frame_cap_bound("); + stage_bound_end = stage_bound != NULL ? strstr(stage_bound, "\n}\n") : NULL; + shard = stage_bound != NULL ? strstr(stage_bound, "cluster_gcs_block_payload_shard(") : NULL; + enqueue = shard != NULL ? strstr(shard, "cluster_lms_outbound_enqueue_cap_bound(") : NULL; + UT_ASSERT_NOT_NULL(stage_bound); + UT_ASSERT_NOT_NULL(stage_bound_end); + UT_ASSERT_NOT_NULL(shard); + UT_ASSERT_NOT_NULL(enqueue); + if (stage_bound != NULL && stage_bound_end != NULL && shard != NULL && enqueue != NULL) + UT_ASSERT(stage_bound < shard && shard < enqueue && enqueue < stage_bound_end); + + transfer = strstr(source, "\ngcs_block_pcm_x_master_drive_transfer("); + transfer_end = transfer != NULL ? strstr(transfer, "\n}\n") : NULL; + auth = transfer != NULL + ? strstr(transfer, "gcs_block_pcm_x_authenticated_session_result(") + : NULL; + arm = auth != NULL ? strstr(auth, "cluster_pcm_x_master_revoke_arm_exact(") : NULL; + sample = arm != NULL ? strstr(arm, "cluster_sf_peer_pcm_x_source_floor_sample(") : NULL; + generation + = sample != NULL ? strstr(sample, "auth_sample.connection_generation_before") : NULL; + guarded_stage + = generation != NULL ? strstr(generation, "gcs_block_pcm_x_stage_frame_cap_bound(") : NULL; + v1_fallback = guarded_stage != NULL + ? strstr(guarded_stage, "cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_REVOKE") + : NULL; + stray_simple_query = transfer != NULL + ? strstr(transfer, "cluster_sf_peer_supports_pcm_x_source_floor(source)") + : NULL; + UT_ASSERT_NOT_NULL(transfer); + UT_ASSERT_NOT_NULL(transfer_end); + UT_ASSERT_NOT_NULL(auth); + UT_ASSERT_NOT_NULL(arm); + UT_ASSERT_NOT_NULL(sample); + UT_ASSERT_NOT_NULL(generation); + UT_ASSERT_NOT_NULL(guarded_stage); + UT_ASSERT_NOT_NULL(v1_fallback); + if (transfer != NULL && transfer_end != NULL && auth != NULL && arm != NULL && sample != NULL + && generation != NULL && guarded_stage != NULL && v1_fallback != NULL) { + UT_ASSERT(transfer < auth && auth < arm && arm < sample && sample < generation + && generation < guarded_stage && guarded_stage < v1_fallback + && v1_fallback < transfer_end); + UT_ASSERT(stray_simple_query == NULL || stray_simple_query > transfer_end); + } free(source); } @@ -3760,7 +5252,7 @@ UT_TEST(test_revoke_handler_silent_refusal_arms_all_note) int main(void) { - UT_PLAN(83); + UT_PLAN(103); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -3820,14 +5312,22 @@ main(void) UT_RUN(test_pcm_x_cancel_cleanup_classifies_exact_wfg_and_post_clear_failure); UT_RUN(test_pcm_x_terminal_retry_reclaims_cancel_cleanup_after_owner_death); UT_RUN(test_pcm_x_invalidate_ack_matches_only_exact_unacked_holder); + UT_RUN(test_pcm_x_invalidate_busy_routes_to_exact_ticket_backoff); + UT_RUN(test_pcm_x_local_pending_s_denial_match_is_attempt_exact); + UT_RUN(test_pcm_x_grant_pending_invalidate_wakes_local_s_before_busy); + UT_RUN(test_pcm_x_grant_pending_orphan_observation_is_identity_exact); UT_RUN(test_pcm_x_final_ack_builds_exact_grd_handoff_token); + UT_RUN(test_pcm_x_final_ack_fail_closed_names_exact_handoff_stage); UT_RUN(test_pcm_x_holder_image_evidence_never_uses_generation_as_presence); UT_RUN(test_pcm_x_pending_x_marker_is_only_a_pre_handoff_gate); UT_RUN(test_pcm_x_ready_publication_follows_exact_retained_commit); UT_RUN(test_pcm_x_ready_materializes_exact_n_s_or_x_source_without_wire_change); + UT_RUN(test_pcm_x_s_source_hard_failure_observation_is_reason_exact); UT_RUN(test_pcm_x_self_and_remote_drain_share_full_image_release_wrapper); UT_RUN(test_pcm_x_ready_admission_marks_before_send_and_rolls_back_refusal); UT_RUN(test_pcm_x_lms_owner_death_and_restart_audit_fail_closed); + UT_RUN(test_pcm_x_lms_reload_acknowledges_finish_flush_injection); + UT_RUN(test_pcm_x_destructive_finish_fault_times_out_in_sql_before_harness); UT_RUN(test_pcm_x_image_fetch_intercepts_canonical_id_before_generic_dedup); UT_RUN(test_pcm_x_requester_fetch_revalidates_queue_and_reservation_before_install); UT_RUN(test_pcm_x_self_source_handoff_is_no_copy_and_drain_preserves_x); @@ -3844,6 +5344,18 @@ main(void) UT_RUN(test_pcm_x_role_refresh_accepts_only_same_member_promotion); UT_RUN(test_legacy_byte_proof_sites_republish_kept_pi_mirror); UT_RUN(test_revoke_handler_silent_refusal_arms_all_note); + UT_RUN(test_local_master_read_image_retries_holder_busy_with_fresh_identity); + UT_RUN(test_local_master_read_image_stops_retrying_displaced_holder_exactly); + UT_RUN(test_local_master_read_image_refusal_evidence_is_attempt_exact); + UT_RUN(test_remote_downgrade_prepares_exact_image_before_notify_and_reply); + UT_RUN(test_local_master_x_transfer_revalidates_exact_authority_and_retries_stale); + UT_RUN(test_pending_x_apply_race_maps_to_retryable_block_denial); + UT_RUN(test_gcs_apply_state_drift_restarts_with_fresh_request_identity); + UT_RUN(test_master_direct_copy_busy_uses_only_fresh_identity_retry_boundary); + UT_RUN(test_master_not_holder_producers_log_one_coherent_authority_snapshot); + UT_RUN(test_pi_durable_note_drain_stages_before_consuming_on_data_plane); + UT_RUN(test_pi_durable_note_receive_is_observable_before_apply); + UT_RUN(test_pcm_x_source_floor_v2_is_connection_bound_until_lms_drain); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_gcs_block_dedup.c b/src/test/cluster_unit/test_cluster_gcs_block_dedup.c index 2289fe4f38..d0014fef40 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_dedup.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_dedup.c @@ -82,7 +82,6 @@ UT_DEFINE_GLOBALS(); - void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), @@ -472,7 +471,8 @@ stage_pcm_x_ready(int worker_id, const GcsBlockDedupKey *key, const BufferTag *t result = cluster_gcs_block_dedup_pcm_x_reserve(worker_id, key, tag, &reserved); if (result != GCS_BLOCK_PCM_X_IMAGE_RESERVED && result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) return result; - result = cluster_gcs_block_dedup_pcm_x_materialize(worker_id, key, tag, binding, hdr, page); + result = cluster_gcs_block_dedup_pcm_x_materialize(worker_id, key, tag, binding, UINT64_C(41), + (uint8)PCM_STATE_X, hdr, page); if (result != GCS_BLOCK_PCM_X_IMAGE_STORED && result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) return result; return cluster_gcs_block_dedup_pcm_x_publish_ready_exact(worker_id, key, tag, binding); @@ -1054,7 +1054,7 @@ UT_TEST(u16_capability_routing_truth_table) * field or increasing the entry size. */ UT_TEST(u17_pcm_x_binding_layout_is_zero_entry_growth) { - UT_ASSERT_EQ((int)sizeof(GcsBlockPcmXImageBinding), 136); + UT_ASSERT_EQ((int)sizeof(GcsBlockPcmXImageBinding), 144); UT_ASSERT_EQ((int)offsetof(GcsBlockDedupEntry, entry_kind), 46); UT_ASSERT_EQ((int)offsetof(GcsBlockDedupEntry, pcm_x_master_session), 48); UT_ASSERT_EQ((int)offsetof(GcsBlockDedupEntry, reply_header), 56); @@ -1099,6 +1099,7 @@ UT_TEST(u18_pcm_x_stage_duplicate_and_generic_overwrite_refused) UT_ASSERT_EQ((int)stage_pcm_x_ready(0, &key, &tag, &binding, &hdr, page), (int)GCS_BLOCK_PCM_X_IMAGE_DUPLICATE); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &conflicting_binding, + UINT64_C(41), (uint8)PCM_STATE_X, &conflicting_hdr, overwrite), (int)GCS_BLOCK_PCM_X_IMAGE_STALE); @@ -1186,14 +1187,14 @@ UT_TEST(u20_pcm_x_entry_survives_generic_gc_and_retires_exactly) UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &wrong, &cached), (int)GCS_BLOCK_PCM_X_IMAGE_STALE); UT_ASSERT_EQ((int)cached.entry_kind, (int)GCS_BLOCK_DEDUP_ENTRY_GENERIC); - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &wrong), + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &wrong, 2), (int)GCS_BLOCK_PCM_X_IMAGE_STALE); - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &binding), + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &binding, 2), (int)GCS_BLOCK_PCM_X_IMAGE_RELEASED); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &binding, &cached), (int)GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND); UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_release_count(), 1); - UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_failclosed_count(), 3); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_failclosed_count(), 2); } @@ -1263,7 +1264,7 @@ UT_TEST(u22_pcm_x_reserved_entry_waits_for_exact_release) cluster_gcs_block_dedup_cleanup_on_node_dead(1); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &reserved, &cached), (int)GCS_BLOCK_PCM_X_IMAGE_NOT_READY); - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &reserved), + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &reserved, -1), (int)GCS_BLOCK_PCM_X_IMAGE_RELEASED); UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_stage_count(), 0); UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_replay_count(), 0); @@ -1305,53 +1306,53 @@ UT_TEST(u23_pcm_x_materialize_validation_is_fail_closed_and_byte_stable) bad_binding.identity.image.source_node = 1; bad_hdr = hdr; bad_hdr.sender_node = 1; - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &bad_binding, &bad_hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &bad_binding, UINT64_C(41), (uint8)PCM_STATE_X, &bad_hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); bad_hdr = hdr; bad_hdr.sender_node = 1; - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &binding, &bad_hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &binding, UINT64_C(41), (uint8)PCM_STATE_X, &bad_hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); memcpy(bad_page, page, sizeof(bad_page)); bad_page[sizeof(bad_page) - 1] ^= 0x1; - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &binding, &hdr, bad_page), - (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &binding, UINT64_C(41), (uint8)PCM_STATE_X, &hdr, bad_page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); bad_binding = binding; bad_binding.identity.image.page_lsn++; bad_hdr = hdr; bad_hdr.page_lsn++; - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &bad_binding, &bad_hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &bad_binding, UINT64_C(41), (uint8)PCM_STATE_X, &bad_hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); bad_binding = binding; bad_binding.identity.image.page_scn++; - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &bad_binding, &hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &bad_binding, UINT64_C(41), (uint8)PCM_STATE_X, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); bad_binding = binding; bad_binding.identity.image.page_checksum++; bad_hdr = hdr; bad_hdr.checksum++; - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &bad_binding, &bad_hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &bad_binding, UINT64_C(41), (uint8)PCM_STATE_X, &bad_hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); bad_binding = binding; bad_binding.master_session++; - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &bad_binding, &hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_STALE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &bad_binding, UINT64_C(41), (uint8)PCM_STATE_X, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STALE); - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &binding, &hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &binding, UINT64_C(41), (uint8)PCM_STATE_X, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_stage_count(), 1); UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_failclosed_count(), 7); } @@ -1400,6 +1401,7 @@ UT_TEST(u25_pcm_x_work_prefers_reserved_and_marks_ready_staged) GcsBlockDedupKey reserved_key; GcsBlockPcmXImageBinding ready_binding; GcsBlockPcmXImageBinding reserved_binding; + GcsBlockPcmXImageBinding wrong_floor; GcsBlockPcmXImageWork work; GcsBlockReplyHeader hdr; uint64 ready_image_id; @@ -1424,9 +1426,15 @@ UT_TEST(u25_pcm_x_work_prefers_reserved_and_marks_ready_staged) reserved_binding.identity.image.page_scn = 0; reserved_binding.identity.image.page_lsn = 0; reserved_binding.identity.image.page_checksum = 0; + reserved_binding.required_page_scn = UINT64_C(72057594037950810); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &reserved_key, &reserved_tag, - &reserved_binding), + &reserved_binding), (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + wrong_floor = reserved_binding; + wrong_floor.required_page_scn++; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &reserved_key, &reserved_tag, + &wrong_floor), + (int)GCS_BLOCK_PCM_X_IMAGE_STALE); memset(&work, 0, sizeof(work)); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), @@ -1434,7 +1442,7 @@ UT_TEST(u25_pcm_x_work_prefers_reserved_and_marks_ready_staged) UT_ASSERT_EQ(memcmp(&work.key, &reserved_key, sizeof(reserved_key)), 0); UT_ASSERT_EQ(memcmp(&work.binding, &reserved_binding, sizeof(reserved_binding)), 0); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &reserved_key, &reserved_tag, - &reserved_binding), + &reserved_binding, -1), (int)GCS_BLOCK_PCM_X_IMAGE_RELEASED); memset(&work, 0, sizeof(work)); @@ -1577,8 +1585,9 @@ UT_TEST(u28_pcm_x_idle_hint_avoids_empty_rescan_and_reserve_rearms) /* Materialized bytes are retained evidence, not a sendable READY image. The * ownership X->N commit must happen between materialize and the explicit - * publication call; a restarted LMS scanning the intermediate state must not - * synthesize type 50 from it. */ + * publication call. A live owner must receive a distinct commit-only work + * token so conditional BufferContent contention can retry without recopying, + * aborting the A-record, or synthesizing type 50. */ UT_TEST(u29_pcm_x_materialized_bytes_require_explicit_ready_publication) { BufferTag tag = make_tag(123); @@ -1605,13 +1614,17 @@ UT_TEST(u29_pcm_x_materialized_bytes_require_explicit_ready_publication) UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key, &tag, &reserved), (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &binding, &hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &binding, UINT64_C(41), (uint8)PCM_STATE_X, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &binding, &cached), (int)GCS_BLOCK_PCM_X_IMAGE_NOT_READY); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), - (int)GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND); + (int)GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING); + UT_ASSERT_EQ((int)work.entry_kind, (int)GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED); + UT_ASSERT_EQ(memcmp(&work.binding, &binding, sizeof(binding)), 0); + UT_ASSERT_EQ(work.reservation_token, UINT64_C(41)); + UT_ASSERT_EQ((int)work.source_pcm_state, (int)PCM_STATE_X); UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_stage_count(), 1); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_publish_ready_exact(0, &key, &tag, &binding), @@ -1651,9 +1664,9 @@ UT_TEST(u30_pcm_x_owner_restart_audit_detects_and_retains_evidence) UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key, &tag, &reserved), (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &binding, &hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &binding, UINT64_C(41), (uint8)PCM_STATE_X, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); UT_ASSERT(cluster_gcs_block_dedup_pcm_x_restart_audit(0)); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &binding, &cached), (int)GCS_BLOCK_PCM_X_IMAGE_NOT_READY); @@ -1937,10 +1950,178 @@ UT_TEST(u34_pending_x_new_reader_exact_deny_precedes_cached_shortcut) | GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND)); } + +/* A contended tag A may remain at the commit-only boundary, but its exact + * retry must not prevent the same DATA worker from advancing independent tag + * B. This exercises the production HTAB scan/cursor, not a scheduler model. */ +UT_TEST(u35_pcm_x_commit_pending_rotates_to_independent_reserved_tag) +{ + BufferTag tag_a = make_tag(130); + BufferTag tag_b = make_tag(131); + GcsBlockDedupKey key_a; + GcsBlockDedupKey key_b; + GcsBlockPcmXImageBinding binding_a; + GcsBlockPcmXImageBinding reserved_a; + GcsBlockPcmXImageBinding reserved_b; + GcsBlockPcmXImageWork work; + GcsBlockReplyHeader hdr_a; + char page_a[GCS_BLOCK_DATA_SIZE]; + uint64 image_a; + uint64 image_b; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 30, &image_a)); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 31, &image_b)); + key_a = make_key(1, 3, image_a, 13); + key_b = make_key(1, 4, image_b, 13); + binding_a = make_pcm_x_binding(tag_a, 1, 5, gcs_reqid_requester(1, 2, 100), 13, image_a, 120); + hdr_a = make_pcm_x_reply_header(&key_a, &binding_a); + memset(page_a, 0x68, sizeof(page_a)); + prepare_pcm_x_page(page_a, &binding_a, &hdr_a); + reserved_a = binding_a; + reserved_a.identity.image.page_scn = 0; + reserved_a.identity.image.page_lsn = 0; + reserved_a.identity.image.page_checksum = 0; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key_a, &tag_a, &reserved_a), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key_a, &tag_a, &binding_a, + UINT64_C(51), (uint8)PCM_STATE_X, + &hdr_a, page_a), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + + reserved_b = make_pcm_x_binding(tag_b, 1, 6, gcs_reqid_requester(1, 3, 101), 13, image_b, 121); + reserved_b.identity.image.page_scn = 0; + reserved_b.identity.image.page_lsn = 0; + reserved_b.identity.image.page_checksum = 0; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key_b, &tag_b, &reserved_b), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING); + UT_ASSERT_EQ(memcmp(&work.key, &key_a, sizeof(key_a)), 0); + UT_ASSERT_EQ(work.reservation_token, UINT64_C(51)); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ(memcmp(&work.key, &key_b, sizeof(key_b)), 0); +} + + +/* A local terminal DRAIN is not an ACK boundary by itself. Exact byte and + * descriptor cleanup publishes a replayable tombstone; duplicate DRAIN stays + * provably complete until the matching RETIRE watermark removes that proof. */ +UT_TEST(u36_pcm_x_drain_cleanup_is_replayable_until_exact_retire) +{ + BufferTag tag = make_tag(132); + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageBinding reserved; + GcsBlockReplyHeader hdr; + GcsBlockDedupEntry cached; + char page[GCS_BLOCK_DATA_SIZE]; + uint64 image_id; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 32, &image_id)); + /* The initial live cluster episode is epoch zero; RETIRE must preserve the + * same exact-match semantics there as after the first reconfiguration. */ + key = make_key(1, 3, image_id, 0); + binding = make_pcm_x_binding(tag, 1, 5, gcs_reqid_requester(1, 2, 102), 0, image_id, 122); + hdr = make_pcm_x_reply_header(&key, &binding); + memset(page, 0x79, sizeof(page)); + prepare_pcm_x_page(page, &binding, &hdr); + UT_ASSERT_EQ((int)stage_pcm_x_ready(0, &key, &tag, &binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_drain_status_exact(0, &key, &tag, &binding), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_READY); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &binding, 2), + (int)GCS_BLOCK_PCM_X_IMAGE_RELEASED); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_drain_status_exact(0, &key, &tag, &binding), + (int)GCS_BLOCK_PCM_X_IMAGE_DUPLICATE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &binding, 2), + (int)GCS_BLOCK_PCM_X_IMAGE_DUPLICATE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &binding, &cached), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND); + reserved = binding; + reserved.identity.image.page_scn = 0; + reserved.identity.image.page_lsn = 0; + reserved.identity.image.page_checksum = 0; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key, &tag, &reserved), + (int)GCS_BLOCK_PCM_X_IMAGE_DUPLICATE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_rearm_exact(0, &key, &tag, &reserved), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_READY); + + cluster_gcs_block_dedup_sweep_expired((TimestampTz)INT64_MAX); + cluster_gcs_block_dedup_cleanup_on_backend_exit(1, 3); + cluster_gcs_block_dedup_cleanup_on_node_dead(1); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); + UT_ASSERT(cluster_gcs_block_dedup_pcm_x_retire_up_to(12, 2, 122, 29)); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); + UT_ASSERT(cluster_gcs_block_dedup_pcm_x_retire_up_to(0, 1, 122, 29)); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); + UT_ASSERT(cluster_gcs_block_dedup_pcm_x_retire_up_to(0, 2, 121, 29)); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); + UT_ASSERT(cluster_gcs_block_dedup_pcm_x_retire_up_to(0, 2, 122, 28)); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); + UT_ASSERT(cluster_gcs_block_dedup_pcm_x_retire_up_to(0, 2, 122, 29)); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 0); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_release_count(), 1); +} + + +/* A FlushBuffer ERROR occurs after materialization. Its catch handler must + * validate, but never delete or rewrite, the immutable A-record and revoke + * token needed by recovery. */ +UT_TEST(u37_pcm_x_finish_error_preserves_exact_materialized_evidence) +{ + BufferTag tag = make_tag(133); + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageBinding reserved; + GcsBlockPcmXImageWork work; + GcsBlockReplyHeader hdr; + char page[GCS_BLOCK_DATA_SIZE]; + uint64 image_id; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 33, &image_id)); + key = make_key(1, 3, image_id, 13); + binding = make_pcm_x_binding(tag, 1, 5, gcs_reqid_requester(1, 2, 103), 13, image_id, 123); + hdr = make_pcm_x_reply_header(&key, &binding); + memset(page, 0x6a, sizeof(page)); + prepare_pcm_x_page(page, &binding, &hdr); + reserved = binding; + reserved.identity.image.page_scn = 0; + reserved.identity.image.page_lsn = 0; + reserved.identity.image.page_checksum = 0; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key, &tag, &reserved), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &binding, UINT64_C(61), (uint8)PCM_STATE_X, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_preserve_finish_error_exact( + 0, &key, &tag, &binding, UINT64_C(62), (uint8)PCM_STATE_X), + (int)GCS_BLOCK_PCM_X_IMAGE_STALE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_preserve_finish_error_exact( + 0, &key, &tag, &binding, UINT64_C(61), (uint8)PCM_STATE_X), + (int)GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING); + UT_ASSERT_EQ(memcmp(&work.binding, &binding, sizeof(binding)), 0); + UT_ASSERT_EQ(work.reservation_token, UINT64_C(61)); + UT_ASSERT_EQ((int)work.source_pcm_state, (int)PCM_STATE_X); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &binding, -1), + (int)GCS_BLOCK_PCM_X_IMAGE_RELEASED); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 0); +} + int main(void) { - UT_PLAN(34); + UT_PLAN(37); UT_RUN(u1_per_worker_isolation); UT_RUN(u2_dedup_lifecycle_per_shard); UT_RUN(u3_counters_sum_across_shards); @@ -1975,6 +2156,9 @@ main(void) UT_RUN(u32_pcm_x_staged_marker_rolls_back_exactly); UT_RUN(u33_pending_x_arm_terminates_inflight_legacy_s_exactly); UT_RUN(u34_pending_x_new_reader_exact_deny_precedes_cached_shortcut); + UT_RUN(u35_pcm_x_commit_pending_rotates_to_independent_reserved_tag); + UT_RUN(u36_pcm_x_drain_cleanup_is_replayable_until_exact_retire); + UT_RUN(u37_pcm_x_finish_error_preserves_exact_materialized_evidence); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_gcs_block_shard.c b/src/test/cluster_unit/test_cluster_gcs_block_shard.c index 4277ac8248..ad9c3e3a06 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_shard.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_shard.c @@ -405,6 +405,7 @@ UT_TEST(test_pcm_x_route_truth_table) { PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, sizeof(PcmXBlockerSetHeaderPayload) }, { PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK, sizeof(PcmXPhasePayload) }, { PGRAC_IC_MSG_PCM_X_REVOKE, sizeof(PcmXRevokePayload) }, + { PGRAC_IC_MSG_PCM_X_REVOKE, sizeof(PcmXRevokePayloadV2) }, { PGRAC_IC_MSG_PCM_X_IMAGE_READY, sizeof(PcmXGrantPayload) }, { PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, sizeof(PcmXGrantPayload) }, { PGRAC_IC_MSG_PCM_X_INSTALL_READY, sizeof(PcmXInstallReadyPayload) }, @@ -444,10 +445,39 @@ UT_TEST(test_pcm_x_route_truth_table) -1); } +/* A status-3 PI durable note is an INVALIDATE_ACK frame and must use the + * exact tag shard. Exercise every live worker, including worker 1 (the + * t/400 failure arm), against the real payload router. */ +UT_TEST(test_pi_durable_note_routes_to_exact_tag_worker) +{ + bool seen[CLUSTER_LMS_MAX_WORKERS] = { false }; + int seen_count = 0; + int i; + + for (i = 0; i < 4096 && seen_count < CLUSTER_LMS_MAX_WORKERS; i++) { + BufferTag tag = make_tag(1663, 5, 16384 + (i % 4), MAIN_FORKNUM, (BlockNumber)i); + GcsBlockInvalidateAckPayload note = { 0 }; + int expected = cluster_lms_shard_for_tag(&tag, CLUSTER_LMS_MAX_WORKERS); + int routed; + + note.tag = tag; + note.ack_status = GCS_BLOCK_INVALIDATE_ACK_STATUS_PI_DURABLE_NOTE; + routed = cluster_gcs_block_payload_shard(PGRAC_IC_MSG_GCS_BLOCK_INVALIDATE_ACK, ¬e, + sizeof(note), CLUSTER_LMS_MAX_WORKERS); + UT_ASSERT_EQ(routed, expected); + if (!seen[routed]) { + seen[routed] = true; + seen_count++; + } + } + UT_ASSERT(seen[1]); + UT_ASSERT_EQ(seen_count, CLUSTER_LMS_MAX_WORKERS); +} + int main(void) { - UT_PLAN(8); + UT_PLAN(9); UT_RUN(test_route_matches_shard_for_tag); UT_RUN(test_route_ack_request_interleave_affinity); UT_RUN(test_route_registry_partition); @@ -456,6 +486,7 @@ main(void) UT_RUN(test_route_n1_degenerate_zero); UT_RUN(test_route_ignores_non_tag_fields); UT_RUN(test_pcm_x_route_truth_table); + UT_RUN(test_pi_durable_note_routes_to_exact_tag_worker); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_ic.c b/src/test/cluster_unit/test_cluster_ic.c index 1f02607bdc..5a88a437d5 100644 --- a/src/test/cluster_unit/test_cluster_ic.c +++ b/src/test/cluster_unit/test_cluster_ic.c @@ -597,10 +597,11 @@ UT_TEST(test_hello_wire_reference_bytes) * (0x10) + round-3 P0-1 xid wrap barrier (0x20) + round-4 P0-1 * authority flock (0x40) + ownership-gen ruling② invalidate BUSY * (0x80) + TT-lane undo-horizon idle sentinel (0x100) + PCM-X - * conversion (0x200) + A' rebase V2 INSTALL_READY (0x400) + * conversion (0x200) + A' rebase V2 INSTALL_READY (0x400) + PCM-X + * source-floor type49 V2 (0x800) * (smart-fusion is off in this fixture) */ UT_ASSERT_EQ(wire[36], 0xFE); - UT_ASSERT_EQ(wire[37], 0x07); + UT_ASSERT_EQ(wire[37], 0x0F); UT_ASSERT_EQ(wire[38], 0x00); UT_ASSERT_EQ(wire[39], 0x00); /* remaining _pad must be all zero: a CONTROL-plane HELLO with @@ -699,11 +700,12 @@ UT_TEST(test_hello_smart_fusion_capability_gate) | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 - | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1); + | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1 + | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1); /* Keep the aggregate word byte-exact as well as symbolically composed: * parallel protocol lanes have collided while preserving the same symbolic * expectation, so the literal catches accidental bit reuse. */ - UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), (uint32)0x000007FEU); + UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), (uint32)0x00000FFEU); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_2; @@ -717,7 +719,8 @@ UT_TEST(test_hello_smart_fusion_capability_gate) | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 - | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1); + | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1 + | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_3; @@ -732,7 +735,8 @@ UT_TEST(test_hello_smart_fusion_capability_gate) | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 - | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1); + | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1 + | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1); cluster_smart_fusion = false; cluster_interconnect_tier = CLUSTER_IC_TIER_STUB; diff --git a/src/test/cluster_unit/test_cluster_lms_outbound.c b/src/test/cluster_unit/test_cluster_lms_outbound.c index fd42a223ac..1146beb6e2 100644 --- a/src/test/cluster_unit/test_cluster_lms_outbound.c +++ b/src/test/cluster_unit/test_cluster_lms_outbound.c @@ -54,6 +54,7 @@ #include "cluster/cluster_ic_router.h" /* cluster_ic_send_envelope prototype */ #include "cluster/cluster_lms.h" #include "cluster/cluster_shmem.h" +#include "cluster/cluster_sf_dep.h" #include "cluster/cluster_write_fence.h" #include "miscadmin.h" #include "storage/lwlock.h" @@ -75,6 +76,41 @@ int cluster_node_id = 0; static PcmXRuntimeState ut_pcm_x_runtime_state = PCM_X_RUNTIME_ACTIVE; static bool ut_write_fence_enforcing = false; static bool ut_write_fence_allowed = true; +static uint32 ut_peer_capabilities[CLUSTER_MAX_NODES]; +static uint32 ut_peer_cap_generation[CLUSTER_MAX_NODES]; +static int ut_pcm_x_boundary_note_count = 0; +static uint8 ut_pcm_x_boundary_msg_types[8]; + +void +cluster_lms_note_pcm_x_image_ready_boundary( + uint8 msg_type, const char *boundary, int result, int runtime_state, bool fence_enforcing, + bool fence_allowed, uint32 dest_node_id, uint64 request_id, uint64 ticket_id, uint64 grant_generation, + uint64 image_id) +{ + if (ut_pcm_x_boundary_note_count < (int)lengthof(ut_pcm_x_boundary_msg_types)) + ut_pcm_x_boundary_msg_types[ut_pcm_x_boundary_note_count] = msg_type; + ut_pcm_x_boundary_note_count++; + (void)boundary; + (void)result; + (void)runtime_state; + (void)fence_enforcing; + (void)fence_allowed; + (void)dest_node_id; + (void)request_id; + (void)ticket_id; + (void)grant_generation; + (void)image_id; +} + +bool +cluster_sf_peer_capability_generation_matches(int32 peer_id, uint32 required_capabilities, + uint32 expected_generation) +{ + if (peer_id < 0 || peer_id >= CLUSTER_MAX_NODES || required_capabilities == 0) + return false; + return (ut_peer_capabilities[peer_id] & required_capabilities) == required_capabilities + && ut_peer_cap_generation[peer_id] == expected_generation; +} PcmXRuntimeSnapshot cluster_pcm_x_runtime_snapshot(void) @@ -178,6 +214,7 @@ cluster_lms_wakeup(int worker_id) /* Drain honesty counters (shmem-backed in production): count-only stubs. */ static int ut_not_admitted_count = 0; static int ut_requeue_drop_count = 0; +static int ut_cap_guard_drop_count = 0; void cluster_lms_obs_note_outbound_not_admitted(int worker_id) @@ -193,6 +230,13 @@ cluster_lms_obs_note_outbound_requeue_drop(int worker_id) ut_requeue_drop_count++; } +void +cluster_lms_obs_note_outbound_cap_guard_drop(int worker_id) +{ + (void)worker_id; + ut_cap_guard_drop_count++; +} + void cluster_gcs_block_note_send_outcome(GcsBlockSendFamily family, ClusterICSendResult rc) { @@ -315,6 +359,11 @@ ut_reset_log(void) ut_pcm_x_runtime_state = PCM_X_RUNTIME_ACTIVE; ut_write_fence_enforcing = false; ut_write_fence_allowed = true; + ut_cap_guard_drop_count = 0; + ut_pcm_x_boundary_note_count = 0; + memset(ut_pcm_x_boundary_msg_types, 0, sizeof(ut_pcm_x_boundary_msg_types)); + memset(ut_peer_capabilities, 0, sizeof(ut_peer_capabilities)); + memset(ut_peer_cap_generation, 0, sizeof(ut_peer_cap_generation)); memset(ut_sent_log, 0, sizeof(ut_sent_log)); } @@ -530,6 +579,27 @@ UT_TEST(test_pcm_x_grant_frame_waits_behind_write_fence) UT_ASSERT_EQ(cluster_lms_outbound_depth(7), 0); } + +/* Both sides of the first reliable transfer hop share PcmXGrantPayload. The + * injected transport trace must identify type 50 and type 51 independently, + * otherwise a successful master consume is indistinguishable from a lost + * PREPARE_GRANT admission. */ +UT_TEST(test_pcm_x_image_ready_and_prepare_transport_boundaries_are_observable) +{ + PcmXGrantPayload payload; + + ut_reset_log(); + memset(&payload, 0, sizeof(payload)); + UT_ASSERT(cluster_lms_outbound_enqueue( + 0, PGRAC_IC_MSG_PCM_X_IMAGE_READY, UT_PEER_X, &payload, sizeof(payload))); + UT_ASSERT(cluster_lms_outbound_enqueue( + 0, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, UT_PEER_Y, &payload, sizeof(payload))); + UT_ASSERT_EQ(cluster_lms_outbound_drain_send(0), 2); + UT_ASSERT_EQ(ut_pcm_x_boundary_note_count, 2); + UT_ASSERT_EQ((int)ut_pcm_x_boundary_msg_types[0], (int)PGRAC_IC_MSG_PCM_X_IMAGE_READY); + UT_ASSERT_EQ((int)ut_pcm_x_boundary_msg_types[1], (int)PGRAC_IC_MSG_PCM_X_PREPARE_GRANT); +} + /* * Shape-B denial replay is driven by LMON, which owns only plane 0. The * reply is an ABI-sized header + zero block, so LMON stages its compact @@ -588,10 +658,86 @@ UT_TEST(test_direct_zero_block_reply_uses_data_owner_direct_lane) UT_ASSERT_EQ(ut_direct_zero_reply_header.checksum, UINT32_C(0xA55A7E11)); } +/* A producer must receive false when the selected worker ring is full. The + * PI durable-note drain couples this real return contract with its structural + * false->break-before-seq-advance unit, so a full shard retains the source + * note for the next tick instead of losing it. */ +UT_TEST(test_full_worker_ring_refuses_without_overwrite) +{ + int accepted = 0; + int sent = 0; + + ut_reset_log(); + while (accepted < 1024 && ut_enqueue_marker(1, UT_PEER_X, 0xE2)) + accepted++; + UT_ASSERT(accepted > 0); + UT_ASSERT(accepted < 1024); + UT_ASSERT_EQ((int)cluster_lms_outbound_depth(1), accepted); + UT_ASSERT(!ut_enqueue_marker(1, UT_PEER_X, 0xE3)); + UT_ASSERT_EQ((int)cluster_lms_outbound_depth(1), accepted); + + ut_peer_rc[UT_PEER_X] = CLUSTER_IC_SEND_DONE; + while (cluster_lms_outbound_depth(1) > 0) + sent += cluster_lms_outbound_drain_send(1); + UT_ASSERT_EQ(sent, accepted); + UT_ASSERT_EQ(cluster_lms_outbound_depth(1), 0); +} + +/* A V2 wire frame is legal only on the exact HELLO-authenticated connection + * generation sampled by its producer. A reconnect or capability downgrade + * consumes the stale ring copy without transport admission; the reliable + * protocol leg remains armed and the periodic master drive reconstructs it. */ +UT_TEST(test_cap_bound_frame_drops_on_connection_generation_drift) +{ + const uint32 cap = PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1; + uint8 marker = 0x91; + + ut_reset_log(); + ut_peer_capabilities[UT_PEER_X] = cap; + ut_peer_cap_generation[UT_PEER_X] = 18; + UT_ASSERT(cluster_lms_outbound_enqueue_cap_bound( + 0, PGRAC_IC_MSG_PCM_X_REVOKE, UT_PEER_X, &marker, sizeof(marker), cap, 17)); + UT_ASSERT_EQ(cluster_lms_outbound_drain_send(0), 0); + UT_ASSERT_EQ(cluster_lms_outbound_depth(0), 0); + UT_ASSERT_EQ(ut_count_marker(marker), 0); + UT_ASSERT_EQ(ut_cap_guard_drop_count, 1); +} + +UT_TEST(test_cap_bound_frame_drops_on_capability_downgrade) +{ + const uint32 cap = PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1; + uint8 marker = 0x92; + + ut_reset_log(); + ut_peer_cap_generation[UT_PEER_X] = 21; + UT_ASSERT(cluster_lms_outbound_enqueue_cap_bound( + 0, PGRAC_IC_MSG_PCM_X_REVOKE, UT_PEER_X, &marker, sizeof(marker), cap, 21)); + UT_ASSERT_EQ(cluster_lms_outbound_drain_send(0), 0); + UT_ASSERT_EQ(cluster_lms_outbound_depth(0), 0); + UT_ASSERT_EQ(ut_count_marker(marker), 0); + UT_ASSERT_EQ(ut_cap_guard_drop_count, 1); +} + +UT_TEST(test_cap_bound_frame_sends_on_exact_connection_capability) +{ + const uint32 cap = PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1; + uint8 marker = 0x93; + + ut_reset_log(); + ut_peer_capabilities[UT_PEER_X] = cap; + ut_peer_cap_generation[UT_PEER_X] = 34; + UT_ASSERT(cluster_lms_outbound_enqueue_cap_bound( + 0, PGRAC_IC_MSG_PCM_X_REVOKE, UT_PEER_X, &marker, sizeof(marker), cap, 34)); + UT_ASSERT_EQ(cluster_lms_outbound_drain_send(0), 1); + UT_ASSERT_EQ(cluster_lms_outbound_depth(0), 0); + UT_ASSERT_EQ(ut_count_marker(marker), 1); + UT_ASSERT_EQ(ut_cap_guard_drop_count, 0); +} + int main(void) { - UT_PLAN(10); + UT_PLAN(15); UT_RUN(test_ring_shmem_init); UT_RUN(test_admitted_frame_is_never_resubmitted); @@ -601,8 +747,13 @@ main(void) UT_RUN(test_self_frame_dispatches_on_owning_worker); UT_RUN(test_pcm_x_grant_frame_waits_for_active_runtime); UT_RUN(test_pcm_x_grant_frame_waits_behind_write_fence); + UT_RUN(test_pcm_x_image_ready_and_prepare_transport_boundaries_are_observable); UT_RUN(test_zero_block_reply_is_expanded_by_data_owner); UT_RUN(test_direct_zero_block_reply_uses_data_owner_direct_lane); + UT_RUN(test_full_worker_ring_refuses_without_overwrite); + UT_RUN(test_cap_bound_frame_drops_on_connection_generation_drift); + UT_RUN(test_cap_bound_frame_drops_on_capability_downgrade); + UT_RUN(test_cap_bound_frame_sends_on_exact_connection_capability); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/cluster_unit/test_cluster_pcm_direct_init.c b/src/test/cluster_unit/test_cluster_pcm_direct_init.c index d52a0ff80b..a2e26eb5c1 100644 --- a/src/test/cluster_unit/test_cluster_pcm_direct_init.c +++ b/src/test/cluster_unit/test_cluster_pcm_direct_init.c @@ -26,6 +26,9 @@ UT_DEFINE_GLOBALS(); #ifndef HEAPAM_SOURCE_PATH #error "HEAPAM_SOURCE_PATH must identify production heapam.c" #endif +#ifndef HIO_SOURCE_PATH +#error "HIO_SOURCE_PATH must identify production hio.c" +#endif void ExceptionalCondition(const char *conditionName pg_attribute_unused(), @@ -441,11 +444,13 @@ UT_TEST(test_precrit_vm_barrier_refusal_unwinds_to_caller) static const char *const pretoast_order[] = { "PGRAC: vm barrier unwind (update pre-toast)", "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", "cluster_heap_vm_barrier_warm", - "LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE)", "goto l2;" }; + "ReleaseBuffer(vmbuffer)", "vmbuffer = InvalidBuffer", + "cluster_heap_lock_with_vm_repin", "goto l2;" }; static const char *const requalify_order[] = { "PGRAC: vm barrier unwind (update requalify)", "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", "cluster_heap_vm_barrier_warm", - "LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE)", "goto l2;" }; + "ReleaseBuffer(vmbuffer)", "vmbuffer = InvalidBuffer", + "cluster_heap_lock_with_vm_repin", "goto l2;" }; static const char *const reacquire_order[] = { "PGRAC: vm barrier unwind (update reacquire)", "LockBuffer(newbuf, BUFFER_LOCK_UNLOCK)", @@ -456,7 +461,8 @@ UT_TEST(test_precrit_vm_barrier_refusal_unwinds_to_caller) static const char *const delete_order[] = { "PGRAC: vm barrier unwind (delete requalify)", "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", "cluster_heap_vm_barrier_warm", - "LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE)", "goto l1;" }; + "ReleaseBuffer(vmbuffer)", "vmbuffer = InvalidBuffer", + "cluster_heap_lock_with_vm_repin", "goto l1;" }; /* The warm helper itself may only run with no content lock held: * it must take and drop the map-page lock, nothing else. */ @@ -473,10 +479,151 @@ UT_TEST(test_precrit_vm_barrier_refusal_unwinds_to_caller) } } +/* P0-20: an UPDATE must not keep its visibility-map pin while it can block + * acquiring heap X. Another writer can already be transferring that same + * VM page, and VM/FSM deliberately cannot become a retained PI while pinned. + * Pre-read the VM page, release the pin before the heap PCM wait, then repin + * only the exact recent descriptor without I/O after heap X is held. */ +UT_TEST(test_heap_update_drops_vm_pin_across_heap_pcm_wait) +{ + char *heapam = read_source(HEAPAM_SOURCE_PATH); + char *visibilitymap = read_source(VM_SOURCE_PATH); + + UT_ASSERT(heapam != NULL); + UT_ASSERT(visibilitymap != NULL); + if (heapam != NULL) { + const char *helper = strstr(heapam, "cluster_heap_lock_with_vm_repin("); + const char *helper_end = helper != NULL ? strstr(helper, "\n}\n") : NULL; + const char *update = strstr(heapam, "\nheap_update("); + const char *update_lock + = update != NULL ? strstr(update, "cluster_heap_lock_with_vm_repin(") : NULL; + const char *repin_branch = update != NULL + ? strstr(update, "if (vmbuffer == InvalidBuffer && PageIsAllVisible(page))") + : NULL; + const char *repin_branch_end + = repin_branch != NULL ? strstr(repin_branch, "goto l2;") : NULL; + const char *repin_branch_helper = repin_branch != NULL + ? strstr(repin_branch, "cluster_heap_lock_with_vm_repin(") : NULL; + const char *success_tail = update != NULL + ? strstr(update, "recptr = log_heap_update(") + : NULL; + const char *vm_unlock = success_tail != NULL + ? strstr(success_tail, + "if (vm_locked)\n\t\tLockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);") + : NULL; + const char *vm_release = vm_unlock != NULL + ? strstr(vm_unlock, "ReleaseBuffer(vmbuffer);") + : NULL; + const char *heap_unlock = vm_unlock != NULL + ? strstr(vm_unlock, "LockBuffer(buffer, BUFFER_LOCK_UNLOCK);") + : NULL; + static const char *const helper_order[] + = { "visibilitymap_pin(relation, heap_block, vmbuffer)", + "ReleaseBuffer(*vmbuffer)", + "*vmbuffer = InvalidBuffer", + "LockBuffer(heap_buffer, BUFFER_LOCK_EXCLUSIVE)", + "visibilitymap_pin_recent(relation, heap_block, recent_vm, vmbuffer)", + "LockBuffer(heap_buffer, BUFFER_LOCK_UNLOCK)" }; + + UT_ASSERT(helper != NULL); + UT_ASSERT(helper_end != NULL); + UT_ASSERT(update != NULL); + UT_ASSERT(update_lock != NULL); + UT_ASSERT(repin_branch != NULL); + UT_ASSERT(repin_branch_end != NULL); + UT_ASSERT(repin_branch_helper != NULL); + UT_ASSERT(repin_branch_helper < repin_branch_end); + UT_ASSERT(vm_unlock != NULL); + UT_ASSERT(vm_release != NULL); + UT_ASSERT(heap_unlock != NULL); + UT_ASSERT(vm_release < heap_unlock); + if (helper != NULL && helper_end != NULL) + assert_ordered(helper, helper_order, lengthof(helper_order)); + free(heapam); + } + if (visibilitymap != NULL) { + const char *repin = strstr(visibilitymap, "\nvisibilitymap_pin_recent("); + const char *repin_end = repin != NULL ? strstr(repin, "\n}\n") : NULL; + static const char *const repin_order[] + = { "HEAPBLK_TO_MAPBLOCK(heapBlk)", "ReadRecentBuffer(", + "VISIBILITYMAP_FORKNUM", "recent_buffer" }; + + UT_ASSERT(repin != NULL); + UT_ASSERT(repin_end != NULL); + if (repin != NULL && repin_end != NULL) + assert_ordered(repin, repin_order, lengthof(repin_order)); + free(visibilitymap); + } +} + +/* P0-20: RelationGetBufferForTuple acquires two heap content locks for a + * cross-page UPDATE. If the second queue acquire sees the first page under + * a frozen revoke barrier, it must release the first lock, resolve the second + * conversion with no content lock held, and retry the ordered pair. */ +UT_TEST(test_cross_page_heap_pair_barrier_refusal_retries_unlocked) +{ + char *hio = read_source(HIO_SOURCE_PATH); + const char *helper; + const char *helper_end; + const char *pins; + const char *pins_end; + const char *pins_call; + const char *relation; + const char *relation_end; + const char *lower_branch; + const char *lower_call; + const char *upper_branch; + const char *upper_call; + const char *extension; + const char *extension_call; + static const char *const helper_order[] + = { "cluster_hio_lock_buffer_pair(Buffer first, Buffer second)", + "LockBuffer(first, BUFFER_LOCK_EXCLUSIVE)", + "ClusterLockBufferExclusiveBarrierAware(second)", + "LockBuffer(first, BUFFER_LOCK_UNLOCK)", + "LockBuffer(second, BUFFER_LOCK_EXCLUSIVE)", + "LockBuffer(second, BUFFER_LOCK_UNLOCK)" }; + + UT_ASSERT(hio != NULL); + if (hio == NULL) + return; + helper = strstr(hio, "cluster_hio_lock_buffer_pair(Buffer first, Buffer second)"); + helper_end = helper != NULL ? strstr(helper, "\n}\n") : NULL; + pins = strstr(hio, "GetVisibilityMapPins(Relation relation"); + pins_end = pins != NULL ? strstr(pins, "\n}\n") : NULL; + pins_call = pins != NULL ? strstr(pins, "cluster_hio_lock_buffer_pair(") : NULL; + relation = strstr(hio, "\nRelationGetBufferForTuple("); + relation_end = relation != NULL ? strstr(relation, "\n}\n") : NULL; + lower_branch = relation != NULL ? strstr(relation, "else if (otherBlock < targetBlock)") : NULL; + lower_call = lower_branch != NULL ? strstr(lower_branch, "cluster_hio_lock_buffer_pair(") : NULL; + upper_branch = lower_branch != NULL ? strstr(lower_branch, "\n\t\telse\n\t\t{") : NULL; + upper_call = upper_branch != NULL ? strstr(upper_branch, "cluster_hio_lock_buffer_pair(") : NULL; + extension = relation != NULL ? strstr(relation, "Reacquire locks if necessary") : NULL; + extension_call = extension != NULL ? strstr(extension, "cluster_hio_lock_buffer_pair(") : NULL; + + UT_ASSERT(helper != NULL); + UT_ASSERT(helper_end != NULL); + UT_ASSERT(pins != NULL); + UT_ASSERT(pins_end != NULL); + UT_ASSERT(relation != NULL); + UT_ASSERT(relation_end != NULL); + if (helper != NULL && helper_end != NULL) + assert_ordered(helper, helper_order, lengthof(helper_order)); + UT_ASSERT(pins != NULL && pins_end != NULL && pins_call != NULL + && pins_call < pins_end); + UT_ASSERT(lower_branch != NULL && lower_call != NULL && upper_branch != NULL + && lower_call < upper_branch); + UT_ASSERT(upper_branch != NULL && upper_call != NULL && relation_end != NULL + && upper_call < relation_end); + UT_ASSERT(extension != NULL && extension_call != NULL && relation_end != NULL + && extension_call < relation_end); + free(hio); +} + int main(void) { - UT_PLAN(18); + UT_PLAN(20); UT_RUN(test_valid_read_miss_proof); UT_RUN(test_valid_extend_proof); UT_RUN(test_valid_vm_and_fsm_proofs); @@ -495,6 +642,8 @@ main(void) UT_RUN(test_direct_init_one_shot_image_cannot_return_without_x); UT_RUN(test_wire_throw_exact_aborts_reservation_before_rethrow); UT_RUN(test_precrit_vm_barrier_refusal_unwinds_to_caller); + UT_RUN(test_heap_update_drops_vm_pin_across_heap_pcm_wait); + UT_RUN(test_cross_page_heap_pair_barrier_refusal_retries_unlocked); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_pcm_lock.c b/src/test/cluster_unit/test_cluster_pcm_lock.c index 367c1aaebf..590858ffe6 100644 --- a/src/test/cluster_unit/test_cluster_pcm_lock.c +++ b/src/test/cluster_unit/test_cluster_pcm_lock.c @@ -123,11 +123,28 @@ static struct { int holder_node; bool armed; } fake_cv_wake_release = { { 0 }, 0, false }; +static struct { + BufferTag tag; + int requester_node; + uint64 ticket_id; + bool armed; +} fake_cv_wake_pending_x_clear = { { 0 }, 0, 0, false }; static sigjmp_buf ut_ereport_jump; static bool ut_ereport_jump_armed = false; static int ut_ereport_fired_count = 0; static bool fake_local_x_upgrade_result = false; +static bool fake_acquire_entry_handoff_armed = false; +static BufferTag fake_acquire_entry_handoff_tag; +static int fake_acquire_entry_handoff_source = -1; +static int fake_acquire_entry_handoff_target = -1; +static PcmLockTransition fake_acquire_entry_handoff_release = PCM_TRANS_X_TO_N_RELEASE; +static int fake_local_read_image_count = 0; +static int fake_local_read_image_holder = -1; +static PcmAuthoritySnapshot fake_local_read_image_expected; +static int fake_local_x_transfer_count = 0; +static int fake_local_x_transfer_holder = -1; +static PcmAuthoritySnapshot fake_local_x_transfer_expected; void ExceptionalCondition(const char *conditionName pg_attribute_unused(), @@ -191,6 +208,12 @@ reset_fake_pcm_runtime(int max_entries) fake_cv_broadcast_count = 0; fake_cv_sleep_wait_event = 0; fake_cv_wake_release.armed = false; + fake_cv_wake_pending_x_clear.armed = false; + fake_acquire_entry_handoff_armed = false; + fake_acquire_entry_handoff_release = PCM_TRANS_X_TO_N_RELEASE; + fake_local_read_image_count = 0; + fake_local_read_image_holder = -1; + memset(&fake_local_read_image_expected, 0, sizeof(fake_local_read_image_expected)); fake_local_x_upgrade_result = false; cluster_node_id = 0; NBuffers = max_entries; @@ -456,6 +479,12 @@ ConditionVariableSleep(ConditionVariable *cv pg_attribute_unused(), uint32 wait_ cluster_pcm_lock_release(fake_cv_wake_release.tag); cluster_node_id = save_node; } + if (fake_cv_wake_pending_x_clear.armed) { + fake_cv_wake_pending_x_clear.armed = false; + UT_ASSERT(cluster_pcm_lock_clear_queue_pending_x_exact( + fake_cv_wake_pending_x_clear.tag, fake_cv_wake_pending_x_clear.requester_node, + fake_cv_wake_pending_x_clear.ticket_id)); + } } bool @@ -500,8 +529,22 @@ cluster_shmem_register_region(const ClusterShmemRegion *region pg_attribute_unus {} void -cluster_injection_run(const char *name pg_attribute_unused()) -{} +cluster_injection_run(const char *name) +{ + if (fake_acquire_entry_handoff_armed + && strcmp(name, "cluster-pcm-acquire-entry") == 0) { + fake_acquire_entry_handoff_armed = false; + cluster_injection_armed_count = 0; + UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition( + fake_acquire_entry_handoff_tag, fake_acquire_entry_handoff_release, + fake_acquire_entry_handoff_source), + 1); + UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition( + fake_acquire_entry_handoff_tag, PCM_TRANS_N_TO_X, + fake_acquire_entry_handoff_target), + 1); + } +} /* PGRAC spec-2.32 D5 stubs: cluster_pcm_lock.c now calls cluster_gcs * helpers from each mutation entry point (master lookup branch). Test @@ -535,23 +578,35 @@ cluster_gcs_send_block_request_and_wait(struct BufferDesc *buf pg_attribute_unus } /* spec-5.2 D2 sub-case B stub: local-master read-image forward. The pcm_lock - * fixtures force master == self with no remote X holder, so this is never - * reached; abort if it is. */ + * fixture records the selected holder so optimistic-precheck handoff tests can + * prove the buffer-aware S path routes to the existing one-shot image helper + * instead of the tag-only fail-closed terminal. */ bool cluster_gcs_local_master_read_image_and_wait(struct BufferDesc *buf pg_attribute_unused(), - int32 holder_node pg_attribute_unused()) + const PcmAuthoritySnapshot *expected, + bool *out_retry_denied) { - abort(); + *out_retry_denied = false; + fake_local_read_image_count++; + fake_local_read_image_expected = *expected; + fake_local_read_image_holder = expected->x_holder_node; + return false; } -/* spec-5.2 D11 stub: same rationale — the pcm_lock unit fixtures never drive a - * local-master X-transfer, so abort if reached. */ +/* spec-5.2 D11 stub: record the authoritative holder selected by the + * buffer-aware local-master path. The real data-plane behavior is covered by + * the GCS block tests; this fixture only proves PCM routing and authority. */ bool cluster_gcs_local_master_x_transfer_and_wait(struct BufferDesc *buf pg_attribute_unused(), - int32 holder_node pg_attribute_unused(), - bool clean_eligible pg_attribute_unused()) -{ - abort(); + const PcmAuthoritySnapshot *expected, + bool clean_eligible pg_attribute_unused(), + bool *out_retry_denied) +{ + *out_retry_denied = false; + fake_local_x_transfer_count++; + fake_local_x_transfer_expected = *expected; + fake_local_x_transfer_holder = expected->x_holder_node; + return true; } /* spec-2.35 D3 stub: HC110 master_holder lifecycle counter bump invoked @@ -1459,6 +1514,246 @@ UT_TEST(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades) cluster_gcs_block_local_cache = save; } +/* + * P0-26 sibling race: the local-master buffer-aware X path observes shared S + * with no local S bit, then bootstraps a local S declaration before upgrade. + * A queue handoff may replace that S authority with remote X in between. The + * nested acquire must preserve the BufferDesc-aware exact transfer route, + * never escape through the tag-only legacy terminal. + */ +UT_TEST(test_pcm_acquire_buffer_s_bootstrap_revalidates_remote_x) +{ + BufferTag tag = make_tag(970); + BufferDesc buf; + PcmAuthoritySnapshot after; + bool retry_denied = false; + bool acquired = false; + bool escaped_error = false; + + reset_fake_pcm_runtime(4); + cluster_gcs_block_local_cache = true; + cluster_node_id = 1; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + + memset(&buf, 0, sizeof(buf)); + buf.tag = tag; + buf.pcm_state = (uint8)PCM_STATE_N; + cluster_node_id = 0; + fake_local_x_transfer_count = 0; + fake_local_x_transfer_holder = -1; + fake_acquire_entry_handoff_tag = tag; + fake_acquire_entry_handoff_source = 1; + fake_acquire_entry_handoff_target = 2; + fake_acquire_entry_handoff_release = PCM_TRANS_S_TO_N_RELEASE; + cluster_injection_armed_count = 1; + fake_acquire_entry_handoff_armed = true; + + if (sigsetjmp(ut_ereport_jump, 1) == 0) { + ut_ereport_jump_armed = true; + acquired = cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_X, &retry_denied); + ut_ereport_jump_armed = false; + } else { + ut_ereport_jump_armed = false; + escaped_error = true; + } + + UT_ASSERT(!escaped_error); + UT_ASSERT(acquired); + UT_ASSERT(!retry_denied); + UT_ASSERT_EQ(fake_local_x_transfer_count, 1); + UT_ASSERT_EQ(fake_local_x_transfer_holder, 2); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_X); + UT_ASSERT_EQ(after.x_holder_node, 2); + UT_ASSERT_EQ(memcmp(&fake_local_x_transfer_expected, &after, sizeof(after)), 0); +} + +/* P0-26 third entry: the ordinary buffer-aware S path also has an optimistic + * remote-X precheck. A local-X -> remote-X queue handoff between that check + * and the entry-lock acquire must retain the BufferDesc-aware read-image + * route; it must not fall through the tag-only legacy terminal. */ +UT_TEST(test_pcm_acquire_buffer_s_revalidates_remote_x_after_precheck) +{ + BufferTag tag = make_tag(971); + BufferDesc buf; + PcmAuthoritySnapshot after; + bool retry_denied = false; + bool acquired = true; + bool escaped_error = false; + + reset_fake_pcm_runtime(4); + cluster_gcs_block_local_cache = true; + cluster_node_id = 0; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_X); + + memset(&buf, 0, sizeof(buf)); + buf.tag = tag; + fake_acquire_entry_handoff_tag = tag; + fake_acquire_entry_handoff_source = 0; + fake_acquire_entry_handoff_target = 1; + cluster_injection_armed_count = 1; + fake_acquire_entry_handoff_armed = true; + + if (sigsetjmp(ut_ereport_jump, 1) == 0) { + ut_ereport_jump_armed = true; + acquired = cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_S, &retry_denied); + ut_ereport_jump_armed = false; + } else { + ut_ereport_jump_armed = false; + escaped_error = true; + } + + UT_ASSERT(!escaped_error); + UT_ASSERT(!acquired); /* one-shot READ_IMAGE is intentionally non-durable */ + UT_ASSERT(!retry_denied); + UT_ASSERT_EQ(fake_local_read_image_count, 1); + UT_ASSERT_EQ(fake_local_read_image_holder, 1); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_X); + UT_ASSERT_EQ(after.x_holder_node, 1); + UT_ASSERT_EQ(memcmp(&fake_local_read_image_expected, &after, sizeof(after)), 0); +} + +/* P0-26: the buffer-aware local-master X path used to inspect state/holder, + * then call the tag-only acquire without carrying an authoritative token. + * Commit a queue-style local-X -> remote-X handoff at the existing acquire + * entry injection point, exactly after the optimistic precheck and before the + * tag-only entry lock. The buffer-aware caller must redirect to the existing + * safe X-transfer path; the old code escapes through the legacy + * "cross-node block write transfer not supported" ERROR instead. + */ +UT_TEST(test_pcm_acquire_buffer_revalidates_remote_x_after_precheck) +{ + BufferTag tag = make_tag(97); + BufferDesc buf; + PcmAuthoritySnapshot after; + bool retry_denied = false; + bool acquired = false; + bool escaped_error = false; + + reset_fake_pcm_runtime(4); + cluster_gcs_block_local_cache = true; + cluster_node_id = 0; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_X); + + memset(&buf, 0, sizeof(buf)); + buf.tag = tag; + fake_local_x_transfer_count = 0; + fake_local_x_transfer_holder = -1; + fake_acquire_entry_handoff_tag = tag; + fake_acquire_entry_handoff_source = 0; + fake_acquire_entry_handoff_target = 1; + cluster_injection_armed_count = 1; + fake_acquire_entry_handoff_armed = true; + + if (sigsetjmp(ut_ereport_jump, 1) == 0) { + ut_ereport_jump_armed = true; + acquired = cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_X, &retry_denied); + ut_ereport_jump_armed = false; + } else { + ut_ereport_jump_armed = false; + escaped_error = true; + } + + UT_ASSERT(!escaped_error); + UT_ASSERT(acquired); + UT_ASSERT(!retry_denied); + UT_ASSERT_EQ(fake_local_x_transfer_count, 1); + UT_ASSERT_EQ(fake_local_x_transfer_holder, 1); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_X); + UT_ASSERT_EQ(after.x_holder_node, 1); + UT_ASSERT_EQ((long)after.s_holders_bitmap, 0L); + UT_ASSERT_EQ(memcmp(&fake_local_x_transfer_expected, &after, sizeof(after)), 0); +} + +UT_TEST(test_pcm_acquire_buffer_routes_unchanged_remote_x_with_exact_authority) +{ + BufferTag tag = make_tag(98); + BufferDesc buf; + PcmAuthoritySnapshot before; + PcmAuthoritySnapshot after; + bool retry_denied = false; + + reset_fake_pcm_runtime(4); + cluster_gcs_block_local_cache = true; + cluster_node_id = 0; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_X); + UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_X_TO_N_RELEASE, 0), 1); + UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_X, 1), 1); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); + + memset(&buf, 0, sizeof(buf)); + buf.tag = tag; + fake_local_x_transfer_count = 0; + fake_local_x_transfer_holder = -1; + + UT_ASSERT(cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_X, &retry_denied)); + UT_ASSERT(!retry_denied); + UT_ASSERT_EQ(fake_local_x_transfer_count, 1); + UT_ASSERT_EQ(fake_local_x_transfer_holder, 1); + UT_ASSERT_EQ(memcmp(&fake_local_x_transfer_expected, &before, sizeof(before)), 0); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ(memcmp(&after, &before, sizeof(before)), 0); +} + +UT_TEST(test_pcm_x_transfer_commit_is_exact_and_late_reply_safe) +{ + BufferTag tag = make_tag(99); + PcmAuthoritySnapshot expected; + PcmAuthoritySnapshot stale; + PcmAuthoritySnapshot after; + PcmAuthoritySnapshot committed; + + reset_fake_pcm_runtime(4); + cluster_node_id = 0; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_X); + UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_X_TO_N_RELEASE, 0), 1); + UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_X, 1), 1); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &expected)); + UT_ASSERT(cluster_pcm_lock_authority_matches(tag, &expected)); + + stale = expected; + stale.transition_count++; + UT_ASSERT_EQ(cluster_pcm_lock_master_take_x_after_transfer(tag, &stale, (XLogRecPtr)0x1234, + InvalidScn, 1, 44, 900, 12), + PCM_X_TRANSFER_COMMIT_STALE); + stale = expected; + stale.master_holder.request_id++; + UT_ASSERT_EQ(cluster_pcm_lock_master_take_x_after_transfer(tag, &stale, (XLogRecPtr)0x1234, + InvalidScn, 1, 44, 900, 12), + PCM_X_TRANSFER_COMMIT_STALE); + stale = expected; + stale.master_holder.cluster_epoch++; + UT_ASSERT_EQ(cluster_pcm_lock_master_take_x_after_transfer(tag, &stale, (XLogRecPtr)0x1234, + InvalidScn, 1, 44, 900, 12), + PCM_X_TRANSFER_COMMIT_STALE); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ(memcmp(&after, &expected, sizeof(expected)), 0); + + UT_ASSERT_EQ(cluster_pcm_lock_master_take_x_after_transfer(tag, &expected, (XLogRecPtr)0x1234, + InvalidScn, 1, 44, 900, 12), + PCM_X_TRANSFER_COMMIT_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &committed)); + UT_ASSERT_EQ((int)committed.state, (int)PCM_STATE_X); + UT_ASSERT_EQ(committed.x_holder_node, 0); + UT_ASSERT_EQ((long)committed.s_holders_bitmap, 0L); + UT_ASSERT_EQ(committed.pending_x_requester_node, -1); + UT_ASSERT_EQ((long)committed.master_holder.node_id, 0L); + UT_ASSERT_EQ((long)committed.master_holder.procno, 44L); + UT_ASSERT_EQ((uint64)committed.master_holder.cluster_epoch, (uint64)12); + UT_ASSERT_EQ((uint64)committed.master_holder.request_id, (uint64)900); + UT_ASSERT_EQ((uint64)committed.transition_count, (uint64)expected.transition_count + 1); + + /* A duplicate/late reply carries the displaced remote-X token. */ + UT_ASSERT_EQ(cluster_pcm_lock_master_take_x_after_transfer(tag, &expected, (XLogRecPtr)0x1234, + InvalidScn, 1, 44, 900, 12), + PCM_X_TRANSFER_COMMIT_STALE); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ(memcmp(&after, &committed, sizeof(committed)), 0); +} + + UT_TEST(test_pcm_dead_node_cleanup_drops_holder_records) { BufferTag stag = make_tag(94); @@ -1541,6 +1836,33 @@ make_pcm_x_grd_handoff_token(BufferTag tag, const PcmAuthoritySnapshot *authorit return token; } +static char * +read_gcs_block_source(void) +{ + FILE *file; + char *source; + long length; + + file = fopen(GCS_BLOCK_SOURCE_PATH, "rb"); + UT_ASSERT_NOT_NULL(file); + if (file == NULL) + return NULL; + UT_ASSERT_EQ(fseek(file, 0, SEEK_END), 0); + length = ftell(file); + UT_ASSERT(length > 0); + UT_ASSERT_EQ(fseek(file, 0, SEEK_SET), 0); + source = malloc((size_t)length + 1); + UT_ASSERT_NOT_NULL(source); + if (source == NULL) { + fclose(file); + return NULL; + } + UT_ASSERT_EQ(fread(source, 1, (size_t)length, file), (size_t)length); + source[length] = '\0'; + fclose(file); + return source; +} + UT_TEST(test_pcm_authority_snapshot_is_one_entry_lock_view) { BufferTag tag = make_tag(97); @@ -1632,6 +1954,84 @@ UT_TEST(test_pcm_queue_pending_x_reservation_never_overwrites_another_node) PCM_PENDING_X_RESERVE_NO_CAPACITY); } +/* P0-25: pending-X is an admission barrier, not only an advisory wire gate. + * The final decision and the S-holder bitmap mutation share entry_lock, so an + * N->S request that raced past an earlier handler preflight cannot publish a + * new holder after the queue writer claimed the tag. An already-recorded S + * holder may re-enter without changing the authority bytes. */ +UT_TEST(test_pcm_pending_x_blocks_new_remote_s_holder_atomically) +{ + BufferTag tag = make_tag(110); + PcmAuthoritySnapshot before; + PcmAuthoritySnapshot after; + + reset_fake_pcm_runtime(4); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9010), PCM_PENDING_X_RESERVE_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); + UT_ASSERT_EQ((int)before.state, (int)PCM_STATE_N); + UT_ASSERT_EQ(cluster_pcm_lock_apply_gcs_transition_result(tag, PCM_TRANS_N_TO_S, 1), + PCM_GCS_TRANSITION_PENDING_X); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_N); + UT_ASSERT_EQ(after.s_holders_bitmap, (uint32)0); + UT_ASSERT_EQ(after.transition_count, before.transition_count); + + UT_ASSERT(cluster_pcm_lock_clear_queue_pending_x_exact(tag, 3, 9010)); + UT_ASSERT(cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_S, 1)); + UT_ASSERT_EQ(cluster_pcm_lock_query_s_holders_bitmap(tag), (uint32)(1u << 1)); + + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9011), PCM_PENDING_X_RESERVE_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); + UT_ASSERT(cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_S, 1)); + UT_ASSERT_EQ(cluster_pcm_lock_apply_gcs_transition_result(tag, PCM_TRANS_N_TO_S, 2), + PCM_GCS_TRANSITION_PENDING_X); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ(after.s_holders_bitmap, before.s_holders_bitmap); + UT_ASSERT_EQ(after.transition_count, before.transition_count); + + UT_ASSERT(cluster_pcm_lock_clear_queue_pending_x_exact(tag, 3, 9011)); + UT_ASSERT(cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_S, 2)); + UT_ASSERT_EQ(cluster_pcm_lock_query_s_holders_bitmap(tag), (uint32)((1u << 1) | (1u << 2))); +} + +/* P0-25 local-master mirror: an existing local S holder re-enters immediately, + * while a different local node waits until the queue cookie is cleared. The + * CV callback deterministically performs that clear in this single-threaded + * harness, proving both no pre-clear bitmap publication and post-clear liveness. */ +UT_TEST(test_pcm_pending_x_blocks_new_local_s_holder_until_clear) +{ + BufferTag tag = make_tag(111); + + reset_fake_pcm_runtime(4); + cluster_node_id = 0; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9012), PCM_PENDING_X_RESERVE_OK); + + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(fake_cv_sleep_count, 0); + UT_ASSERT_EQ(cluster_pcm_lock_query_s_holders_bitmap(tag), (uint32)(1u << 0)); + + fake_cv_wake_pending_x_clear.tag = tag; + fake_cv_wake_pending_x_clear.requester_node = 3; + fake_cv_wake_pending_x_clear.ticket_id = 9012; + fake_cv_wake_pending_x_clear.armed = true; + cluster_node_id = 1; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(fake_cv_sleep_count, 1); + UT_ASSERT_EQ((int)fake_cv_sleep_wait_event, (int)WAIT_EVENT_PCM_COMPATIBLE_STATE_WAIT); + UT_ASSERT_EQ(cluster_pcm_lock_query_pending_x_requester(tag), -1); + UT_ASSERT_EQ(cluster_pcm_lock_query_s_holders_bitmap(tag), (uint32)((1u << 0) | (1u << 1))); + + /* A reader woken by FINAL may observe the newly granted X and sleep again. + * The later holder X->S downgrade must wake it when S becomes compatible. */ + tag = make_tag(112); + reset_fake_pcm_runtime(4); + cluster_node_id = 0; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_X); + UT_ASSERT(cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_X_TO_S_DOWNGRADE, 0)); + UT_ASSERT_EQ(fake_cv_broadcast_count, 1); +} + UT_TEST(test_pcm_queue_handoff_x_exact_rejects_authority_drift) { BufferTag tag = make_tag(98); @@ -1646,12 +2046,13 @@ UT_TEST(test_pcm_queue_handoff_x_exact_rejects_authority_drift) UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); token = make_pcm_x_grd_handoff_token(tag, &before, 1, 3, 41, 9001); - cluster_node_id = 2; - cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + /* A holder release remains legal under pending-X and still invalidates the + * optimistic authority token; a new S-holder admission is now prohibited. */ + cluster_pcm_lock_release(tag); UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&token), PCM_X_GRD_HANDOFF_STALE); UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); - UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_S); - UT_ASSERT_EQ(after.s_holders_bitmap, (uint32)((1u << 1) | (1u << 2))); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_N); + UT_ASSERT_EQ(after.s_holders_bitmap, (uint32)0); } UT_TEST(test_pcm_queue_handoff_x_exact_rejects_residual_s_holder) @@ -1851,6 +2252,152 @@ UT_TEST(test_pcm_queue_handoff_x_exact_uses_scn_not_cross_stream_lsn) UT_ASSERT_EQ(after.pending_x_requester_node, 3); } +/* + * P0-20: reproduce the real ordering that failed t/400 at 14:23 without a + * scheduler race. A and B first hold S. A is the pre-acked transfer source; + * the production-equivalent effects of B's exact slotless INVALIDATE_ACK then + * remove B and advance the monotone watermark to W. A subsequently presents + * an older materialized image S. + * + * The final GRD handoff gate remains the non-negotiable last defence, but the + * protocol must classify this as a recoverable stale source before type 50 is + * accepted and PREPARE_GRANT is emitted. This test deliberately combines the + * real GRD authority/watermark operations with a bounded source contract over + * the production ACK and IMAGE_READY handlers. It does not call the static + * ACK handler directly; separate gate/provenance tests did not expose this + * inter-handler gap. + */ +UT_TEST(test_pcm_x_slotless_ack_floor_fences_stale_source_before_prepare) +{ + BufferTag tag = make_tag(111); + PcmAuthoritySnapshot authority; + PcmAuthoritySnapshot after; + PcmXGrdHandoffToken stale_handoff; + ClusterPcmWmProv provenance; + const SCN source_scn = (SCN)0x4000; + const SCN watermark_scn = (SCN)0x5000; + char *source; + const char *ack_handler; + const char *ack_end; + const char *ack_match; + const char *holder_remove; + const char *watermark_advance; + const char *bitmap_replace; + const char *drive; + const char *ready_handler; + const char *ready_end; + const char *floor_query; + const char *floor_verdict; + const char *image_ready; + const char *prepare; + bool source_floor_gate; + + reset_fake_pcm_runtime(4); + cluster_node_id = 0; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + cluster_node_id = 1; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9011), + PCM_PENDING_X_RESERVE_OK); + + /* Source A (node 0) is pre-acked by the transfer driver. Apply the two + * production-equivalent state effects of non-source B's ACK (node 1), then + * use the bounded source contract below to prove their real handler order. */ + UT_ASSERT(cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_S_TO_N_INVALIDATE, 1)); + cluster_pcm_lock_pi_watermark_scn_advance(tag, watermark_scn, + CLUSTER_PCM_WM_SRC_ACK_SLOTLESS, 1, 9011, 17); + UT_ASSERT(cluster_pcm_lock_pi_watermark_prov_query(tag, &provenance)); + UT_ASSERT_EQ((int)provenance.source, (int)CLUSTER_PCM_WM_SRC_ACK_SLOTLESS); + UT_ASSERT_EQ(provenance.sender_node, 1); + UT_ASSERT_EQ(provenance.request_id, UINT64_C(9011)); + UT_ASSERT_EQ((uint64)provenance.new_scn, (uint64)watermark_scn); + UT_ASSERT_EQ(cluster_pcm_lock_query_s_holders_bitmap(tag), UINT32_C(1) << 0); + UT_ASSERT(cluster_pcm_lock_queue_pending_x_exact(tag, 3, 9011)); + UT_ASSERT_EQ((int)gcs_block_lost_write_verdict(watermark_scn, source_scn), + (int)GCS_LOST_WRITE_FAIL_STALE); + + /* The existing last line of defence must still refuse S < W and retain + * the pending-X authority for an exact retry/re-source path. */ + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &authority)); + stale_handoff = make_pcm_x_grd_handoff_token(tag, &authority, 0, 3, 52, 9011); + stale_handoff.page_scn = source_scn; + UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&stale_handoff), + PCM_X_GRD_HANDOFF_BAD_STATE); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_S); + UT_ASSERT_EQ(after.s_holders_bitmap, UINT32_C(1) << 0); + UT_ASSERT_EQ(after.pending_x_requester_node, 3); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_pi_watermark_scn_query(tag), + (uint64)watermark_scn); + + /* Pin the production ordering: B's exact ACK must publish W before the + * bitmap can drive type 49. IMAGE_READY must then consume that current + * floor and classify SX self handoff. That is progress work, not a permanent refusal: + * pin and flush it before opening REVOKING, then retry. Keep I/O and both + * content-lock refusals separately diagnosed so a native stall cannot be + * collapsed back into an opaque materialize-begin BUSY. */ + assert_ordered_in_function( + source, "\ncluster_bufmgr_pcm_own_prepare_s_source_image(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_abort_s_revoke(", prepare_contract, + lengthof(prepare_contract)); + free(source); +} + UT_TEST(test_revoke_finish_mode_rejects_pinned_vm_fsm_and_retains_main) { BufferTag tag; @@ -1179,6 +1308,104 @@ UT_TEST(test_revoke_finish_mode_rejects_pinned_vm_fsm_and_retains_main) UT_ASSERT_EQ(cluster_pcm_x_revoke_finish_mode(NULL, 0), CLUSTER_PCM_X_REVOKE_FINISH_INVALID); } +UT_TEST(test_pcm_tracking_excludes_only_fsm_for_user_and_shared_catalog_relations) +{ + BufferTag tag; + + memset(&tag, 0, sizeof(tag)); + tag.relNumber = FirstNormalObjectId; + tag.forkNum = MAIN_FORKNUM; + UT_ASSERT(cluster_pcm_x_buffer_tag_tracked(&tag, false)); + tag.forkNum = INIT_FORKNUM; + UT_ASSERT(cluster_pcm_x_buffer_tag_tracked(&tag, false)); + tag.forkNum = VISIBILITYMAP_FORKNUM; + UT_ASSERT(cluster_pcm_x_buffer_tag_tracked(&tag, false)); + tag.forkNum = FSM_FORKNUM; + UT_ASSERT(!cluster_pcm_x_buffer_tag_tracked(&tag, false)); + + /* shared_catalog widens the relation-number domain, but must not put its + * advisory FSM fork back into the PCM/PCM-X authority domain. */ + tag.relNumber = FirstNormalObjectId - 1; + tag.forkNum = MAIN_FORKNUM; + UT_ASSERT(!cluster_pcm_x_buffer_tag_tracked(&tag, false)); + UT_ASSERT(cluster_pcm_x_buffer_tag_tracked(&tag, true)); + tag.forkNum = INIT_FORKNUM; + UT_ASSERT(cluster_pcm_x_buffer_tag_tracked(&tag, true)); + tag.forkNum = VISIBILITYMAP_FORKNUM; + UT_ASSERT(cluster_pcm_x_buffer_tag_tracked(&tag, true)); + tag.forkNum = FSM_FORKNUM; + UT_ASSERT(!cluster_pcm_x_buffer_tag_tracked(&tag, true)); + UT_ASSERT(!cluster_pcm_x_buffer_tag_tracked(NULL, true)); +} + +UT_TEST(test_pcm_tracking_uses_one_tag_gate_for_acquire_direct_init_and_eviction) +{ + char *source = read_bufmgr_source(); + const char *should_track; + const char *should_track_end; + const char *invalidate_commit; + const char *invalidate_tail; + const char *victim; + const char *victim_end; + const char *direct_init; + const char *direct_init_end; + const char *direct_gate; + const char *arm; + const char *consume; + const char *content_lock; + + /* A saved tag must make exactly the same fork decision as a live + * BufferDesc. A relnumber-only eviction gate would leak an FSM release + * back into a domain from which acquire was excluded. */ + UT_ASSERT_NULL(strstr(source, "cluster_bufmgr_reln_pcm_tracked")); + should_track = strstr(source, "\ncluster_bufmgr_should_pcm_track("); + should_track_end = should_track != NULL ? strstr(should_track, "\n}") : NULL; + invalidate_commit = strstr(source, "\nInvalidateBufferCommitLocked("); + invalidate_tail = strstr(source, "\nInvalidateBufferCommitTailLocked("); + victim = strstr(source, "\nInvalidateVictimBuffer("); + victim_end = strstr(source, "\nstatic Buffer\nGetVictimBuffer("); + UT_ASSERT_NOT_NULL(should_track); + UT_ASSERT_NOT_NULL(should_track_end); + UT_ASSERT_NOT_NULL(invalidate_commit); + UT_ASSERT_NOT_NULL(invalidate_tail); + UT_ASSERT_NOT_NULL(victim); + UT_ASSERT_NOT_NULL(victim_end); + if (should_track != NULL && should_track_end != NULL) + assert_source_range_contains(should_track, should_track_end, + "cluster_pcm_x_buffer_tag_tracked"); + if (invalidate_commit != NULL && invalidate_tail != NULL) + assert_source_range_contains(invalidate_commit, invalidate_tail, + "cluster_pcm_x_buffer_tag_tracked"); + if (victim != NULL && victim_end != NULL) + assert_source_range_contains(victim, victim_end, + "cluster_pcm_x_buffer_tag_tracked"); + + /* The dedicated VM/FSM initialization wrapper retains provenance. Its + * common tag gate decides whether PCM proof is armed: FSM falls through to + * the local content lock, while VM still arms and consumes the exact proof. */ + direct_init = strstr(source, "\nLockBufferForAuxiliaryPageInit("); + direct_init_end = strstr(source, "\nvoid\nLockBufferForVisibilityMapPageInit("); + UT_ASSERT_NOT_NULL(direct_init); + UT_ASSERT_NOT_NULL(direct_init_end); + direct_gate = direct_init != NULL ? strstr(direct_init, "cluster_bufmgr_should_pcm_track(buf)") + : NULL; + arm = direct_init != NULL ? strstr(direct_init, "cluster_bufmgr_pcm_arm_direct_init") : NULL; + consume + = direct_init != NULL ? strstr(direct_init, "cluster_bufmgr_pcm_gate_direct_init") : NULL; + content_lock = direct_init != NULL + ? strstr(direct_init, "LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE)") + : NULL; + UT_ASSERT_NOT_NULL(direct_gate); + UT_ASSERT_NOT_NULL(arm); + UT_ASSERT_NOT_NULL(consume); + UT_ASSERT_NOT_NULL(content_lock); + if (direct_gate != NULL && arm != NULL && consume != NULL && content_lock != NULL) + UT_ASSERT(direct_gate < arm && arm < consume && consume < content_lock + && content_lock < direct_init_end); + + free(source); +} + UT_TEST(test_queue_revoke_retains_main_but_drops_unpinned_vm_fsm) { static const char *const begin_contract[] = { "expected_s->pcm_state != (uint8) PCM_STATE_S", @@ -1197,32 +1424,46 @@ UT_TEST(test_queue_revoke_retains_main_but_drops_unpinned_vm_fsm) "cluster_pcm_own_reservation_abort_exact", "PCM_OWN_FLAG_REVOKING", "UnlockBufHdr" }; - static const char *const finish_contract[] = { "LWLockAcquire(content_lock, LW_EXCLUSIVE)", - "LockBufHdr", - "cluster_pcm_own_gen_get", - "cluster_bufmgr_pcm_current_image_locked", - "BM_IO_IN_PROGRESS", - "PageGetLSN", - "UnlockBufHdr", - "FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL)", - "LockBufHdr", - "cluster_pcm_own_gen_get", - "cluster_bufmgr_pcm_current_image_locked", - "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED", - "result = CLUSTER_PCM_OWN_BUSY", - "PageGetLSN", - "cluster_pcm_own_revoke_retain_commit_exact", - "buf->pcm_state = (uint8) PCM_STATE_N", - "buf->buffer_type = (uint8) BUF_TYPE_PI", - "BM_DIRTY | BM_JUST_DIRTIED", - "BM_CHECKPOINT_NEEDED | BM_IO_ERROR", - "cluster_pcm_own_snapshot_locked" }; + static const char *const finish_contract[] + = { "BufTableHashCode", + "LWLockAcquire(partition_lock, LW_SHARED)", + "BufTableLookup", + "LockBufHdr", + "cluster_pcm_own_snapshot_matches_locked", + "cluster_bufmgr_pcm_current_image_locked", + "BM_IO_IN_PROGRESS", + "cluster_bufmgr_pin_for_gcs_locked", + "LWLockRelease(partition_lock)", + "LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)", + "PG_TRY();", + "cluster_pcm_own_snapshot_matches_locked", + "PageGetLSN", + "FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL)", + "LockBufHdr", + "cluster_pcm_own_snapshot_matches_locked", + "cluster_bufmgr_pcm_current_image_locked", + "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED", + "result = CLUSTER_PCM_OWN_BUSY", + "PageGetLSN", + "cluster_pcm_own_revoke_retain_commit_exact", + "buf->pcm_state = (uint8)PCM_STATE_N", + "buf->buffer_type = (uint8)BUF_TYPE_PI", + "BM_DIRTY | BM_JUST_DIRTIED", + "BM_CHECKPOINT_NEEDED | BM_IO_ERROR", + "cluster_pcm_own_snapshot_locked", + "LWLockRelease(content_lock)", + "cluster_bufmgr_unpin_for_gcs" }; static const char *const drop_contract[] = { "BufMappingPartitionLock", "LWLockAcquire(partition_lock, LW_EXCLUSIVE)", "BufTableLookup", "LockBufHdr", "BUF_STATE_GET_REFCOUNT", + "cluster_pcm_own_flags_get", + "BM_IO_IN_PROGRESS", "CLUSTER_PCM_X_REVOKE_FINISH_BUSY", + "CLUSTER_PCM_OWN_FINISH_REFUSAL_VM_FSM_PINNED", + "CLUSTER_PCM_OWN_FINISH_REFUSAL_IO_IN_PROGRESS", + "CLUSTER_PCM_OWN_FINISH_REFUSAL_LIVE_FLAGS", "PageGetLSN", "cluster_pcm_own_revoke_commit_exact", "buf->pcm_state = (uint8)PCM_STATE_N", @@ -1288,11 +1529,16 @@ UT_TEST(test_queue_revoke_retains_main_but_drops_unpinned_vm_fsm) const char *drop = strstr(finish, "InvalidateBuffer"); const char *legacy_pi = strstr(finish, "cluster_bufmgr_convert_to_pi_locked"); const char *mapping = strstr(finish, "partition_lock"); + const char *conditional + = strstr(finish, "LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)"); UT_ASSERT(refcount == NULL || refcount >= finish_end); UT_ASSERT(drop == NULL || drop >= finish_end); UT_ASSERT(legacy_pi == NULL || legacy_pi >= finish_end); - UT_ASSERT(mapping == NULL || mapping >= finish_end); + UT_ASSERT_NOT_NULL(mapping); + UT_ASSERT_NOT_NULL(conditional); + if (mapping != NULL && conditional != NULL) + UT_ASSERT(mapping < conditional); } free(source); } @@ -1314,7 +1560,7 @@ UT_TEST(test_retained_image_release_and_writeback_gates_are_exact) "LockBufHdr", "PCM_OWN_FLAG_REVOKING", "LWLockRelease(partition_lock)", - "LWLockAcquire(content_lock, LW_EXCLUSIVE)", + "LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)", "LockBufHdr", "BufferTagsEqual", "BM_VALID", @@ -1659,6 +1905,10 @@ UT_TEST(test_bufmgr_pcm_x_holder_gate_retry_is_bounded_outside_content_lock) CLUSTER_PCM_X_HOLDER_RETRY_WAIT); UT_ASSERT_EQ(cluster_pcm_x_holder_register_retry_action(PCM_X_QUEUE_NOT_READY, false), CLUSTER_PCM_X_HOLDER_RETRY_FAIL); + UT_ASSERT_EQ(cluster_pcm_x_holder_register_retry_action(PCM_X_QUEUE_BUSY, true), + CLUSTER_PCM_X_HOLDER_RETRY_WAIT); + UT_ASSERT_EQ(cluster_pcm_x_holder_register_retry_action(PCM_X_QUEUE_BUSY, false), + CLUSTER_PCM_X_HOLDER_RETRY_FAIL); UT_ASSERT_EQ(cluster_pcm_x_holder_register_retry_action(PCM_X_QUEUE_NO_CAPACITY, true), CLUSTER_PCM_X_HOLDER_RETRY_FAIL); for (i = 0; i < CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS; i++) { @@ -1834,7 +2084,7 @@ UT_TEST(test_queue_passive_pinned_s_release_serializes_bytes_and_ownership) "cluster_pcm_x_revoke_finish_mode(tag, shared_refcount)", "cluster_bufmgr_pin_for_gcs_locked", "LWLockRelease(partition_lock)", - "LWLockAcquire(content_lock, LW_EXCLUSIVE)", + "LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)", "cluster_pcm_own_snapshot_matches_locked", "PageGetLSN", "FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL)", @@ -1923,6 +2173,38 @@ UT_TEST(test_queue_passive_n_mirror_is_never_gcs_ship_authority) free(source); } +UT_TEST(test_gcs_ship_copy_reports_exact_nonblocking_refusal_stage) +{ + static const char *const refusal_contract[] + = { "CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT", + "CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID", + "CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST", + "CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_SECOND", + "CLUSTER_BUFMGR_GCS_COPY_REFUSAL_HC89_LSN_DRIFT" }; + char *source = read_bufmgr_source(); + const char *copy + = source != NULL ? strstr(source, "\ncluster_bufmgr_copy_block_for_gcs(") : NULL; + const char *copy_end + = copy != NULL ? strstr(copy, "\n/*\n * Borrow a live shared_buffers page") : NULL; + int i; + + /* P0-21 observation contract: every nonblocking false return that can + * become holder-side DENIED_MASTER_NOT_HOLDER identifies the precise + * residency/current-image/content-lock/HC89 refusal stage. */ + UT_ASSERT_NOT_NULL(copy); + UT_ASSERT_NOT_NULL(copy_end); + if (copy != NULL && copy_end != NULL) { + for (i = 0; i < lengthof(refusal_contract); i++) { + const char *hit = strstr(copy, refusal_contract[i]); + + UT_ASSERT_NOT_NULL(hit); + if (hit != NULL) + UT_ASSERT(hit < copy_end); + } + } + free(source); +} + UT_TEST(test_queue_installed_image_publication_is_exact_and_content_locked) { typedef ClusterPcmOwnResult (*PublishImageFn)(BufferDesc *, const ClusterPcmOwnSnapshot *, @@ -2092,7 +2374,8 @@ UT_TEST(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_aut "if (!writer_retry && !holder_retry)", "pg_usleep(1000L)" }; static const char *const snapshot_failure_contract[] = { "release_result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim)", - "cluster_bufmgr_pcm_x_writer_clear(entry)", + "entry->claim_cleanup_complete = true", + "entry->phase = PCM_X_WRITER_LEDGER_DEFERRED", "cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_CORRUPT, buf" }; static const char *const writer_holder_publish_contract[] @@ -2110,7 +2393,8 @@ UT_TEST(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_aut "cluster_gcs_pcm_x_writer_claim_release_and_wake_exact(&entry->claim)" }; static const char *const deferred_cleanup_contract[] = { "result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim)", - "cluster_bufmgr_pcm_x_writer_clear(entry)" }; + "entry->claim_cleanup_complete = true", + "cluster_bufmgr_pcm_x_writer_finish_claim_cleanup(entry" }; static const char *const abort_cleanup_contract[] = { "result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim)", "entry->phase = PCM_X_WRITER_LEDGER_DEFERRED" }; @@ -2299,14 +2583,16 @@ UT_TEST(test_preflight_busy_waits_then_clean_resnapshot_begins_reservation) UT_TEST(test_own_lifecycle_counters_land_on_exact_begin_and_x_commit) { static const char *const exact_begin_contract[] - = { "cluster_pcm_own_reservation_begin_exact", "PCM_OWN_FLAG_GRANT_PENDING", - "UnlockBufHdr", "result == CLUSTER_PCM_OWN_OK", - "cluster_pcm_x_stats_note_own_begin();" }; + = { "cluster_pcm_own_reservation_begin_exact", "PCM_OWN_FLAG_GRANT_PENDING", "UnlockBufHdr", + "result == CLUSTER_PCM_OWN_OK", "cluster_pcm_x_stats_note_own_begin();" }; static const char *const handoff_begin_contract[] - = { "flags == PCM_OWN_FLAG_GRANT_PENDING", "result = CLUSTER_PCM_OWN_OK", + = { "flags == PCM_OWN_FLAG_GRANT_PENDING", + "result = CLUSTER_PCM_OWN_OK", "cluster_pcm_own_revoke_to_grant_handoff_exact", - "handoff_transitioned = result == CLUSTER_PCM_OWN_OK", "UnlockBufHdr", - "if (handoff_transitioned)", "cluster_pcm_x_stats_note_own_begin();" }; + "handoff_transitioned = result == CLUSTER_PCM_OWN_OK", + "UnlockBufHdr", + "if (handoff_transitioned)", + "cluster_pcm_x_stats_note_own_begin();" }; static const char *const direct_init_begin_contract[] = { "cluster_pcm_own_reservation_begin_exact", "PCM_OWN_FLAG_GRANT_PENDING", "UnlockBufHdr", "pending_result == CLUSTER_PCM_OWN_OK", @@ -2350,11 +2636,117 @@ UT_TEST(test_own_lifecycle_counters_land_on_exact_begin_and_x_commit) free(source); } +UT_TEST(test_pcm_x_retain_flush_error_injection_is_exact_and_pre_write) +{ + static const char *const finish_flush_contract[] + = { "cluster_injection_is_armed(\"cluster-pcm-x-retain-flush-error\")", + "buf_state |= BM_DIRTY | BM_JUST_DIRTIED", + "needs_flush =", + "cluster_pcm_x_finish_retain_flush_active = true", + "FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL)", + "cluster_pcm_x_finish_retain_flush_active = false", + "cluster PCM-X retained-image finish FlushBuffer succeeded" }; + static const char *const flush_error_contract[] + = { "if (!StartBufferIO(buf, false))", + "cluster_pcm_x_finish_retain_flush_io_active = true", + "cluster_pcm_x_finish_retain_flush_active", + "cluster_pcm_own_flags_get(buf->buf_id) == PCM_OWN_FLAG_REVOKING", + "CLUSTER_INJECTION_POINT(\"cluster-pcm-x-retain-flush-error\")", + "cluster_injection_should_skip(\"cluster-pcm-x-retain-flush-error\")", + "errmsg(\"injected PCM-X retained-image FlushBuffer failure\")", + "smgrwrite(", + "TerminateBufferIO(buf, true, 0)", + "cluster_pcm_x_finish_retain_flush_io_active = false" }; + static const char *const catch_contract[] + = { "PG_CATCH();", + "cluster_pcm_x_finish_retain_flush_active = false", + "if (cluster_pcm_x_finish_retain_flush_error_context_pushed)", + "error_context_stack = cluster_pcm_x_finish_retain_flush_error_context_previous", + "if (content_locked && LWLockHeldByMe(content_lock))", + "HOLD_INTERRUPTS();", + "LWLockRelease(content_lock)", + "if (cluster_pcm_x_finish_retain_flush_io_active)", + "cluster_pcm_x_finish_retain_flush_io_active = false", + "AbortBufferIO(BufferDescriptorGetBuffer(buf))", + "cluster_bufmgr_unpin_for_gcs(buf)", + "PG_RE_THROW();" }; + char *source = read_bufmgr_source(); + const char *finish; + const char *catch; + const char *rethrow; + const char *resume; + + /* The point is armed only at the finish-revoke-retain seam. Test arming + * makes an otherwise already-flushed copy dirty without changing bytes, + * so the caller-pin/content-EXCLUSIVE FlushBuffer leg is deterministic. + * The generic flush path dispatches the point only while that exact call + * is active and the ownership token is still REVOKING. ERROR clears the + * process interrupt holdoff count before longjmp, so the cleanup must + * restore FlushBuffer's stack-local error callback, release content + * authority with a replacement hold, abort the exact ResourceOwner-tracked + * BufferIO while its raw pin is still live, and only then unpin. Otherwise + * an aux worker that absorbs the ERROR reaches commit-style resource-owner + * cleanup and PANICs with "lost track of buffer IO". LWLockRelease consumes + * the replacement hold itself; a second RESUME would underflow in cassert + * builds and an unconditional HOLD would leak on the no-lock path. */ + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_finish_revoke_retain(", + "\ncluster_bufmgr_pcm_own_release_retained_image(", + finish_flush_contract, lengthof(finish_flush_contract)); + assert_ordered_in_function(source, "\nFlushBuffer(", "\n/*\n * RelationGetNumberOfBlocksInFork", + flush_error_contract, lengthof(flush_error_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_finish_revoke_retain(", + "\ncluster_bufmgr_pcm_own_release_retained_image(", catch_contract, + lengthof(catch_contract)); + finish = strstr(source, "\ncluster_bufmgr_pcm_own_finish_revoke_retain("); + UT_ASSERT_NOT_NULL(finish); + catch = strstr(finish, "PG_CATCH();"); + UT_ASSERT_NOT_NULL(catch); + rethrow = strstr(catch, "PG_RE_THROW();"); + UT_ASSERT_NOT_NULL(rethrow); + resume = strstr(catch, "RESUME_INTERRUPTS();"); + UT_ASSERT(resume == NULL || resume > rethrow); + free(source); +} + +UT_TEST(test_writer_activation_diagnostic_covers_commit_clear_and_unguarded_n_boundaries) +{ + static const char *const commit_contract[] = { + "cluster_pcm_own_writer_grant_commit_exact(", + "buf->pcm_state = new_pcm_state", + "cluster_pcm_own_snapshot_locked(buf, &activation_diag)", + "UnlockBufHdr(buf, buf_state)", + "cluster_pcm_own_activation_diag_emit(\"writer-commit\"" + }; + static const char *const clear_contract[] = { + "cluster_pcm_own_snapshot_locked(buf, &live)", + "cluster_pcm_own_writer_activation_clear_exact(", + "UnlockBufHdr(buf, buf_state)", + "cluster_pcm_own_activation_diag_emit(\"writer-activation-clear\"" + }; + char *source = read_bufmgr_source(); + + UT_ASSERT_EQ(sizeof(ClusterPcmOwnSnapshot), 56); + UT_ASSERT_NOT_NULL(strstr(source, "out->writer_activation_token")); + assert_ordered_in_function(source, "\ncluster_pcm_own_finish_grant_reservation(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_finish_x_commit(", + commit_contract, lengthof(commit_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_writer_activation_clear(", + "\nstatic bool\ncluster_bufmgr_pcm_x_writer_finish_claim_cleanup(", + clear_contract, lengthof(clear_contract)); + UT_ASSERT_NOT_NULL(strstr(source, "\"invalidate-stage-n-pi\"")); + UT_ASSERT_NOT_NULL(strstr(source, "\"invalidate-stage-n-drop\"")); + UT_ASSERT_NOT_NULL(strstr(source, "\"drop-no-wire-stage-n-pi\"")); + UT_ASSERT_NOT_NULL(strstr(source, "\"drop-no-wire-stage-n-drop\"")); + UT_ASSERT_NOT_NULL(strstr(source, "\"discard-pi-stage-n\"")); + free(source); +} + int main(void) { - UT_PLAN(52); + UT_PLAN(60); UT_RUN(test_shmem_initializes_complete_entry); + UT_RUN(test_writer_activation_fence_blocks_revoke_until_exact_clear); UT_RUN(test_begin_abort_is_exact_and_monotonic); UT_RUN(test_invalid_live_flag_shapes_are_corrupt_not_busy); UT_RUN(test_grant_commit_is_exact_and_bumps_once); @@ -2378,6 +2770,7 @@ main(void) UT_RUN(test_lockbuffer_content_error_uses_post_master_rollback_contract); UT_RUN(test_bufmgr_generation_bump_failure_is_classified_under_header_lock); UT_RUN(test_lockbuffer_reservation_failures_use_busy_corrupt_classifier); + UT_RUN(test_pending_x_denied_retry_leaves_master_invalidate_gap); UT_RUN(test_bufmgr_finish_rejects_invalid_state_and_initializes_acquire_result); UT_RUN(test_bufmgr_finish_and_abort_gate_on_exact_base_state); UT_RUN(test_d5a_release_error_keeps_descriptor_out_of_freelist); @@ -2385,7 +2778,10 @@ main(void) UT_RUN(test_queue_contract_exposes_prepare_only_begin_api); UT_RUN(test_queue_contract_exposes_opaque_retained_revoke_api); UT_RUN(test_queue_n_source_refresh_is_exact_and_publishes_only_complete_image); + UT_RUN(test_queue_s_source_dirty_flush_makes_progress_and_reports_exact_refusal); UT_RUN(test_revoke_finish_mode_rejects_pinned_vm_fsm_and_retains_main); + UT_RUN(test_pcm_tracking_excludes_only_fsm_for_user_and_shared_catalog_relations); + UT_RUN(test_pcm_tracking_uses_one_tag_gate_for_acquire_direct_init_and_eviction); UT_RUN(test_queue_revoke_retains_main_but_drops_unpinned_vm_fsm); UT_RUN(test_retained_image_release_and_writeback_gates_are_exact); UT_RUN(test_retained_drain_retags_invalid_only_after_exact_token_release); @@ -2402,10 +2798,13 @@ main(void) UT_RUN(test_queue_installed_image_publication_is_exact_and_content_locked); UT_RUN(test_queue_self_source_handoff_is_single_lifecycle_and_readonly_drain); UT_RUN(test_queue_passive_n_mirror_is_never_gcs_ship_authority); + UT_RUN(test_gcs_ship_copy_reports_exact_nonblocking_refusal_stage); UT_RUN(test_queue_writer_grant_snapshot_is_claim_and_generation_exact); UT_RUN(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_authority); UT_RUN(test_preflight_busy_waits_then_clean_resnapshot_begins_reservation); UT_RUN(test_own_lifecycle_counters_land_on_exact_begin_and_x_commit); + UT_RUN(test_pcm_x_retain_flush_error_injection_is_exact_and_pre_write); + UT_RUN(test_writer_activation_diagnostic_covers_commit_clear_and_unguarded_n_boundaries); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index eb87be8844..aa7e387fb8 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -112,6 +112,11 @@ static bool grant_interlock_armed; static LWLock *grant_interlock_lock; static PcmXMasterTicketSlot *grant_interlock_ticket; static uint64 grant_interlock_generation; +static bool drive_terminal_interlock_armed; +static LWLock *drive_terminal_interlock_lock; +static PcmXMasterTagSlot *drive_terminal_interlock_tag; +static PcmXMasterTicketSlot *drive_terminal_interlock_ticket; +static bool drive_terminal_interlock_reclaim; static LWLock *local_generation_interlock_lock; static PcmXSlotHeader *local_generation_interlock_slot; static LWLock *holder_abort_interlock_lock; @@ -151,6 +156,7 @@ static int lock_acquire_during_iteration_count; static void maybe_publish_staged_prehandle_insert_exists(void); static void maybe_inject_local_rekey_insert_failure(void); static uint64 test_slot_generation(PcmXSlotHeader *slot); +static void test_set_slot_generation(PcmXSlotHeader *slot, uint64 generation); static uint32 test_slot_state(PcmXSlotHeader *slot); static void test_set_slot_state(PcmXSlotHeader *slot, uint32 state); static uint32 test_slot_flags(PcmXSlotHeader *slot); @@ -384,6 +390,48 @@ LWLockAcquire(LWLock *lock, LWLockMode mode) UT_ASSERT_NOT_NULL(grant_interlock_ticket); grant_interlock_ticket->ref.grant_generation = grant_interlock_generation; } + if (lock->tranche == LWTRANCHE_CLUSTER_PCM_X_MASTER && drive_terminal_interlock_armed + && lock == drive_terminal_interlock_lock && mode == LW_EXCLUSIVE) { + uint32 node_bit; + + drive_terminal_interlock_armed = false; + UT_ASSERT_NOT_NULL(drive_terminal_interlock_tag); + UT_ASSERT_NOT_NULL(drive_terminal_interlock_ticket); + node_bit = UINT32_C(1) << drive_terminal_interlock_ticket->ref.identity.node_id; + /* Model an exact FINAL_ACK/FINAL_CONFIRM actor that wins after the + * retry pump's allocator lookup but before its master-domain lock. + * Terminal state is published last, as in final_confirm_exact(). */ + drive_terminal_interlock_tag->head_index = PCM_X_INVALID_SLOT_INDEX; + drive_terminal_interlock_tag->tail_index = PCM_X_INVALID_SLOT_INDEX; + drive_terminal_interlock_tag->active_index = PCM_X_INVALID_SLOT_INDEX; + drive_terminal_interlock_tag->active_slot_generation = 0; + drive_terminal_interlock_tag->queue_state_sequence++; + (void)pg_atomic_fetch_and_u32(&drive_terminal_interlock_tag->queued_node_bitmap, + ~node_bit); + drive_terminal_interlock_ticket->next_index = PCM_X_INVALID_SLOT_INDEX; + drive_terminal_interlock_ticket->prev_index = PCM_X_INVALID_SLOT_INDEX; + drive_terminal_interlock_ticket->reliable.retry_deadline_ms = 0; + drive_terminal_interlock_ticket->reliable.expected_responder_session = 0; + drive_terminal_interlock_ticket->reliable.retry_count = 0; + drive_terminal_interlock_ticket->reliable.last_responder_node + = (uint32)drive_terminal_interlock_ticket->ref.identity.node_id; + drive_terminal_interlock_ticket->reliable.expected_responder_node = 0; + drive_terminal_interlock_ticket->reliable.pending_opcode = 0; + drive_terminal_interlock_ticket->reliable.last_response_opcode + = PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM; + drive_terminal_interlock_ticket->reliable.phase = 0; + drive_terminal_interlock_ticket->reliable.flags = 0; + drive_terminal_interlock_ticket->reliable.reserved = 0; + drive_terminal_interlock_ticket->reliable.response_tombstone_mask + |= PCM_X_RESPONSE_TOMBSTONE_COMPLETE; + test_set_slot_state(&drive_terminal_interlock_ticket->slot, PCM_XT_COMPLETE); + if (drive_terminal_interlock_reclaim) { + test_set_slot_generation( + &drive_terminal_interlock_ticket->slot, + test_slot_generation(&drive_terminal_interlock_ticket->slot) + 1); + test_set_slot_state(&drive_terminal_interlock_ticket->slot, PCM_XT_FREE); + } + } if (lock->tranche == LWTRANCHE_CLUSTER_PCM_X_LOCAL && lock == local_generation_interlock_lock && mode == LW_SHARED && local_generation_interlock_slot != NULL) { @@ -580,6 +628,11 @@ reset_fake_shmem(void) grant_interlock_lock = NULL; grant_interlock_ticket = NULL; grant_interlock_generation = 0; + drive_terminal_interlock_armed = false; + drive_terminal_interlock_lock = NULL; + drive_terminal_interlock_tag = NULL; + drive_terminal_interlock_ticket = NULL; + drive_terminal_interlock_reclaim = false; local_generation_interlock_lock = NULL; local_generation_interlock_slot = NULL; holder_abort_interlock_lock = NULL; @@ -2122,6 +2175,7 @@ UT_TEST(test_wire_abi_sizes_are_exact) UT_ASSERT_EQ(sizeof(PcmXAdmitAckPayload), 112); UT_ASSERT_EQ(sizeof(PcmXPhasePayload), 96); UT_ASSERT_EQ(sizeof(PcmXRevokePayload), 96); + UT_ASSERT_EQ(sizeof(PcmXRevokePayloadV2), 104); UT_ASSERT_EQ(sizeof(PcmXGrantPayload), 128); UT_ASSERT_EQ(sizeof(PcmXInstallReadyPayload), 112); UT_ASSERT_EQ(offsetof(PcmXInstallReadyPayload, rebased_own_generation), @@ -2152,6 +2206,7 @@ UT_TEST(test_wire_abi_offsets_are_exact) UT_ASSERT_EQ(offsetof(PcmXEnqueuePayload, prehandle), 64); UT_ASSERT_EQ(offsetof(PcmXAdmitAckPayload, prehandle), 88); UT_ASSERT_EQ(offsetof(PcmXAdmitAckPayload, result), 104); + UT_ASSERT_EQ(offsetof(PcmXRevokePayloadV2, required_page_scn), 96); UT_ASSERT_EQ(offsetof(PcmXGrantPayload, image), 88); UT_ASSERT_EQ(offsetof(PcmXBlockerSetHeaderPayload, set_generation), 88); UT_ASSERT_EQ(offsetof(PcmXBlockerChunkPayload, blocker), 80); @@ -2177,7 +2232,7 @@ UT_TEST(test_wire_abi_offsets_are_exact) UT_TEST(test_runtime_layout_abi_and_offsets_are_exact) { - UT_ASSERT_EQ(PCM_X_SHMEM_LAYOUT_VERSION, 13); + UT_ASSERT_EQ(PCM_X_SHMEM_LAYOUT_VERSION, 14); UT_ASSERT_EQ(PCM_X_LOCK_PARTITIONS, NUM_BUFFER_PARTITIONS); UT_ASSERT_EQ(PCM_X_LWLOCK_COUNT, 257); UT_ASSERT_EQ(sizeof(PcmXShmemLayout), 440); @@ -2194,16 +2249,17 @@ UT_TEST(test_runtime_layout_abi_and_offsets_are_exact) UT_ASSERT_EQ(sizeof(PcmXLocalProgress), 248); UT_ASSERT_EQ(offsetof(PcmXLocalProgress, master_session_incarnation), 224); UT_ASSERT_EQ(offsetof(PcmXLocalProgress, master_node), 232); - UT_ASSERT_EQ(sizeof(PcmXLocalHolderProgress), 160); - UT_ASSERT_EQ(offsetof(PcmXLocalHolderProgress, master_session_incarnation), 144); - UT_ASSERT_EQ(offsetof(PcmXLocalHolderProgress, master_node), 152); + UT_ASSERT_EQ(sizeof(PcmXLocalHolderProgress), 168); + UT_ASSERT_EQ(offsetof(PcmXLocalHolderProgress, required_page_scn), 128); + UT_ASSERT_EQ(offsetof(PcmXLocalHolderProgress, master_session_incarnation), 152); + UT_ASSERT_EQ(offsetof(PcmXLocalHolderProgress, master_node), 160); UT_ASSERT_EQ(sizeof(PcmXLocalBlockerSnapshot), 112); UT_ASSERT_EQ(sizeof(PcmXMasterTagSlot), 120); UT_ASSERT_EQ(sizeof(PcmXMasterTicketSlot), 392); UT_ASSERT_EQ(offsetof(PcmXMasterTicketSlot, grant_base_own_generation), 384); UT_ASSERT_EQ(sizeof(PcmXBlockerSlot), 128); - UT_ASSERT_EQ(sizeof(PcmXLocalTagSlot), 760); - UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, grant_base_own_generation), 752); + UT_ASSERT_EQ(sizeof(PcmXLocalTagSlot), 768); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, grant_base_own_generation), 760); UT_ASSERT_EQ(sizeof(PcmXLocalMembershipSlot), 168); UT_ASSERT_EQ(sizeof(PcmXPeerFrontier), 48); UT_ASSERT_EQ(sizeof(PcmXPeerBinding), 16); @@ -2231,10 +2287,11 @@ UT_TEST(test_runtime_layout_abi_and_offsets_are_exact) UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, committed_own_generation), 408); UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_ref), 416); UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_image), 504); - UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_reliable), 544); - UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_terminal_drain_generation), 600); - UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, blocker_snapshot_ref), 608); - UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, blocker_snapshot_reliable), 696); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_required_page_scn), 544); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_reliable), 552); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_terminal_drain_generation), 608); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, blocker_snapshot_ref), 616); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, blocker_snapshot_reliable), 704); UT_ASSERT_EQ(offsetof(PcmXLocalMembershipSlot, identity), 24); UT_ASSERT_EQ(offsetof(PcmXLocalMembershipSlot, tag_slot_index), 128); UT_ASSERT_EQ(offsetof(PcmXLocalMembershipSlot, admitted_round), 160); @@ -2299,7 +2356,7 @@ UT_TEST(test_layout_v13_records_transfer_and_terminal_frontiers) PcmXShmemLayout layout; cluster_pcm_x_layout_compute(122, 25, 16384, 1024, &layout); - UT_ASSERT_EQ(layout.version, 13); + UT_ASSERT_EQ(layout.version, 14); UT_ASSERT_EQ(layout.lock_partition_count, PCM_X_LOCK_PARTITIONS); UT_ASSERT_EQ(layout.lwlock_count, PCM_X_LWLOCK_COUNT); UT_ASSERT_EQ(sizeof(PcmXPeerFrontier), 48); @@ -3838,6 +3895,7 @@ UT_TEST(test_local_queue_invalidate_authority_is_exact_and_read_only) PcmXLocalHolderSnapshot holder_snapshot; PcmXLocalBlockerSnapshot blocker_snapshot; PcmXTicketRef probe_ref; + ClusterLmdVertex blocker; uint64 master_session = UINT64_C(8302); init_active_pcm_x(UINT64_C(7216)); @@ -3860,6 +3918,13 @@ UT_TEST(test_local_queue_invalidate_authority_is_exact_and_read_only) &holder_snapshot, &holder, 1, NULL, 0, &blocker_snapshot), PCM_X_QUEUE_OK); + /* An exact INVALIDATE is itself master transfer-phase evidence. An + * already-staged empty blocker set has no graph edge to omit, so passive + * pinned S release must not wait for its reordered type-48 ACK. */ + UT_ASSERT_EQ(cluster_pcm_x_local_queue_invalidate_authorize_exact( + &probe_ref.identity.tag, probe_ref.identity.cluster_epoch, + probe_ref.identity.request_id, probe_ref.identity.node_id, 1, master_session), + PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, 1, master_session), PCM_X_QUEUE_OK); @@ -3882,6 +3947,17 @@ UT_TEST(test_local_queue_invalidate_authority_is_exact_and_read_only) probe_ref.identity.request_id, probe_ref.identity.node_id, 1, master_session + 1), PCM_X_QUEUE_STALE); + /* An in-flight nonempty set still needs its exact ACK/graph proof. */ + blocker = make_blocker(holder_key.identity.node_id, holder_key.identity.procno, + holder_key.identity.request_id); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact( + &probe_ref, 1, master_session, &holder_snapshot, &holder, 1, + &blocker, 1, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_queue_invalidate_authorize_exact( + &probe_ref.identity.tag, probe_ref.identity.cluster_epoch, + probe_ref.identity.request_id, probe_ref.identity.node_id, 1, master_session), + PCM_X_QUEUE_NOT_READY); UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); } @@ -6457,6 +6533,167 @@ capture_commit_retry_stage(uint8 msg_type, int32 dest_node_id, const void *paylo return capture->accept; } +/* + * P0-20/type49: an exact queue INVALIDATE BUSY is a zero-mutation negative + * proof. It must pause this ticket's retry pump without crediting the busy + * holder, changing authority, or minting a new transfer identity. + */ +UT_TEST(test_master_invalidate_busy_backs_off_exact_revoke_ticket) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerSnapshot blocker_snapshot; + PcmXBlockerSetHeaderPayload blocker_commit; + PcmXMasterDriveSnapshot armed; + PcmXMasterDriveSnapshot waiting; + PcmXMasterDriveSnapshot backed_off; + PcmXMasterDriveSnapshot replay; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot after; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef transfer; + PcmXRevokePayload revoke; + const uint64 requester_session = UINT64_C(7484); + const uint64 source_session = UINT64_C(8484); + const uint64 graph_generation = UINT64_C(9484); + const uint32 source_bit = UINT32_C(1) << 2; + const uint32 busy_bit = UINT32_C(1) << 3; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(814, 0, 24, UINT64_C(8484), requester_session, 1, &admission); + arm_blocker_probe(&admission.ref, 2, source_session); + blocker_commit = make_blocker_header(&admission.ref, 1, NULL, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 2, + source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact( + &blocker_commit, 2, source_session, NULL, 0, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_graph_commit_exact( + &blocker_snapshot, NULL, 0, graph_generation), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 2, + source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&admission.ref, graph_generation, + &transfer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(&transfer, 2, source_session, &revoke), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact( + &transfer.identity.tag, transfer.identity.cluster_epoch, &armed), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_bitmap_replace_exact( + &armed, source_bit | busy_bit, source_bit, &waiting), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + before = *ticket; + + UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact( + &waiting, 3, UINT64_C(1000), UINT64_C(100), &backed_off), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(backed_off.retry_deadline_ms, UINT64_C(1100)); + UT_ASSERT_EQ(backed_off.pending_s_holders_bitmap, source_bit | busy_bit); + UT_ASSERT_EQ(backed_off.acked_s_holders_bitmap, source_bit); + UT_ASSERT_EQ(backed_off.retry_count, waiting.retry_count); + after = *ticket; + after.reliable.retry_deadline_ms = before.reliable.retry_deadline_ms; + UT_ASSERT(memcmp(&after, &before, sizeof(after)) == 0); + + /* The pre-BUSY snapshot is consumed exactly. Replayed BUSY during the + * same live window is an idempotent scheduling no-op. */ + memset(&replay, 0xA5, sizeof(replay)); + UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact( + &waiting, 3, UINT64_C(1001), UINT64_C(100), &replay), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(&replay, &(PcmXMasterDriveSnapshot){ 0 }, sizeof(replay)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact( + &backed_off, 3, UINT64_C(1099), UINT64_C(100), &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(replay.retry_deadline_ms, UINT64_C(1100)); + UT_ASSERT_EQ(ticket->reliable.retry_deadline_ms, UINT64_C(1100)); + + /* The source already supplied the retained image and is ACKed; a BUSY + * attributed to it cannot pause invalidation of another holder. */ + UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact( + &backed_off, 2, UINT64_C(1100), UINT64_C(100), &replay), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(ticket->reliable.retry_deadline_ms, UINT64_C(1100)); + + /* Expiry permits one new bounded window, still on the same identity. */ + UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact( + &backed_off, 3, UINT64_C(1100), UINT64_C(100), &replay), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(replay.retry_deadline_ms, UINT64_C(1200)); + UT_ASSERT(ticket_refs_equal(&replay.ref, &waiting.ref)); + UT_ASSERT(memcmp(&replay.image, &waiting.image, sizeof(replay.image)) == 0); +} + +/* + * A requester can consume COMMIT_X and finish the exact FINAL protocol after + * the periodic pump's allocator lookup but before its master-domain lock. + * That monotone terminal successor is a stale retry, never queue corruption. + */ +UT_TEST(test_master_commit_retry_loses_race_to_terminal_progress_without_fail_closed) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterDriveSnapshot armed; + PcmXMasterDriveSnapshot retried; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef transfer; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload first_commit; + PcmXPhasePayload retry_commit; + uint64 image_id; + uint32 partition; + int mode; + const uint64 requester_session = UINT64_C(7383); + const uint64 source_session = UINT64_C(8383); + + for (mode = 0; mode < 2; mode++) { + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe_with_base(816 + mode, 0, 22 + mode, UINT64_C(8383), + requester_session, 1, UINT64_C(5), &admission); + drive_master_ticket_to_prepared(&admission, source_session, UINT64_C(9383), &transfer, + &image_id); + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + + memset(&install_ready, 0, sizeof(install_ready)); + install_ready.ref = transfer; + install_ready.image_id = image_id; + install_ready.result = PCM_X_QUEUE_OK; + install_ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact( + &install_ready, 0, 0, requester_session, &first_commit), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact( + &transfer.identity.tag, transfer.identity.cluster_epoch, &armed), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(armed.pending_opcode, PGRAC_IC_MSG_PCM_X_COMMIT_X); + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&transfer.identity.tag)); + drive_terminal_interlock_lock = &header->master_locks[partition].lock; + drive_terminal_interlock_tag = tag_slot; + drive_terminal_interlock_ticket = ticket; + drive_terminal_interlock_reclaim = mode != 0; + drive_terminal_interlock_armed = true; + memset(&retry_commit, 0xA5, sizeof(retry_commit)); + memset(&retried, 0xA5, sizeof(retried)); + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_exact( + &armed, UINT64_C(1000), UINT64_C(1100), &retry_commit, &retried), + mode == 0 ? PCM_X_QUEUE_NOT_READY : PCM_X_QUEUE_STALE); + UT_ASSERT(!drive_terminal_interlock_armed); + UT_ASSERT(memcmp(&retry_commit, &(PcmXPhasePayload){ 0 }, sizeof(retry_commit)) == 0); + UT_ASSERT(memcmp(&retried, &(PcmXMasterDriveSnapshot){ 0 }, sizeof(retried)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + } +} + /* * P0-14: the master has crossed the handoff boundary and durably armed type * 53, but the first COMMIT_X frame is dropped. The real bounded master-work @@ -7832,9 +8069,11 @@ UT_TEST(test_master_debug_iterators_report_live_slots) { PcmXMasterAdmission admission; PcmXEnqueuePayload request; + PcmXWaitIdentity local_identity; + PcmXLocalHandle local_leader; Size cursor = 0; Size index = 0; - char line[256]; + char line[512]; init_active_pcm_x(UINT64_C(77)); /* Nothing admitted yet: both iterators must report no occupied slots. */ @@ -7853,6 +8092,19 @@ UT_TEST(test_master_debug_iterators_report_live_slots) UT_ASSERT(cluster_pcm_x_master_ticket_debug_next(&cursor, &index, line, sizeof(line))); UT_ASSERT(strstr(line, "state=") != NULL); UT_ASSERT(strstr(line, "leg_op=") != NULL); + + init_active_pcm_x(UINT64_C(78)); + local_identity = make_wait_identity(717, 0, 26, UINT64_C(21002)); + bind_local_master(1, local_identity.cluster_epoch, UINT64_C(4501)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&local_identity, 1, UINT64_C(4501), + &local_leader), PCM_X_QUEUE_OK); + cursor = 0; + UT_ASSERT(cluster_pcm_x_local_tag_debug_next(&cursor, &index, line, sizeof(line))); + UT_ASSERT(strstr(line, "blocker_gen=") != NULL); + UT_ASSERT(strstr(line, "blocker_op=") != NULL); + UT_ASSERT(strstr(line, "blocker_last=") != NULL); + UT_ASSERT(strstr(line, "blocker_count=") != NULL); + UT_ASSERT(strstr(line, "blocker_head=") != NULL); } UT_TEST(test_master_cancel_is_exact_and_unlinks_middle_without_fifo_damage) @@ -9934,8 +10186,10 @@ UT_TEST(test_local_tag_only_holder_transfer_persists_until_exact_drain) PcmXLocalFollowerWfgSnapshot wfg; PcmXLocalCutoff cutoff; PcmXLocalCutoff promoted_cutoff; + PcmXLocalImageReadyRefusal ready_refusal; PcmXLocalHandle late; PcmXLocalHandle next_leader; + PcmXLocalTagSlot *tag_slot; PcmXLocalReliableToken token; PcmXLocalWriterClaim writer_claim; PcmXTicketRef probe_ref; @@ -9951,6 +10205,7 @@ UT_TEST(test_local_tag_only_holder_transfer_persists_until_exact_drain) PcmXWaitIdentity wake_items[1]; PcmXLocalWakeBatch wake_batch; const uint64 master_session = UINT64_C(179); + uint32 state_flags; init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; @@ -10002,15 +10257,28 @@ UT_TEST(test_local_tag_only_holder_transfer_persists_until_exact_drain) bad_revoke.image_id = UINT64CONST(0xf0000000000000c8); UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&bad_revoke, 1, master_session), PCM_X_QUEUE_INVALID); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), - PCM_X_QUEUE_NOT_READY); + /* Complete generation 1 normally. The separate gate-contention test below + * covers an ACK frame that is delivered but cannot apply before REVOKE. */ UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, - 1, master_session), + 1, master_session), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, 1, master_session), PCM_X_QUEUE_DUPLICATE); UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + /* The master may consume the first ACK before a same-ticket re-PROBE + * publishes a later empty blocker COMMIT. Authenticated REVOKE proves the + * master crossed that ACK boundary, so the source must consume this exact + * empty replay instead of waiting forever for an ACK the master no longer + * needs to send. */ + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + NULL, 0, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(blocker_snapshot.set_generation, 2); /* The exact blocker ACK tombstone is durable evidence, but it cannot * reopen the writer round before REVOKE/IMAGE_READY/DRAIN completes. */ UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&identity.tag, identity.cluster_epoch, @@ -10038,9 +10306,30 @@ UT_TEST(test_local_tag_only_holder_transfer_persists_until_exact_drain) ready.image.page_lsn = UINT64_C(53); ready.image.source_node = 0; ready.image.page_checksum = UINT32_C(54); + tag_slot = &local_tag_slots(header)[holder_snapshot.tag_slot.slot_index]; + state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); + pg_atomic_write_u32(&tag_slot->slot.state_flags, + state_flags + | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); + ready_refusal = PCM_X_LOCAL_IMAGE_READY_REFUSAL_NONE; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( + &ready, 1, master_session, &replay, &ready_refusal), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(ready_refusal, PCM_X_LOCAL_IMAGE_READY_REFUSAL_LOCAL_GATE); + pg_atomic_write_u32(&tag_slot->slot.state_flags, state_flags); + tag_slot->active_holder_head_index = 0; + ready_refusal = PCM_X_LOCAL_IMAGE_READY_REFUSAL_NONE; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( + &ready, 1, master_session, &replay, &ready_refusal), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(ready_refusal, PCM_X_LOCAL_IMAGE_READY_REFUSAL_ACTIVE_HOLDER); + tag_slot->active_holder_head_index = PCM_X_INVALID_SLOT_INDEX; + ready_refusal = PCM_X_LOCAL_IMAGE_READY_REFUSAL_ACTIVE_HOLDER; UT_ASSERT_EQ( - cluster_pcm_x_local_holder_image_ready_arm_exact(&ready, 1, master_session, &replay), + cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( + &ready, 1, master_session, &replay, &ready_refusal), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ready_refusal, PCM_X_LOCAL_IMAGE_READY_REFUSAL_NONE); UT_ASSERT(memcmp(&ready, &replay, sizeof(ready)) == 0); memset(&replay, 0, sizeof(replay)); UT_ASSERT_EQ( @@ -10139,13 +10428,15 @@ UT_TEST(test_local_non_source_blocker_participant_drains_and_retires_exactly) PcmXDrainPollPayload stale_poll; PcmXRetirePayload retire; PcmXSlotRef found; + ClusterLmdVertex blocker; const uint64 master_session = UINT64_C(1801); int i; - /* Model the local evidence left on a non-source participant: two independent - * S pins are captured by PROBE, both are later released, and no holder_ref - * image-source lane is installed. DRAIN must therefore close from the exact - * type-48 blocker ACK tombstone alone. */ + /* Model the local evidence left on a non-source participant after a + * same-ticket re-PROBE races behind the ACK already consumed by the master: + * two independent S pins are captured, both are later released, and an empty + * blocker COMMIT is still staged locally. The exact terminal DRAIN must + * consume that empty generation without waiting forever for a replayed ACK. */ init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; bind_local_master(1, UINT64_C(9), master_session); @@ -10162,12 +10453,11 @@ UT_TEST(test_local_non_source_blocker_participant_drains_and_retires_exactly) &probe_ref, 1, master_session, holder_copies, 2, &holder_snapshot), PCM_X_QUEUE_OK); UT_ASSERT_EQ(holder_snapshot.holder_count, 2); + blocker = make_blocker(holder_keys[0].identity.node_id, holder_keys[0].identity.procno, + holder_keys[0].identity.request_id); UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, &holder_snapshot, holder_copies, 2, - NULL, 0, &blocker_snapshot), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, - 1, master_session), + &blocker, 1, &blocker_snapshot), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact(&probe_ref.identity.tag, probe_ref.identity.cluster_epoch, @@ -10179,13 +10469,31 @@ UT_TEST(test_local_non_source_blocker_participant_drains_and_retires_exactly) tag_slot = &local_tag_slots(header)[holder_snapshot.tag_slot.slot_index]; UT_ASSERT(ticket_refs_equal(&tag_slot->holder_ref, &(PcmXTicketRef){ 0 })); UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &probe_ref)); - UT_ASSERT_EQ(tag_slot->blocker_snapshot_reliable.last_response_opcode, - PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK); + UT_ASSERT_EQ(tag_slot->blocker_snapshot_reliable.pending_opcode, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT); memset(&poll, 0, sizeof(poll)); poll.ref = probe_ref; poll.ref.grant_generation = UINT64_C(101); poll.drain_generation = UINT64_C(42); + UT_ASSERT_EQ(tag_slot->blocker_snapshot_count, 1); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact( + &probe_ref, 1, master_session, holder_copies, 2, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holder_snapshot.holder_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact( + &probe_ref, 1, master_session, &holder_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(blocker_snapshot.set_generation, 2); + UT_ASSERT_EQ(tag_slot->blocker_snapshot_count, 0); + UT_ASSERT_EQ(tag_slot->blocker_snapshot_reliable.pending_opcode, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT); UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_DUPLICATE); @@ -12788,6 +13096,7 @@ UT_TEST(test_local_blocker_snapshot_replays_exact_holder_set_and_gates_revoke) PcmXBlockerChunkPayload edge; PcmXRevokePayload revoke; PcmXTicketRef probe_ref; + ClusterLmdVertex blocker; const uint64 master_session = UINT64_C(180); init_active_pcm_x(UINT64_C(77)); @@ -12824,8 +13133,6 @@ UT_TEST(test_local_blocker_snapshot_replays_exact_holder_set_and_gates_revoke) revoke.ref = probe_ref; revoke.ref.grant_generation = UINT64_C(101); UT_ASSERT(cluster_pcm_x_image_id_encode(1, UINT64_C(201), &revoke.image_id)); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), - PCM_X_QUEUE_NOT_READY); UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, 1, master_session), PCM_X_QUEUE_OK); @@ -12833,10 +13140,81 @@ UT_TEST(test_local_blocker_snapshot_replays_exact_holder_set_and_gates_revoke) UT_ASSERT_EQ( cluster_pcm_x_local_blocker_snapshot_copy_exact(&blocker_snapshot, &header, &edge, 1), PCM_X_QUEUE_NOT_READY); + /* A later nonempty generation never qualifies for the replay exception. */ + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact( + &probe_ref, 1, master_session, &holder_copy, 1, &holder_snapshot), + PCM_X_QUEUE_OK); + blocker = make_blocker(holder_key.identity.node_id, holder_key.identity.procno, + holder_key.identity.request_id); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact( + &probe_ref, 1, master_session, &holder_snapshot, &holder_copy, 1, &blocker, 1, + &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(blocker_snapshot.set_generation, 2); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + PCM_X_QUEUE_OK); +} + +UT_TEST(test_local_first_blocker_ack_busy_then_exact_revoke_consumes_empty_commit) +{ + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalTagSlot *tag_slot; + PcmXRevokePayload revoke; + PcmXTicketRef probe_ref; + uint32 state_flags; + const uint64 master_session = UINT64_C(1810); + + init_active_pcm_x(UINT64_C(77)); + holder_key = make_local_holder_key(7091, 0, 1, UINT64_C(70311), 2); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + bind_local_master(1, holder_key.identity.cluster_epoch, master_session); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&holder_key.identity.tag, &holder_copy, 1, + &holder_snapshot), + PCM_X_QUEUE_OK); + + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(7091, 2, 2, UINT64_C(70312)); + probe_ref.handle.ticket_id = UINT64_C(90111); + probe_ref.handle.queue_generation = UINT64_C(11); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact( + &probe_ref, 1, master_session, &holder_snapshot, &holder_copy, 1, NULL, 0, + &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(blocker_snapshot.set_generation, 1); + UT_ASSERT_EQ(blocker_snapshot.blocker_count, 0); + + /* The master stages this ACK before the same-tag REVOKE. Model a local + * mutator owning ADMISSION_GATE when the ACK handler runs: the frame was + * delivered in FIFO order, but its application is retryably BUSY. */ + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[holder.tag_slot.slot_index]; + state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); + pg_atomic_write_u32(&tag_slot->slot.state_flags, + state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, + blocker_snapshot.set_generation, 1, + master_session), + PCM_X_QUEUE_BUSY); + pg_atomic_write_u32(&tag_slot->slot.state_flags, state_flags); + + memset(&revoke, 0, sizeof(revoke)); + revoke.ref = probe_ref; + revoke.ref.grant_generation = UINT64_C(101); + UT_ASSERT(cluster_pcm_x_image_id_encode(1, UINT64_C(2011), &revoke.image_id)); UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); } + UT_TEST(test_local_blocker_snapshot_exact_lookup_replays_when_pool_is_full) { PcmXShmemHeader *header; @@ -16243,7 +16621,7 @@ UT_TEST(test_local_retire_episode_lock_errors_fail_closed) int main(void) { - UT_PLAN(272); + UT_PLAN(275); UT_RUN(test_image_id_domain_is_canonical_and_bounded); UT_RUN(test_wire_abi_sizes_are_exact); UT_RUN(test_wire_abi_offsets_are_exact); @@ -16363,6 +16741,8 @@ main(void) UT_RUN(test_master_drive_bitmap_replace_rejects_reliable_leg_drift); UT_RUN(test_master_drive_snapshot_fails_closed_on_bitmap_corruption); UT_RUN(test_master_transfer_wire_49_56_is_generation_exact); + UT_RUN(test_master_invalidate_busy_backs_off_exact_revoke_ticket); + UT_RUN(test_master_commit_retry_loses_race_to_terminal_progress_without_fail_closed); UT_RUN(test_master_periodic_drive_replays_post_handoff_commit_x_exactly); UT_RUN(test_master_install_ready_rebase_grant_base_is_one_shot); UT_RUN(test_master_install_ready_rebase_conflict_fails_closed); @@ -16454,6 +16834,7 @@ main(void) UT_RUN(test_local_retire_requires_exact_holder_and_blocker_evidence); UT_RUN(test_local_blocker_allocator_corruption_retains_new_tag_gate_evidence); UT_RUN(test_local_blocker_snapshot_replays_exact_holder_set_and_gates_revoke); + UT_RUN(test_local_first_blocker_ack_busy_then_exact_revoke_consumes_empty_commit); UT_RUN(test_local_blocker_snapshot_exact_lookup_replays_when_pool_is_full); UT_RUN(test_local_blocker_snapshot_accepts_only_holder_bound_nested_waits); UT_RUN(test_local_duplicate_generation_overlap_is_stale_not_fail_closed); diff --git a/src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c b/src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c index db72c591b2..79ca409fc9 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c @@ -151,6 +151,7 @@ UT_TEST(test_fetch_holder_authenticates_requester_master_and_record_generation) PcmXLocalHolderProgress holder; GcsBlockRequestPayload request; ClusterICEnvelope env; + PcmXImageFetchRequestRefusal refusal; UT_ASSERT(cluster_pcm_x_image_fetch_build_request(&requester, 1, 3, &request)); memset(&holder, 0, sizeof(holder)); @@ -167,15 +168,23 @@ UT_TEST(test_fetch_holder_authenticates_requester_master_and_record_generation) env.dest_node_id = 3; env.payload_length = sizeof(request); - UT_ASSERT(cluster_pcm_x_image_fetch_request_exact(&env, &request, &holder, 3, 2, 0)); + UT_ASSERT(cluster_pcm_x_image_fetch_request_exact_diagnosed( + &env, &request, &holder, 3, 2, 0, &refusal)); + UT_ASSERT_EQ((int)refusal, (int)PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_NONE); env.source_node_id = 0; - UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact(&env, &request, &holder, 3, 2, 0)); + UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact_diagnosed( + &env, &request, &holder, 3, 2, 0, &refusal)); + UT_ASSERT_EQ((int)refusal, (int)PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_ENVELOPE); env.source_node_id = 1; holder.image.image_id++; - UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact(&env, &request, &holder, 3, 2, 0)); + UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact_diagnosed( + &env, &request, &holder, 3, 2, 0, &refusal)); + UT_ASSERT_EQ((int)refusal, (int)PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_IMAGE); holder.image = requester.image; holder.master_node = 0; - UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact(&env, &request, &holder, 3, 2, 0)); + UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact_diagnosed( + &env, &request, &holder, 3, 2, 0, &refusal)); + UT_ASSERT_EQ((int)refusal, (int)PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_MASTER); } diff --git a/src/test/cluster_unit/test_cluster_sf_dep.c b/src/test/cluster_unit/test_cluster_sf_dep.c index 886a384d42..3d39718038 100644 --- a/src/test/cluster_unit/test_cluster_sf_dep.c +++ b/src/test/cluster_unit/test_cluster_sf_dep.c @@ -314,6 +314,23 @@ UT_TEST(test_pcm_x_capability_family_sample_is_record_coherent) UT_ASSERT(!rebase); } +UT_TEST(test_pcm_x_source_floor_capability_guard_is_generation_exact) +{ + ClusterSfPeerCap cap; + const uint32 family = PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 + | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1; + + memset(&cap, 0, sizeof(cap)); + cluster_sf_peer_cap_note(&cap, family, 41); + UT_ASSERT(cluster_sf_peer_cap_generation_matches_exact( + &cap, PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1, 41)); + UT_ASSERT(!cluster_sf_peer_cap_generation_matches_exact( + &cap, PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1, 42)); + cluster_sf_peer_cap_note(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, 42); + UT_ASSERT(!cluster_sf_peer_cap_generation_matches_exact( + &cap, PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1, 42)); +} + int main(void) { @@ -329,5 +346,6 @@ main(void) UT_RUN(test_pcm_x_capability_does_not_alias_idle_horizon); UT_RUN(test_pcm_x_capability_generation_snapshot_is_exact); UT_RUN(test_pcm_x_capability_family_sample_is_record_coherent); + UT_RUN(test_pcm_x_source_floor_capability_guard_is_generation_exact); UT_DONE(); } diff --git a/src/test/cluster_unit/test_cluster_visibility_fork.c b/src/test/cluster_unit/test_cluster_visibility_fork.c index 42c632310e..8e70f79054 100644 --- a/src/test/cluster_unit/test_cluster_visibility_fork.c +++ b/src/test/cluster_unit/test_cluster_visibility_fork.c @@ -51,11 +51,13 @@ #include +#include "access/htup_details.h" #include "cluster/cluster_itl_slot.h" #include "cluster/cluster_scn.h" #include "cluster/cluster_tt_slot.h" #include "cluster/cluster_tt_status.h" #include "cluster/cluster_visibility_inject.h" +#include "cluster/cluster_visibility_resolve.h" #include "utils/errcodes.h" #undef printf @@ -74,6 +76,13 @@ UT_DEFINE_GLOBALS(); +#ifndef HEAPAM_SOURCE_PATH +#error "HEAPAM_SOURCE_PATH must identify production heapam.c" +#endif +#ifndef TT_LOCAL_SOURCE_PATH +#error "TT_LOCAL_SOURCE_PATH must identify production cluster_tt_local.c" +#endif + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), @@ -103,6 +112,71 @@ void cluster_visibility_inject_shmem_register(void) {} +static char * +read_source(const char *path) +{ + FILE *fp; + char *source; + long length; + + fp = fopen(path, "rb"); + UT_ASSERT(fp != NULL); + if (fp == NULL) + return NULL; + UT_ASSERT_EQ(fseek(fp, 0, SEEK_END), 0); + length = ftell(fp); + UT_ASSERT(length > 0); + UT_ASSERT_EQ(fseek(fp, 0, SEEK_SET), 0); + source = malloc((size_t) length + 1); + UT_ASSERT(source != NULL); + if (source == NULL) + { + fclose(fp); + return NULL; + } + UT_ASSERT_EQ((long) fread(source, 1, (size_t) length, fp), length); + source[length] = '\0'; + fclose(fp); + return source; +} + +static void +assert_data_active_publish(const char *source, const char *start_marker, + const char *end_marker, const char *uba_name) +{ + const char *start = strstr(source, start_marker); + const char *end = start != NULL ? strstr(start + strlen(start_marker), end_marker) : NULL; + char publish_call[160]; + const char *publish; + const char *crit_end = start != NULL ? strstr(start, "END_CRIT_SECTION();") : NULL; + + snprintf(publish_call, sizeof(publish_call), + "cluster_tt_local_record_data_active(xid, %s);", uba_name); + publish = start != NULL ? strstr(start, publish_call) : NULL; + + if (publish != NULL && end != NULL && publish >= end) + publish = NULL; + + UT_ASSERT(start != NULL); + UT_ASSERT(end != NULL); + UT_ASSERT(publish != NULL); + UT_ASSERT(crit_end != NULL); + if (start == NULL || end == NULL || publish == NULL || crit_end == NULL) + return; + + /* The ACTIVE identity is published only after the tuple + ITL stamp is + * WAL-protected, and before the function can release its heap buffer. */ + while (true) + { + const char *next = strstr(crit_end + 1, "END_CRIT_SECTION();"); + + if (next == NULL || next >= publish) + break; + crit_end = next; + } + UT_ASSERT(crit_end < publish); +} + /* ===== T1: ClusterUndoTTSlotRef size 32B ===== */ UT_TEST(test_t1_undo_tt_slot_ref_sizeof_32) @@ -244,6 +318,68 @@ UT_TEST(test_t12_status_enum_5_values) UT_ASSERT_EQ((int)CLUSTER_TT_STATUS_CLEANED_OUT, 4); } +/* ===== P0-33: every ordinary data-DML producer must publish the exact + * binding as IN_PROGRESS after its ACTIVE ITL stamp, while it still owns the + * buffer content lock. Lock-only already did this; missing data-DML calls + * made a fresh active remote ref miss the overlay and surface 53R97. ===== */ +UT_TEST(test_p033_data_dml_publishes_active_identity) +{ + char *heap_source = read_source(HEAPAM_SOURCE_PATH); + char *tt_source = read_source(TT_LOCAL_SOURCE_PATH); + + if (heap_source == NULL || tt_source == NULL) + { + free(heap_source); + free(tt_source); + return; + } + assert_data_active_publish(heap_source, "\nheap_insert(Relation", + "\nheap_prepare_insert(Relation", "cluster_itl_uba"); + assert_data_active_publish(heap_source, "\nheap_multi_insert(Relation", + "\nsimple_heap_insert(Relation", "cluster_mi_uba"); + assert_data_active_publish(heap_source, "\nheap_delete(Relation", + "\nsimple_heap_delete(Relation", "cluster_itl_uba"); + assert_data_active_publish(heap_source, "\nheap_update(Relation", + "\nsimple_heap_update(Relation", "cluster_itl_uba"); + + /* A real undo-record UBA may live in a different record segment from the + * transaction's canonical TT segment after rollover. The producer must + * therefore remember and publish the exact page-ref alias, and every + * terminal install must converge those aliases to COMMITTED/ABORTED. */ + UT_ASSERT(strstr(tt_source, "cluster_tt_local_record_data_active(TransactionId xid, UBA uba)") + != NULL); + UT_ASSERT(strstr(tt_source, "active_alias_segments") != NULL); + UT_ASSERT(strstr(tt_source, "install_binding_aliases") != NULL); + UT_ASSERT(strstr(tt_source, "install_binding_aliases(binding, status, commit_scn)") != NULL); + + free(heap_source); + free(tt_source); +} + +/* P0-33 safety matrix: a proved remote ACTIVE status is non-terminal and + * follows each consumer's existing truth table. None of the authoritative + * terminal/FROZEN/stale boundaries are widened by the producer fix. */ +UT_TEST(test_p033_active_and_safety_boundary_matrix) +{ + UT_ASSERT_EQ((int) cluster_vis_evidence_route(CLUSTER_VIS_EVIDENCE_REMOTE, false), + (int) CLUSTER_VIS_ROUTE_REMOTE_VERDICT); + UT_ASSERT_EQ((int) cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_IN_PROGRESS), + (int) CVV_INVISIBLE); + UT_ASSERT_EQ((int) cluster_vis_update_xmax_verdict(CLUSTER_TT_STATUS_IN_PROGRESS, false), + (int) CVV_BEING_MODIFIED); + + UT_ASSERT_EQ((int) cluster_vis_xmin_needs_resolution(HEAP_XMIN_FROZEN), 0); + UT_ASSERT_EQ((int) cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_COMMITTED), + (int) CVV_VISIBLE); + UT_ASSERT_EQ((int) cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_ABORTED), + (int) CVV_INVISIBLE); + UT_ASSERT_EQ((int) cluster_vis_evidence_route( + CLUSTER_VIS_EVIDENCE_STALE_OR_AMBIGUOUS, false), + (int) CLUSTER_VIS_ROUTE_FAILCLOSED_UNKNOWN); + UT_ASSERT_EQ((int) cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_UNKNOWN), + (int) CVV_FAILCLOSED_UNKNOWN); +} + int main(void) @@ -260,5 +396,7 @@ main(void) UT_RUN(test_t10_itl_slot_unallocated_sentinel); UT_RUN(test_t11_no_is_xid_local_origin_in_this_unit); UT_RUN(test_t12_status_enum_5_values); + UT_RUN(test_p033_data_dml_publishes_active_identity); + UT_RUN(test_p033_active_and_safety_boundary_matrix); UT_DONE(); } diff --git a/src/test/cluster_unit/test_cluster_visibility_variants.c b/src/test/cluster_unit/test_cluster_visibility_variants.c index 794fb780d4..1a760d4b2a 100644 --- a/src/test/cluster_unit/test_cluster_visibility_variants.c +++ b/src/test/cluster_unit/test_cluster_visibility_variants.c @@ -25,6 +25,7 @@ */ #include "postgres.h" +#include "access/htup_details.h" #include "cluster/cluster_tt_status.h" #include "cluster/cluster_visibility_resolve.h" @@ -43,6 +44,43 @@ static const ClusterTTStatus all_states[] CLUSTER_TT_STATUS_ABORTED, CLUSTER_TT_STATUS_CLEANED_OUT, CLUSTER_TT_STATUS_SUBCOMMITTED }; +/* + * P0-27: VACUUM freeze is an authoritative xmin-committed proof. A frozen + * tuple may retain its historical raw xmin and ITL index after that page slot + * is legally recycled, so resolving the stale xmin identity is both needless + * and wrong. Ordinary COMMITTED/INVALID hints are deliberately NOT widened: + * only the exact FROZEN bit pair bypasses cluster resolution. + */ +UT_TEST(test_update_xmin_frozen_precheck) +{ + UT_ASSERT_EQ((int)cluster_vis_xmin_needs_resolution(HEAP_XMIN_FROZEN), 0); + UT_ASSERT_EQ((int)cluster_vis_xmin_needs_resolution(HEAP_XMIN_FROZEN | HEAP_XMAX_INVALID), 0); + UT_ASSERT_EQ((int)cluster_vis_xmin_needs_resolution(HEAP_XMIN_COMMITTED), 1); + UT_ASSERT_EQ((int)(cluster_vis_xmin_needs_resolution(HEAP_XMIN_COMMITTED) + && cluster_vis_evidence_route(CLUSTER_VIS_EVIDENCE_STALE_OR_AMBIGUOUS, false) + == CLUSTER_VIS_ROUTE_FAILCLOSED_UNKNOWN), + 1); + UT_ASSERT_EQ((int)cluster_vis_xmin_needs_resolution(HEAP_XMIN_INVALID), 1); + UT_ASSERT_EQ((int)cluster_vis_xmin_needs_resolution(0), 1); +} + + +/* + * P0-28: a TID selected on one node remains a legal statement target while + * PCM-X waits for a newer page image. Once an ITL slot is recycled, neither + * xmin hints nor remote-evidence lookup can prove that no peer still names + * that TID. Until a real cluster-wide horizon exists, shared-storage tuples + * must bypass every local prune verdict, including xmin-invalid/unknown forms. + */ +UT_TEST(test_prune_requires_cluster_wide_horizon) +{ + UT_ASSERT_EQ((int)cluster_vis_prune_must_defer(true, false), 1); + UT_ASSERT_EQ((int)cluster_vis_prune_must_defer(true, true), 0); + UT_ASSERT_EQ((int)cluster_vis_prune_must_defer(false, false), 0); + UT_ASSERT_EQ((int)cluster_vis_prune_must_defer(false, true), 0); +} + + /* ---- OBS-4 Self ---- */ UT_TEST(test_obs4_self_full_table) { @@ -237,6 +275,8 @@ UT_TEST(test_evidence_route_full_table) int main(void) { + UT_RUN(test_update_xmin_frozen_precheck); + UT_RUN(test_prune_requires_cluster_wide_horizon); UT_RUN(test_obs4_self_full_table); UT_RUN(test_obs5_toast_full_table); UT_RUN(test_obs2_update_xmin_full_table); From 117a6dca60c0724cc795bb2ef817b1ea9478ea51 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 22 Jul 2026 20:38:08 +0800 Subject: [PATCH 56/56] style(cluster): format PCM-X liveness closure Apply the repository clang-format 18 rules to the PCM-X liveness implementation and unit coverage so the fast-gate validates the published tree. Validation: format gate 580/580 and t/400 228/228 PASS. --- src/backend/cluster/cluster_gcs_block.c | 896 ++++++++---------- src/backend/cluster/cluster_gcs_block_dedup.c | 33 +- src/backend/cluster/cluster_gcs_block_shard.c | 3 +- src/backend/cluster/cluster_lms.c | 33 +- src/backend/cluster/cluster_lms_outbound.c | 42 +- src/backend/cluster/cluster_pcm_lock.c | 15 +- src/backend/cluster/cluster_pcm_own.c | 6 +- src/backend/cluster/cluster_pcm_x_convert.c | 136 +-- .../cluster/cluster_pcm_x_image_fetch.c | 14 +- src/backend/cluster/cluster_sf_dep.c | 7 +- src/backend/cluster/cluster_tt_local.c | 52 +- src/include/cluster/cluster_gcs_block.h | 37 +- src/include/cluster/cluster_gcs_block_dedup.h | 8 +- src/include/cluster/cluster_lms.h | 21 +- src/include/cluster/cluster_pcm_own.h | 20 +- src/include/cluster/cluster_pcm_x_bufmgr.h | 18 +- src/include/cluster/cluster_pcm_x_convert.h | 14 +- src/include/cluster/cluster_sf_dep.h | 6 +- .../test_cluster_bufmgr_pcm_hook.c | 3 +- .../cluster_unit/test_cluster_gcs_block.c | 410 ++++---- .../test_cluster_gcs_block_dedup.c | 77 +- src/test/cluster_unit/test_cluster_ic.c | 3 +- .../cluster_unit/test_cluster_lms_outbound.c | 37 +- .../test_cluster_pcm_direct_init.c | 81 +- src/test/cluster_unit/test_cluster_pcm_lock.c | 78 +- src/test/cluster_unit/test_cluster_pcm_own.c | 184 ++-- .../cluster_unit/test_cluster_pcm_x_convert.c | 136 ++- .../test_cluster_pcm_x_image_fetch.c | 16 +- src/test/cluster_unit/test_cluster_sf_dep.c | 4 +- .../test_cluster_visibility_fork.c | 62 +- 30 files changed, 1155 insertions(+), 1297 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 1e83e03684..2f86529350 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -260,11 +260,11 @@ typedef struct ClusterGcsBlockShared { pg_atomic_uint64 block_x_granted_from_holder_count; /* sender install X_GRANTED_FROM_HOLDER */ pg_atomic_uint64 starvation_denied_pending_x_count; /* N→S short-circuit by pending_x */ /* PGRAC: spec-2.37 D12 — 4 NEW counters for PI watermark + lost-write detection. */ - pg_atomic_uint64 pi_watermark_advance_count; /* X→N/S downgrade caller advance ticks */ - pg_atomic_uint64 pi_watermark_retire_count; /* tag lifecycle + durable-confirm retire */ + pg_atomic_uint64 pi_watermark_advance_count; /* X→N/S downgrade caller advance ticks */ + pg_atomic_uint64 pi_watermark_retire_count; /* tag lifecycle + durable-confirm retire */ pg_atomic_uint64 pi_durable_note_apply_count; /* target accepted status-3 DATA/CONTROL note */ - pg_atomic_uint64 lost_write_detected_count; /* master direct OR holder forward detect */ - pg_atomic_uint64 lost_write_avoid_count; /* durable-confirm retire avoided false-pos */ + pg_atomic_uint64 lost_write_detected_count; /* master direct OR holder forward detect */ + pg_atomic_uint64 lost_write_avoid_count; /* durable-confirm retire avoided false-pos */ /* PGRAC: spec-2.41 D7 — SCN lost-write detector + redo-coverage observability. * Pure counters (no behavior change): the verdict still maps STALE+ANOMALY to * DENIED_LOST_WRITE and bumps lost_write_detected_count; these break that down @@ -905,10 +905,9 @@ gcs_block_release_slot(ClusterGcsBlockOutstandingSlot *slot) "cluster GCS block slot released with live direct target observation: " "backend=%d slot=%d request_id=%llu epoch=%llu rel=%u fork=%d blk=%u " "reply_received=%d direct_state=%d direct_prepared=1", - (int)MyBackendId - 1, released_slot_index, - (unsigned long long)released_request_id, (unsigned long long)released_epoch, - released_tag.relNumber, (int)released_tag.forkNum, released_tag.blockNum, - released_reply_received ? 1 : 0, released_direct_state); + (int)MyBackendId - 1, released_slot_index, (unsigned long long)released_request_id, + (unsigned long long)released_epoch, released_tag.relNumber, (int)released_tag.forkNum, + released_tag.blockNum, released_reply_received ? 1 : 0, released_direct_state); } static uint32 @@ -1004,8 +1003,7 @@ gcs_block_direct_prepare_attempt(ClusterGcsBlockOutstandingSlot *slot, BufferDes slot_idx = gcs_block_slot_index(blk, slot); LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); - if (!slot->in_use || slot->reply_received - || slot->direct_state != GCS_BLOCK_DIRECT_UNARMED + if (!slot->in_use || slot->reply_received || slot->direct_state != GCS_BLOCK_DIRECT_UNARMED || slot->direct_target_prepared) { LWLockRelease(&blk->lock.lock); gcs_block_direct_finish_target(buf, true, false, InvalidXLogRecPtr); @@ -1105,21 +1103,22 @@ gcs_block_log_master_not_holder_producer(const char *reason, BufferTag tag, uint " transition_count=" UINT64_FORMAT, reason != NULL ? reason : "unknown", cluster_node_id, requester_node, request_id, request_epoch, (unsigned int)transition_id, tag.spcOid, tag.dbOid, - (unsigned int)BufTagGetRelNumber(&tag), (int)tag.forkNum, - (unsigned int)tag.blockNum, (int)authority_found, (unsigned int)authority.state, - authority.x_holder_node, (unsigned int)authority.s_holders_bitmap, - authority.master_holder.node_id, authority.master_holder.procno, - authority.master_holder.cluster_epoch, authority.master_holder.request_id, - authority.pending_x_requester_node, authority.pending_x_since_lsn, - authority.transition_count))); -} - -#define GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, reason) \ - gcs_block_log_master_not_holder_producer((reason), (req)->tag, (req)->request_id, (req)->epoch, \ - (req)->sender_node, (req)->transition_id) -#define GCS_BLOCK_LOG_MASTER_NOT_HOLDER_FORWARD(fwd, reason) \ - gcs_block_log_master_not_holder_producer((reason), (fwd)->tag, (fwd)->request_id, (fwd)->epoch, \ - (fwd)->original_requester_node, (fwd)->transition_id) + (unsigned int)BufTagGetRelNumber(&tag), (int)tag.forkNum, (unsigned int)tag.blockNum, + (int)authority_found, (unsigned int)authority.state, authority.x_holder_node, + (unsigned int)authority.s_holders_bitmap, authority.master_holder.node_id, + authority.master_holder.procno, authority.master_holder.cluster_epoch, + authority.master_holder.request_id, authority.pending_x_requester_node, + authority.pending_x_since_lsn, authority.transition_count))); +} + +#define GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, reason) \ + gcs_block_log_master_not_holder_producer((reason), (req)->tag, (req)->request_id, \ + (req)->epoch, (req)->sender_node, \ + (req)->transition_id) +#define GCS_BLOCK_LOG_MASTER_NOT_HOLDER_FORWARD(fwd, reason) \ + gcs_block_log_master_not_holder_producer((reason), (fwd)->tag, (fwd)->request_id, \ + (fwd)->epoch, (fwd)->original_requester_node, \ + (fwd)->transition_id) static void gcs_block_release_ship_image(ClusterICSgeReleaseCallback release_cb, void *release_arg) @@ -1264,8 +1263,7 @@ gcs_block_get_ship_image(BufferTag tag, int32 dest_node, bool allow_live_sge, XLogRecPtr *out_page_lsn, char *copy_buf, const char **out_block_payload, uint32 *out_block_lkey, ClusterICSgeReleaseCallback *out_release_cb, void **out_release_arg, ClusterSfDepVec *out_sf_dep_vec, - bool *out_sf_dep_valid, - ClusterBufmgrGcsCopyRefusal *out_copy_refusal) + bool *out_sf_dep_valid, ClusterBufmgrGcsCopyRefusal *out_copy_refusal) { void *scratch = NULL; uint32 scratch_lkey = 0; @@ -1327,11 +1325,10 @@ gcs_block_get_ship_image(BufferTag tag, int32 dest_node, bool allow_live_sge, tag, out_page_lsn, (char *)scratch, out_sf_dep_vec); else copied = cluster_bufmgr_copy_block_for_gcs(tag, out_page_lsn, (char *)scratch, - out_copy_refusal); + out_copy_refusal); if (!copied) { if (smart_fusion_reply && out_copy_refusal != NULL) - *out_copy_refusal - = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_SMART_FUSION_UNCLASSIFIED; + *out_copy_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_SMART_FUSION_UNCLASSIFIED; if (release_cb != NULL) release_cb(release_arg); return false; @@ -1354,17 +1351,15 @@ gcs_block_get_ship_image(BufferTag tag, int32 dest_node, bool allow_live_sge, if (smart_fusion_reply) { if (!cluster_bufmgr_copy_block_for_gcs_smart_fusion(tag, out_page_lsn, copy_buf, - out_sf_dep_vec)) { + out_sf_dep_vec)) { if (out_copy_refusal != NULL) - *out_copy_refusal - = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_SMART_FUSION_UNCLASSIFIED; + *out_copy_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_SMART_FUSION_UNCLASSIFIED; return false; } if (out_sf_dep_valid != NULL) *out_sf_dep_valid = true; gcs_block_note_scratch_copy(); - } else if (!cluster_bufmgr_copy_block_for_gcs(tag, out_page_lsn, copy_buf, - out_copy_refusal)) + } else if (!cluster_bufmgr_copy_block_for_gcs(tag, out_page_lsn, copy_buf, out_copy_refusal)) return false; else gcs_block_note_scratch_copy(); @@ -2050,9 +2045,8 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans Assert(out_retry_denied != NULL); if (out_retry_denied == NULL) - ereport(ERROR, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("cluster_gcs_send_block_request_and_wait: NULL retry result"))); + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cluster_gcs_send_block_request_and_wait: NULL retry result"))); *out_retry_denied = false; if (buf == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), @@ -3349,29 +3343,29 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle || !BufferTagsEqual(&leader->identity.tag, &reservation_base->tag) || reservation_base->generation != (progress_before.grant_base_own_generation != 0 - ? progress_before.grant_base_own_generation - : leader->identity.base_own_generation)) { + ? progress_before.grant_base_own_generation + : leader->identity.base_own_generation)) { if (diagnostic) - ereport(LOG, - (errmsg_internal("PCM-X image fetch requester boundary: pre-slot reject"), - errdetail("branch=identity-base buf_tag_exact=%s leader_tag_exact=%s " - "base_generation=%llu progress_identity_base=%llu " - "progress_grant_base=%llu effective_grant_base=%llu " - "base_reservation_token=%llu base_writer_activation_token=%llu " - "base_flags=0x%x base_state=%u", - BufferTagsEqual(&buf->tag, &reservation_base->tag) ? "true" : "false", - BufferTagsEqual(&leader->identity.tag, &reservation_base->tag) - ? "true" - : "false", - (unsigned long long)reservation_base->generation, - (unsigned long long)progress_before.identity.base_own_generation, - (unsigned long long)progress_before.grant_base_own_generation, - (unsigned long long)(progress_before.grant_base_own_generation != 0 - ? progress_before.grant_base_own_generation - : leader->identity.base_own_generation), - (unsigned long long)reservation_base->reservation_token, - (unsigned long long)reservation_base->writer_activation_token, - reservation_base->flags, (unsigned)reservation_base->pcm_state))); + ereport( + LOG, + (errmsg_internal("PCM-X image fetch requester boundary: pre-slot reject"), + errdetail("branch=identity-base buf_tag_exact=%s leader_tag_exact=%s " + "base_generation=%llu progress_identity_base=%llu " + "progress_grant_base=%llu effective_grant_base=%llu " + "base_reservation_token=%llu base_writer_activation_token=%llu " + "base_flags=0x%x base_state=%u", + BufferTagsEqual(&buf->tag, &reservation_base->tag) ? "true" : "false", + BufferTagsEqual(&leader->identity.tag, &reservation_base->tag) ? "true" + : "false", + (unsigned long long)reservation_base->generation, + (unsigned long long)progress_before.identity.base_own_generation, + (unsigned long long)progress_before.grant_base_own_generation, + (unsigned long long)(progress_before.grant_base_own_generation != 0 + ? progress_before.grant_base_own_generation + : leader->identity.base_own_generation), + (unsigned long long)reservation_base->reservation_token, + (unsigned long long)reservation_base->writer_activation_token, + reservation_base->flags, (unsigned)reservation_base->pcm_state))); return PCM_X_QUEUE_STALE; } if (!cluster_pcm_x_image_fetch_build_request(&progress_before, cluster_node_id, @@ -3384,29 +3378,29 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle if (queue_result != PCM_X_QUEUE_OK) return queue_result; if (!cluster_pcm_x_image_fetch_reservation_exact(&live_own, reservation_base, - reservation_token)) { + reservation_token)) { queue_result = gcs_block_pcm_x_fetch_reservation_mismatch(&live_own); if (diagnostic) ereport(LOG, (errmsg_internal("PCM-X image fetch requester boundary: pre-slot reject"), - errdetail("branch=reservation-live result=%d live_tag_exact=%s " - "base_generation=%llu live_generation=%llu " - "base_reservation_token=%llu reservation_token=%llu " - "live_reservation_token=%llu base_writer_activation_token=%llu " - "live_writer_activation_token=%llu base_flags=0x%x live_flags=0x%x " - "base_state=%u live_state=%u", - (int)queue_result, - BufferTagsEqual(&live_own.tag, &reservation_base->tag) ? "true" - : "false", - (unsigned long long)reservation_base->generation, - (unsigned long long)live_own.generation, - (unsigned long long)reservation_base->reservation_token, - (unsigned long long)reservation_token, - (unsigned long long)live_own.reservation_token, - (unsigned long long)reservation_base->writer_activation_token, - (unsigned long long)live_own.writer_activation_token, - reservation_base->flags, live_own.flags, - (unsigned)reservation_base->pcm_state, (unsigned)live_own.pcm_state))); + errdetail( + "branch=reservation-live result=%d live_tag_exact=%s " + "base_generation=%llu live_generation=%llu " + "base_reservation_token=%llu reservation_token=%llu " + "live_reservation_token=%llu base_writer_activation_token=%llu " + "live_writer_activation_token=%llu base_flags=0x%x live_flags=0x%x " + "base_state=%u live_state=%u", + (int)queue_result, + BufferTagsEqual(&live_own.tag, &reservation_base->tag) ? "true" : "false", + (unsigned long long)reservation_base->generation, + (unsigned long long)live_own.generation, + (unsigned long long)reservation_base->reservation_token, + (unsigned long long)reservation_token, + (unsigned long long)live_own.reservation_token, + (unsigned long long)reservation_base->writer_activation_token, + (unsigned long long)live_own.writer_activation_token, + reservation_base->flags, live_own.flags, + (unsigned)reservation_base->pcm_state, (unsigned)live_own.pcm_state))); return queue_result; } @@ -3516,8 +3510,7 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle retry_attempt, enqueue_result ? "true" : "false", progress_now.image.source_node, (unsigned long long)request.request_id, - request.requester_backend_id, - (unsigned)request.transition_id))); + request.requester_backend_id, (unsigned)request.transition_id))); if (!enqueue_result) { ConditionVariableCancelSleep(); if (retry_attempt < max_retries) @@ -3565,8 +3558,7 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle errdetail("attempt=%d got_reply=%s slot_stale=%s fence_lost=%s " "status=%d request_id=%llu", retry_attempt, got_reply ? "true" : "false", - slot_stale ? "true" : "false", - fence_lost ? "true" : "false", + slot_stale ? "true" : "false", fence_lost ? "true" : "false", got_reply ? (int)slot->reply_header.status : -1, (unsigned long long)request.request_id))); if (fence_lost) { @@ -3670,12 +3662,11 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle gcs_block_release_slot(slot); if (diagnostic) - ereport(LOG, - (errmsg_internal("PCM-X image fetch requester boundary: complete"), - errdetail("result=%d requester=%d backend=%d source=%u request_id=%llu", - (int)result, cluster_node_id, (int)MyBackendId, - progress_before.image.source_node, - (unsigned long long)request.request_id))); + ereport(LOG, (errmsg_internal("PCM-X image fetch requester boundary: complete"), + errdetail("result=%d requester=%d backend=%d source=%u request_id=%llu", + (int)result, cluster_node_id, (int)MyBackendId, + progress_before.image.source_node, + (unsigned long long)request.request_id))); return (PcmXQueueResult)result; } @@ -3704,8 +3695,7 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle * be obtained — never a silent stale read (Rule 8.A). */ bool -cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, - const PcmAuthoritySnapshot *expected, +cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, const PcmAuthoritySnapshot *expected, bool *out_retry_denied) { ClusterGcsBlockOutstandingSlot *slot; @@ -5202,7 +5192,7 @@ gcs_block_forward_send_admitted(int32 holder_node, const GcsBlockForwardPayload * the normal registered handler just like a wire reply. */ static ClusterICSendResult gcs_block_send_envelope_or_loopback(uint8 msg_type, int32 dest_node, const void *payload, - uint32 payload_len) + uint32 payload_len) { ClusterICEnvelope envelope; @@ -5210,7 +5200,7 @@ gcs_block_send_envelope_or_loopback(uint8 msg_type, int32 dest_node, const void return cluster_ic_send_envelope(msg_type, dest_node, payload, payload_len); if (payload == NULL || !cluster_ic_envelope_build(&envelope, msg_type, (uint32)cluster_node_id, - (uint32)cluster_node_id, payload, payload_len)) + (uint32)cluster_node_id, payload, payload_len)) return CLUSTER_IC_SEND_HARD_ERROR; return cluster_ic_dispatch_envelope(&envelope, payload, cluster_node_id) ? CLUSTER_IC_SEND_DONE @@ -5275,7 +5265,7 @@ gcs_block_send_reply(int32 dest_node, const GcsBlockRequestPayload *req, GcsBloc *wire_hdr = hdr; wire_hdr->checksum = gcs_block_compute_checksum(buf + sizeof(GcsBlockReplyHeader)); rc = gcs_block_send_envelope_or_loopback(PGRAC_IC_MSG_GCS_BLOCK_REPLY, dest_node, buf, - total); + total); pfree(buf); } @@ -5384,7 +5374,7 @@ gcs_block_resend_cached_reply(int32 dest_node, const GcsBlockDedupEntry *entry) "cached-direct-land-nonsendable", entry->tag, entry->key.request_id, entry->key.cluster_epoch, (int32)entry->key.origin_node_id, entry->transition_id); (void)gcs_block_try_send_direct_reply(dest_node, true, hdr, - has_block_payload ? block_data : NULL, 0, NULL, NULL); + has_block_payload ? block_data : NULL, 0, NULL, NULL); pfree(buf); return true; } @@ -5442,13 +5432,12 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe current_epoch = cluster_epoch_get_current(); tag_master = cluster_gcs_lookup_master(req->tag); if (diagnostic) - ereport(LOG, - (errmsg_internal("PCM-X image fetch source boundary: ingress"), - errdetail("worker=%d source=%u dest=%u requester=%d backend=%d " - "request_id=%llu epoch=%llu encoded_master=%d tag_master=%d", - worker_id, env->source_node_id, env->dest_node_id, req->sender_node, - req->requester_backend_id, (unsigned long long)req->request_id, - (unsigned long long)req->epoch, encoded_master, tag_master))); + ereport(LOG, (errmsg_internal("PCM-X image fetch source boundary: ingress"), + errdetail("worker=%d source=%u dest=%u requester=%d backend=%d " + "request_id=%llu epoch=%llu encoded_master=%d tag_master=%d", + worker_id, env->source_node_id, env->dest_node_id, req->sender_node, + req->requester_backend_id, (unsigned long long)req->request_id, + (unsigned long long)req->epoch, encoded_master, tag_master))); if (encoded_master != tag_master || req->epoch != current_epoch || !gcs_block_pcm_x_source_capable(req->sender_node) || !cluster_qvotec_in_quorum() || !cluster_membership_is_member(cluster_node_id) @@ -5457,14 +5446,14 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe || !gcs_block_pcm_x_authenticated_session(req->sender_node, current_epoch, &requester_session) || !gcs_block_pcm_x_revalidate_peer_binding(req->sender_node, current_epoch, - requester_session)) + requester_session)) return true; if (diagnostic) ereport(LOG, (errmsg_internal("PCM-X image fetch source boundary: validate"), errdetail("stage=auth requester_session=%llu current_epoch=%llu tag_master=%d", - (unsigned long long)requester_session, - (unsigned long long)current_epoch, tag_master))); + (unsigned long long)requester_session, (unsigned long long)current_epoch, + tag_master))); progress_result = cluster_pcm_x_local_holder_progress_exact(&req->tag, &holder_before); if (progress_result == PCM_X_QUEUE_OK) @@ -5472,8 +5461,8 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe env, req, &holder_before, cluster_node_id, tag_master, current_epoch, &request_refusal); else request_refusal = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_ARGUMENT; - (void)cluster_gcs_requester_id_decode(holder_before.ref.identity.request_id, - &decoded_requester, &decoded_backend, NULL); + (void)cluster_gcs_requester_id_decode(holder_before.ref.identity.request_id, &decoded_requester, + &decoded_backend, NULL); if (request_exact) master_authenticated = gcs_block_pcm_x_authenticated_session(tag_master, current_epoch, &master_session); @@ -5493,16 +5482,15 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe "ticket=%llu queue_generation=%llu grant_generation=%llu " "image_source=%u holder_request_id=%llu holder_image_id=%llu", (int)progress_result, request_exact ? "true" : "false", - (int)request_refusal, - master_authenticated ? "true" : "false", + (int)request_refusal, master_authenticated ? "true" : "false", master_binding_valid ? "true" : "false", (unsigned long long)(master_authenticated ? master_session : 0), (unsigned long long)holder_before.master_session_incarnation, holder_before.master_node, holder_before.ref.identity.node_id, - (unsigned)holder_before.pending_opcode, - (unsigned)holder_before.phase, (unsigned)holder_before.flags, - (unsigned)env->payload_length, sizeof(*req), req->requester_backend_id, - decoded_backend, decoded_requester, (unsigned)req->transition_id, + (unsigned)holder_before.pending_opcode, (unsigned)holder_before.phase, + (unsigned)holder_before.flags, (unsigned)env->payload_length, + sizeof(*req), req->requester_backend_id, decoded_backend, + decoded_requester, (unsigned)req->transition_id, (unsigned long long)holder_before.ref.identity.cluster_epoch, (unsigned long long)holder_before.ref.identity.wait_seq, (unsigned long long)holder_before.ref.handle.ticket_id, @@ -5529,15 +5517,14 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe image_result = cluster_gcs_block_dedup_pcm_x_lookup(worker_id, &key, &req->tag, &binding, &cached); if (diagnostic) - ereport(LOG, - (errmsg_internal("PCM-X image fetch source boundary: lookup"), - errdetail("result=%d worker=%d requester=%u backend=%d request_id=%llu " - "ticket=%llu grant_generation=%llu image_id=%llu", - (int)image_result, worker_id, key.origin_node_id, - key.requester_backend_id, (unsigned long long)key.request_id, - (unsigned long long)binding.identity.ref.handle.ticket_id, - (unsigned long long)binding.identity.ref.grant_generation, - (unsigned long long)binding.identity.image.image_id))); + ereport(LOG, (errmsg_internal("PCM-X image fetch source boundary: lookup"), + errdetail("result=%d worker=%d requester=%u backend=%d request_id=%llu " + "ticket=%llu grant_generation=%llu image_id=%llu", + (int)image_result, worker_id, key.origin_node_id, + key.requester_backend_id, (unsigned long long)key.request_id, + (unsigned long long)binding.identity.ref.handle.ticket_id, + (unsigned long long)binding.identity.ref.grant_generation, + (unsigned long long)binding.identity.image.image_id))); if (image_result == GCS_BLOCK_PCM_X_IMAGE_NOT_READY) { /* Materialization/publication is still progressing on this same shard. */ GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "pcm-x-image-not-ready"); @@ -5563,12 +5550,11 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe reply_result = gcs_block_resend_cached_reply(req->sender_node, &cached); if (diagnostic) - ereport(LOG, - (errmsg_internal("PCM-X image fetch source boundary: reply"), - errdetail("stage_result=%s requester=%d backend=%d request_id=%llu status=%u", - reply_result ? "true" : "false", req->sender_node, - req->requester_backend_id, (unsigned long long)req->request_id, - (unsigned)cached.status))); + ereport(LOG, (errmsg_internal("PCM-X image fetch source boundary: reply"), + errdetail("stage_result=%s requester=%d backend=%d request_id=%llu status=%u", + reply_result ? "true" : "false", req->sender_node, + req->requester_backend_id, (unsigned long long)req->request_id, + (unsigned)cached.status))); if (ClusterGcsBlock != NULL) { pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_reply_count, 1); pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_ship_bytes_total, GCS_BLOCK_DATA_SIZE); @@ -5599,8 +5585,7 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe * dedup install should happen. */ static bool -gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, - bool preprepared_image, +gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, bool preprepared_image, GcsBlockReplyStatus *out_status, XLogRecPtr *out_page_lsn, const char **out_block_payload, uint32 *out_block_lkey, ClusterICSgeReleaseCallback *out_release_cb, void **out_release_arg, @@ -5686,8 +5671,8 @@ gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, apply_result = cluster_pcm_lock_apply_gcs_transition_result( req->tag, (PcmLockTransition)req->transition_id, req->sender_node); if (apply_result != PCM_GCS_TRANSITION_APPLIED) { - *out_status = GcsBlockApplyRefusalStatus( - apply_result, (PcmLockTransition)req->transition_id); + *out_status + = GcsBlockApplyRefusalStatus(apply_result, (PcmLockTransition)req->transition_id); return true; } pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_storage_fallback_count, 1); @@ -5732,7 +5717,8 @@ gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, if (!preprepared_image && !gcs_block_get_ship_image(req->tag, req->sender_node, true, out_page_lsn, block_buf, out_block_payload, out_block_lkey, out_release_cb, - out_release_arg, out_sf_dep_vec, out_sf_dep_valid, ©_refusal)) { + out_release_arg, out_sf_dep_vec, out_sf_dep_valid, + ©_refusal)) { char producer_reason[96]; snprintf(producer_reason, sizeof(producer_reason), "produce-copy-refused:%s", @@ -5755,8 +5741,8 @@ gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, *out_release_cb = NULL; if (out_release_arg != NULL) *out_release_arg = NULL; - *out_status = GcsBlockApplyRefusalStatus( - apply_result, (PcmLockTransition)req->transition_id); + *out_status + = GcsBlockApplyRefusalStatus(apply_result, (PcmLockTransition)req->transition_id); return true; } @@ -5925,14 +5911,13 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo if (dr != GCS_BLOCK_DEDUP_VALIDATION_FAIL && dr != GCS_BLOCK_DEDUP_FULL) { if (GcsBlockRequestPayloadIsDirectLandArmed(req)) request_flags |= GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND; - if (!cluster_gcs_block_dedup_set_request_flags_exact( - dedup_worker_id, &key, &req->tag, req->transition_id, request_flags)) { + if (!cluster_gcs_block_dedup_set_request_flags_exact(dedup_worker_id, &key, &req->tag, + req->transition_id, request_flags)) { /* The tuple was removed/replaced or its immutable request properties * changed between lookup and pinning. Neither case may inherit the * earlier entry's grant rights. */ - gcs_block_send_reply(req->sender_node, req, - GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, InvalidXLogRecPtr, - NULL); + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, + InvalidXLogRecPtr, NULL); return; } } @@ -5946,9 +5931,8 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo GcsBlockPendingXDenyResult deny_result; if (dr == GCS_BLOCK_DEDUP_VALIDATION_FAIL) { - gcs_block_send_reply(req->sender_node, req, - GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, InvalidXLogRecPtr, - NULL); + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, + InvalidXLogRecPtr, NULL); return; } if (dr == GCS_BLOCK_DEDUP_FULL) { @@ -5960,9 +5944,8 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo dedup_worker_id, &key, &req->tag, req->transition_id, &cached_entry); if (deny_result != GCS_BLOCK_PENDING_X_DENY_NEW && deny_result != GCS_BLOCK_PENDING_X_DENY_REPLAY) { - gcs_block_send_reply(req->sender_node, req, - GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, InvalidXLogRecPtr, - NULL); + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, + InvalidXLogRecPtr, NULL); return; } pg_atomic_fetch_add_u64(&ClusterGcsBlock->starvation_denied_pending_x_count, 1); @@ -6087,8 +6070,7 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo || deny_result == GCS_BLOCK_PENDING_X_DENY_REPLAY) (void)gcs_block_resend_cached_reply(req->sender_node, &cached_entry); else - gcs_block_send_reply(req->sender_node, req, - GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, InvalidXLogRecPtr, NULL); return; } @@ -6104,8 +6086,7 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo || deny_result == GCS_BLOCK_PENDING_X_DENY_REPLAY) (void)gcs_block_resend_cached_reply(req->sender_node, &cached_entry); else - gcs_block_send_reply(req->sender_node, req, - GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, InvalidXLogRecPtr, NULL); return; } @@ -6866,11 +6847,10 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo * state raced / flush unavailable) keeps today's one-shot ship. */ if (cluster_read_scache) { - local_downgrade_outcome - = cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image( - req->tag, &page_lsn, block_buf, NULL); - cluster_lever_a_note_downgrade( - local_downgrade_outcome == CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED); + local_downgrade_outcome = cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image( + req->tag, &page_lsn, block_buf, NULL); + cluster_lever_a_note_downgrade(local_downgrade_outcome + == CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED); if (local_downgrade_outcome == CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY) return; @@ -7041,8 +7021,7 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo * serves a durable GRANTED (image + requester N->S quick re-grant). */ scache_downgraded_fall_through: (void)gcs_block_produce_reply(req, block_buf, scache_image_prepared, &status, &page_lsn, - &block_payload, - &block_payload_lkey, &block_payload_release_cb, + &block_payload, &block_payload_lkey, &block_payload_release_cb, &block_payload_release_arg, &sf_dep_vec, &sf_dep_valid); if (req->transition_id == PCM_TRANS_N_TO_X || req->transition_id == PCM_TRANS_S_TO_X_UPGRADE) (void)cluster_pcm_lock_clear_pending_x_if(req->tag, req->sender_node); @@ -7570,10 +7549,9 @@ gcs_block_direct_fail_slot(ClusterGcsBlockBackendBlock *blk, ClusterGcsBlockOuts "own_result=%d buffer=%d own_state=%u own_generation=%llu own_token=%llu " "own_flags=0x%x", backend_idx, slot_idx, (unsigned long long)request_id, - (unsigned long long)request_epoch, target_tag.relNumber, - (int)target_tag.forkNum, target_tag.blockNum, (int)reason, - authoritative_denial ? 1 : 0, reply_received ? 1 : 0, direct_state, - prepared ? 1 : 0, prepared ? 1 : 0, (int)own_result, buffer_id, + (unsigned long long)request_epoch, target_tag.relNumber, (int)target_tag.forkNum, + target_tag.blockNum, (int)reason, authoritative_denial ? 1 : 0, reply_received ? 1 : 0, + direct_state, prepared ? 1 : 0, prepared ? 1 : 0, (int)own_result, buffer_id, own.pcm_state, (unsigned long long)own.generation, (unsigned long long)own.reservation_token, own.flags); } @@ -8211,11 +8189,9 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo CLUSTER_INJECTION_POINT("cluster-gcs-block-remote-downgrade"); if (!holder_evicted_injected && !cluster_injection_should_skip("cluster-gcs-block-remote-downgrade")) - remote_downgrade_outcome - = cluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image( - fwd->tag, fwd->master_node, &page_lsn, block_buf, ©_refusal); - remote_downgraded - = remote_downgrade_outcome == CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED; + remote_downgrade_outcome = cluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image( + fwd->tag, fwd->master_node, &page_lsn, block_buf, ©_refusal); + remote_downgraded = remote_downgrade_outcome == CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED; cluster_lever_a_note_remote_downgrade(remote_downgraded); } if (remote_downgrade_outcome == CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY) @@ -8528,14 +8504,13 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo hdr->status = (uint8)GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_forward_holder_evicted_count, 1); GCS_BLOCK_LOG_MASTER_NOT_HOLDER_FORWARD(fwd, "holder-copy-refused"); - ereport(LOG, - (errmsg("cluster_gcs_block: holder ship image refused"), - errdetail("reason=%s request_id=" UINT64_FORMAT - " requester=%d master=%d tag spc=%u db=%u relNumber=%u block=%u", - copy_refusal_name, fwd->request_id, fwd->original_requester_node, - fwd->master_node, fwd->tag.spcOid, fwd->tag.dbOid, - (unsigned int)BufTagGetRelNumber(&fwd->tag), - (unsigned int)fwd->tag.blockNum))); + ereport(LOG, (errmsg("cluster_gcs_block: holder ship image refused"), + errdetail("reason=%s request_id=" UINT64_FORMAT + " requester=%d master=%d tag spc=%u db=%u relNumber=%u block=%u", + copy_refusal_name, fwd->request_id, fwd->original_requester_node, + fwd->master_node, fwd->tag.spcOid, fwd->tag.dbOid, + (unsigned int)BufTagGetRelNumber(&fwd->tag), + (unsigned int)fwd->tag.blockNum))); } send_sf_dep = sf_peer_v2 && sf_dep_valid && block_payload != NULL @@ -8825,9 +8800,9 @@ gcs_block_pcm_x_requester_pre_sleep_revalidate(void *callback_arg) GcsBlockPcmXRequesterWaitContext *context = (GcsBlockPcmXRequesterWaitContext *)callback_arg; if (context == NULL - || !gcs_block_pcm_x_requester_authority_exact( - context->request_runtime, context->master_node, context->cluster_epoch, - context->master_session)) + || !gcs_block_pcm_x_requester_authority_exact(context->request_runtime, + context->master_node, context->cluster_epoch, + context->master_session)) return PCM_X_QUEUE_NOT_READY; return cluster_pcm_x_nested_wait_guard_before_block(); } @@ -8867,9 +8842,9 @@ gcs_block_pcm_x_requester_wait_exact(uint32 *wait_index, const PcmXRuntimeSnapsh context.master_session = master_session; context.timeout_ms = cluster_pcm_x_holder_retry_delay_ms(current); CHECK_FOR_INTERRUPTS(); - result = cluster_pcm_x_requester_wait_once_result( - gcs_block_pcm_x_requester_pre_sleep_revalidate, gcs_block_pcm_x_requester_wait_latch, - &context); + result + = cluster_pcm_x_requester_wait_once_result(gcs_block_pcm_x_requester_pre_sleep_revalidate, + gcs_block_pcm_x_requester_wait_latch, &context); if (result != PCM_X_QUEUE_OK) return result; ResetLatch(MyLatch); @@ -9246,7 +9221,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (!cluster_gcs_pcm_x_nested_guard_retryable(result)) GCS_BLOCK_PCM_X_REQUESTER_DONE(); result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session); + cluster_epoch, master_session); if (result != PCM_X_QUEUE_OK) { if (result == PCM_X_QUEUE_CORRUPT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); @@ -9275,8 +9250,8 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (result == PCM_X_QUEUE_CORRUPT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); if (initial_own.pcm_state == (uint8)PCM_STATE_READ_IMAGE || result == PCM_X_QUEUE_BUSY) { - result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session); + result = gcs_block_pcm_x_requester_wait_exact( + &wait_index, &request_runtime, master_node, cluster_epoch, master_session); if (result != PCM_X_QUEUE_OK) { if (result == PCM_X_QUEUE_CORRUPT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); @@ -9316,7 +9291,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session); + cluster_epoch, master_session); if (result != PCM_X_QUEUE_OK) { if (result == PCM_X_QUEUE_CORRUPT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); @@ -9342,8 +9317,8 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim GCS_BLOCK_PCM_X_RETRY_SITE_LEADER_REKEY, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); - result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session); + result = gcs_block_pcm_x_requester_wait_exact( + &wait_index, &request_runtime, master_node, cluster_epoch, master_session); if (result != PCM_X_QUEUE_OK) { if (result == PCM_X_QUEUE_CORRUPT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); @@ -9363,8 +9338,8 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim GCS_BLOCK_PCM_X_RETRY_SITE_CLAIM, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); - result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session); + result = gcs_block_pcm_x_requester_wait_exact( + &wait_index, &request_runtime, master_node, cluster_epoch, master_session); if (result != PCM_X_QUEUE_OK) { if (result == PCM_X_QUEUE_CORRUPT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); @@ -9385,8 +9360,8 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); gcs_block_pcm_x_requester_cleanup_context.cutoff_started = false; - result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session); + result = gcs_block_pcm_x_requester_wait_exact( + &wait_index, &request_runtime, master_node, cluster_epoch, master_session); if (result != PCM_X_QUEUE_OK) { if (result == PCM_X_QUEUE_CORRUPT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); @@ -9519,8 +9494,8 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } - result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session); + result = gcs_block_pcm_x_requester_wait_exact( + &wait_index, &request_runtime, master_node, cluster_epoch, master_session); if (result != PCM_X_QUEUE_OK) { if (result == PCM_X_QUEUE_CORRUPT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); @@ -9662,8 +9637,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim evidence->pcm_state = reservation_base.pcm_state; evidence->own_flags = reservation_base.flags; evidence->live_generation = reservation_base.generation; - evidence->base_own_generation - = progress.identity.base_own_generation; + evidence->base_own_generation = progress.identity.base_own_generation; evidence->reservation_token = reservation_base.reservation_token; evidence->writer_activation_token = reservation_base.writer_activation_token; @@ -9801,7 +9775,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim requester_wait: result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session); + cluster_epoch, master_session); if (result != PCM_X_QUEUE_OK) { if (result == PCM_X_QUEUE_CORRUPT) GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); @@ -9832,8 +9806,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim (unsigned long long)gcs_block_pcm_x_requester_preflight_evidence.live_generation, (unsigned long long) gcs_block_pcm_x_requester_preflight_evidence.base_own_generation, - (unsigned long long) - gcs_block_pcm_x_requester_preflight_evidence.reservation_token, + (unsigned long long)gcs_block_pcm_x_requester_preflight_evidence.reservation_token, (unsigned long long) gcs_block_pcm_x_requester_preflight_evidence.writer_activation_token, (unsigned int)gcs_block_pcm_x_requester_preflight_evidence.pcm_state, @@ -10030,9 +10003,9 @@ gcs_block_pcm_x_collect_formation(PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_L static void gcs_block_pcm_x_master_drive_retry_tick(void); static void gcs_block_pcm_x_terminal_retry_tick(void); static void gcs_block_pcm_x_master_retry_observe(const char *stage, PcmXQueueResult result, - int32 peer_node, Size cursor_before, - Size cursor_after, const BufferTag *tag, - uint64 cluster_epoch); + int32 peer_node, Size cursor_before, + Size cursor_after, const BufferTag *tag, + uint64 cluster_epoch); static PcmXQueueResult gcs_block_pcm_x_cancel_claimed_probe_exact(const PcmXMasterPendingXReleaseToken *token); @@ -10066,20 +10039,20 @@ cluster_gcs_block_pcm_x_formation_tick(void) */ if (!gcs_block_pcm_x_collect_formation(bindings_before, &epoch_before, &self_session_before, NULL)) { - gcs_block_pcm_x_master_retry_observe("collect-before", PCM_X_QUEUE_NOT_READY, -1, 0, - 0, NULL, 0); + gcs_block_pcm_x_master_retry_observe("collect-before", PCM_X_QUEUE_NOT_READY, -1, 0, 0, + NULL, 0); return; } if (!gcs_block_pcm_x_collect_formation(bindings_after, &epoch_after, &self_session_after, NULL)) { - gcs_block_pcm_x_master_retry_observe("collect-after", PCM_X_QUEUE_NOT_READY, -1, 0, - 0, NULL, 0); + gcs_block_pcm_x_master_retry_observe("collect-after", PCM_X_QUEUE_NOT_READY, -1, 0, 0, + NULL, 0); return; } if (!cluster_gcs_pcm_x_formation_samples_stable(true, bindings_before, true, bindings_after) || epoch_before != epoch_after || self_session_before != self_session_after) { - gcs_block_pcm_x_master_retry_observe("stability", PCM_X_QUEUE_NOT_READY, -1, 0, 0, - NULL, epoch_before); + gcs_block_pcm_x_master_retry_observe("stability", PCM_X_QUEUE_NOT_READY, -1, 0, 0, NULL, + epoch_before); return; } for (i = 0; i < PCM_X_PROTOCOL_NODE_LIMIT; i++) { @@ -10091,7 +10064,7 @@ cluster_gcs_block_pcm_x_formation_tick(void) goto fail_closed; if (result != PCM_X_QUEUE_OK) { gcs_block_pcm_x_master_retry_observe("peer-revalidate", result, i, 0, 0, NULL, - epoch_before); + epoch_before); return; } } @@ -10099,8 +10072,8 @@ cluster_gcs_block_pcm_x_formation_tick(void) if (runtime_after.state != PCM_X_RUNTIME_ACTIVE || runtime_after.gate_generation != runtime.gate_generation || runtime_after.master_session_incarnation != runtime.master_session_incarnation) { - gcs_block_pcm_x_master_retry_observe("runtime-resample", PCM_X_QUEUE_NOT_READY, -1, - 0, 0, NULL, epoch_before); + gcs_block_pcm_x_master_retry_observe("runtime-resample", PCM_X_QUEUE_NOT_READY, -1, 0, + 0, NULL, epoch_before); return; } gcs_block_pcm_x_master_drive_retry_tick(); @@ -10153,9 +10126,8 @@ cluster_gcs_pcm_x_stage_frame(uint8 msg_type, int32 dest_node_id, const void *pa * V1 frame, but retain the capability/generation fence in the LMS ring slot * until immediately before transport admission. */ static bool -gcs_block_pcm_x_stage_frame_cap_bound(uint8 msg_type, int32 dest_node_id, - const void *payload, uint16 payload_len, - uint32 required_capability, +gcs_block_pcm_x_stage_frame_cap_bound(uint8 msg_type, int32 dest_node_id, const void *payload, + uint16 payload_len, uint32 required_capability, uint32 connection_generation) { int worker; @@ -10164,13 +10136,12 @@ gcs_block_pcm_x_stage_frame_cap_bound(uint8 msg_type, int32 dest_node_id, || dest_node_id < 0 || dest_node_id >= PCM_X_PROTOCOL_NODE_LIMIT || payload == NULL || payload_len == 0 || required_capability == 0 || cluster_lms_workers <= 0) return false; - worker = cluster_gcs_block_payload_shard(msg_type, payload, payload_len, - cluster_lms_workers); + worker = cluster_gcs_block_payload_shard(msg_type, payload, payload_len, cluster_lms_workers); if (worker < 0 || worker >= cluster_lms_workers) return false; - return cluster_lms_outbound_enqueue_cap_bound( - worker, msg_type, (uint32)dest_node_id, payload, payload_len, required_capability, - connection_generation); + return cluster_lms_outbound_enqueue_cap_bound(worker, msg_type, (uint32)dest_node_id, payload, + payload_len, required_capability, + connection_generation); } @@ -10382,8 +10353,8 @@ gcs_block_pcm_x_deny_legacy_readers(const PcmXMasterDriveSnapshot *snapshot) if (snapshot == NULL || !cluster_pcm_lock_queue_pending_x_exact(snapshot->ref.identity.tag, - snapshot->ref.identity.node_id, - snapshot->ref.handle.ticket_id)) + snapshot->ref.identity.node_id, + snapshot->ref.handle.ticket_id)) return PCM_X_QUEUE_BAD_STATE; worker_id = cluster_lms_shard_for_tag(&snapshot->ref.identity.tag, cluster_lms_workers); if (worker_id < 0 || worker_id >= cluster_lms_workers) @@ -10474,9 +10445,8 @@ gcs_block_pcm_x_invalidate_busy_retry_delay_ms(const PcmXMasterDriveSnapshot *sn * scheduling interval; repeated BUSY replies install fresh exact * deadlines on the same ticket. */ (void)snapshot; - delay_ms = (uint64)Max(Max(cluster_gcs_block_starvation_backoff_ms, - cluster_lmon_main_loop_interval), - 1); + delay_ms = (uint64)Max( + Max(cluster_gcs_block_starvation_backoff_ms, cluster_lmon_main_loop_interval), 1); return delay_ms; } @@ -10507,14 +10477,13 @@ gcs_block_pcm_x_master_drive_transfer(const PcmXMasterDriveSnapshot *snapshot) snapshot, now_ms, retry_delay_ms, gcs_block_pcm_x_stage_frame_callback, NULL, &retried); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) gcs_block_pcm_x_master_retry_observe("commit-retry", result, -1, 0, 0, - &snapshot->ref.identity.tag, - snapshot->ref.identity.cluster_epoch); + &snapshot->ref.identity.tag, + snapshot->ref.identity.cluster_epoch); return result; } if (!cluster_gcs_pcm_x_transfer_pre_handoff_phase(snapshot->pending_opcode)) return PCM_X_QUEUE_NOT_READY; - if (snapshot->pending_opcode == PGRAC_IC_MSG_PCM_X_REVOKE - && snapshot->retry_deadline_ms != 0) { + if (snapshot->pending_opcode == PGRAC_IC_MSG_PCM_X_REVOKE && snapshot->retry_deadline_ms != 0) { now_ms = (uint64)(GetCurrentTimestamp() / 1000); if (now_ms < snapshot->retry_deadline_ms) return PCM_X_QUEUE_NOT_READY; @@ -10542,14 +10511,13 @@ gcs_block_pcm_x_master_drive_transfer(const PcmXMasterDriveSnapshot *snapshot) return PCM_X_QUEUE_CORRUPT; memset(&auth_sample, 0, sizeof(auth_sample)); if ((source == cluster_node_id - ? !gcs_block_pcm_x_authenticated_session(source, - snapshot->ref.identity.cluster_epoch, - &source_session) + ? !gcs_block_pcm_x_authenticated_session(source, snapshot->ref.identity.cluster_epoch, + &source_session) : gcs_block_pcm_x_authenticated_session_result( source, snapshot->ref.identity.cluster_epoch, &source_session, &auth_sample) != PCM_X_SESSION_AUTH_OK) || !gcs_block_pcm_x_revalidate_peer_binding(source, snapshot->ref.identity.cluster_epoch, - source_session)) + source_session)) return PCM_X_QUEUE_NOT_READY; result = cluster_pcm_x_master_revoke_arm_exact(&snapshot->ref, source, source_session, &revoke); cluster_pcm_x_stats_note_queue_result(result); @@ -10564,15 +10532,15 @@ gcs_block_pcm_x_master_drive_transfer(const PcmXMasterDriveSnapshot *snapshot) revoke_v2.required_page_scn = (uint64)cluster_pcm_lock_pi_watermark_scn_query(snapshot->ref.identity.tag); return cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_REVOKE, source, &revoke_v2, - sizeof(revoke_v2)) + sizeof(revoke_v2)) ? PCM_X_QUEUE_OK : PCM_X_QUEUE_BUSY; } /* Resample CONVERT + optional SOURCE_FLOOR under one capability-store * lock after the authority pass. Any reconnect since authentication is * retryable: the already-armed reliable leg remains intact. */ - if (!cluster_sf_peer_pcm_x_source_floor_sample( - source, &source_floor_capable, &source_connection_generation) + if (!cluster_sf_peer_pcm_x_source_floor_sample(source, &source_floor_capable, + &source_connection_generation) || source_connection_generation != auth_sample.connection_generation_before || source_connection_generation != auth_sample.connection_generation_after) return PCM_X_QUEUE_NOT_READY; @@ -10582,10 +10550,9 @@ gcs_block_pcm_x_master_drive_transfer(const PcmXMasterDriveSnapshot *snapshot) revoke_v2.required_page_scn = (uint64)cluster_pcm_lock_pi_watermark_scn_query(snapshot->ref.identity.tag); return gcs_block_pcm_x_stage_frame_cap_bound( - PGRAC_IC_MSG_PCM_X_REVOKE, source, &revoke_v2, sizeof(revoke_v2), - PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 - | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1, - source_connection_generation) + PGRAC_IC_MSG_PCM_X_REVOKE, source, &revoke_v2, sizeof(revoke_v2), + PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1, + source_connection_generation) ? PCM_X_QUEUE_OK : PCM_X_QUEUE_BUSY; } @@ -10839,8 +10806,8 @@ gcs_block_pcm_x_master_drive_note(const char *stage, PcmXQueueResult result, con * repeats stay silent until the exit shape changes. */ static void gcs_block_pcm_x_master_retry_observe(const char *stage, PcmXQueueResult result, int32 peer_node, - Size cursor_before, Size cursor_after, const BufferTag *tag, - uint64 cluster_epoch) + Size cursor_before, Size cursor_after, const BufferTag *tag, + uint64 cluster_epoch) { static const char *last_stage = NULL; static PcmXQueueResult last_result = PCM_X_QUEUE_OK; @@ -10863,17 +10830,15 @@ gcs_block_pcm_x_master_retry_observe(const char *stage, PcmXQueueResult result, last_result = result; last_peer_node = peer_node; if (tag != NULL) - ereport(LOG, - (errmsg("PCM-X periodic retry exit (stage %s, result %d, peer %d, " - "cursor %zu->%zu, epoch %llu, rel %u, block %u)", - stage, (int)result, peer_node, cursor_before, cursor_after, - (unsigned long long)cluster_epoch, tag->relNumber, tag->blockNum))); + ereport(LOG, (errmsg("PCM-X periodic retry exit (stage %s, result %d, peer %d, " + "cursor %zu->%zu, epoch %llu, rel %u, block %u)", + stage, (int)result, peer_node, cursor_before, cursor_after, + (unsigned long long)cluster_epoch, tag->relNumber, tag->blockNum))); else - ereport(LOG, - (errmsg("PCM-X periodic retry exit (stage %s, result %d, peer %d, " - "cursor %zu->%zu, epoch %llu)", - stage, (int)result, peer_node, cursor_before, cursor_after, - (unsigned long long)cluster_epoch))); + ereport(LOG, (errmsg("PCM-X periodic retry exit (stage %s, result %d, peer %d, " + "cursor %zu->%zu, epoch %llu)", + stage, (int)result, peer_node, cursor_before, cursor_after, + (unsigned long long)cluster_epoch))); } @@ -10887,37 +10852,37 @@ gcs_block_pcm_x_master_drive_tag(const BufferTag *tag, uint64 cluster_epoch) if (tag == NULL) { gcs_block_pcm_x_master_retry_observe("drive-precheck-tag", PCM_X_QUEUE_INVALID, -1, 0, 0, - NULL, cluster_epoch); + NULL, cluster_epoch); return; } if (cluster_epoch != cluster_epoch_get_current()) { - gcs_block_pcm_x_master_retry_observe("drive-precheck-epoch", PCM_X_QUEUE_STALE, -1, 0, - 0, tag, cluster_epoch); + gcs_block_pcm_x_master_retry_observe("drive-precheck-epoch", PCM_X_QUEUE_STALE, -1, 0, 0, + tag, cluster_epoch); return; } if (cluster_node_id < 0 || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) { gcs_block_pcm_x_master_retry_observe("drive-precheck-node", PCM_X_QUEUE_INVALID, - cluster_node_id, 0, 0, tag, cluster_epoch); + cluster_node_id, 0, 0, tag, cluster_epoch); return; } if (cluster_gcs_lookup_master(*tag) != cluster_node_id) { gcs_block_pcm_x_master_retry_observe("drive-precheck-master", PCM_X_QUEUE_STALE, - cluster_node_id, 0, 0, tag, cluster_epoch); + cluster_node_id, 0, 0, tag, cluster_epoch); return; } if (cluster_gcs_block_phase_for_tag(*tag) == GCS_BLOCK_RECOVERING) { - gcs_block_pcm_x_master_retry_observe("drive-precheck-recovering", PCM_X_QUEUE_NOT_READY, - -1, 0, 0, tag, cluster_epoch); + gcs_block_pcm_x_master_retry_observe("drive-precheck-recovering", PCM_X_QUEUE_NOT_READY, -1, + 0, 0, tag, cluster_epoch); return; } if (!cluster_qvotec_in_quorum()) { - gcs_block_pcm_x_master_retry_observe("drive-precheck-quorum", PCM_X_QUEUE_NOT_READY, -1, - 0, 0, tag, cluster_epoch); + gcs_block_pcm_x_master_retry_observe("drive-precheck-quorum", PCM_X_QUEUE_NOT_READY, -1, 0, + 0, tag, cluster_epoch); return; } if (!cluster_membership_is_member(cluster_node_id)) { gcs_block_pcm_x_master_retry_observe("drive-precheck-member", PCM_X_QUEUE_NOT_READY, - cluster_node_id, 0, 0, tag, cluster_epoch); + cluster_node_id, 0, 0, tag, cluster_epoch); return; } result = cluster_pcm_x_master_promote_head_exact(tag, cluster_epoch, &active); @@ -11433,7 +11398,7 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en if (result == PCM_X_QUEUE_DUPLICATE && snapshot.graph_generation != 0) goto complete_blocker_probe; result = cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&snapshot, entries, - commit->nblockers); + commit->nblockers); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK) { cluster_pcm_x_runtime_fail_closed(); @@ -12188,11 +12153,12 @@ gcs_block_pcm_x_abort_image_before_finish(int worker_id, const GcsBlockPcmXImage static void gcs_block_pcm_x_revoke_refusal_note_exact(const char *site, int own_result, - const ClusterPcmOwnSnapshot *snap, - const PcmXTicketRef *ref, uint64 image_id, - uint64 source_generation); -static void gcs_block_pcm_x_image_ready_arm_refusal_note_work( - int result, const GcsBlockPcmXImageWork *work, PcmXLocalImageReadyRefusal refusal); + const ClusterPcmOwnSnapshot *snap, + const PcmXTicketRef *ref, uint64 image_id, + uint64 source_generation); +static void gcs_block_pcm_x_image_ready_arm_refusal_note_work(int result, + const GcsBlockPcmXImageWork *work, + PcmXLocalImageReadyRefusal refusal); static void @@ -12220,15 +12186,14 @@ gcs_block_pcm_x_stage_ready_work(int worker_id, const GcsBlockPcmXImageWork *wor &image_ready, master_node, work->binding.master_session, &replay, &refusal); cluster_pcm_x_stats_note_queue_result(result); if (diagnostic) - ereport(LOG, - (errmsg_internal("PCM-X IMAGE_READY stage boundary: arm"), - errdetail("result=%d refusal=%d master=%d request_id=%llu ticket=%llu " - "grant_generation=%llu image_id=%llu", - (int)result, (int)refusal, master_node, - (unsigned long long)image_ready.ref.identity.request_id, - (unsigned long long)image_ready.ref.handle.ticket_id, - (unsigned long long)image_ready.ref.grant_generation, - (unsigned long long)image_ready.image.image_id))); + ereport(LOG, (errmsg_internal("PCM-X IMAGE_READY stage boundary: arm"), + errdetail("result=%d refusal=%d master=%d request_id=%llu ticket=%llu " + "grant_generation=%llu image_id=%llu", + (int)result, (int)refusal, master_node, + (unsigned long long)image_ready.ref.identity.request_id, + (unsigned long long)image_ready.ref.handle.ticket_id, + (unsigned long long)image_ready.ref.grant_generation, + (unsigned long long)image_ready.image.image_id))); gcs_block_pcm_x_image_ready_arm_refusal_note_work((int)result, work, refusal); if (result == PCM_X_QUEUE_BUSY || result == PCM_X_QUEUE_NOT_READY) return; @@ -12253,29 +12218,27 @@ gcs_block_pcm_x_stage_ready_work(int worker_id, const GcsBlockPcmXImageWork *wor return; } stage_result = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_IMAGE_READY, master_node, - &replay, sizeof(replay)); + &replay, sizeof(replay)); if (diagnostic) - ereport(LOG, - (errmsg_internal("PCM-X IMAGE_READY stage boundary: DATA ring"), - errdetail("stage_result=%s worker=%d master=%d request_id=%llu ticket=%llu " - "image_id=%llu", - stage_result ? "true" : "false", worker_id, master_node, - (unsigned long long)image_ready.ref.identity.request_id, - (unsigned long long)image_ready.ref.handle.ticket_id, - (unsigned long long)image_ready.image.image_id))); + ereport(LOG, (errmsg_internal("PCM-X IMAGE_READY stage boundary: DATA ring"), + errdetail("stage_result=%s worker=%d master=%d request_id=%llu ticket=%llu " + "image_id=%llu", + stage_result ? "true" : "false", worker_id, master_node, + (unsigned long long)image_ready.ref.identity.request_id, + (unsigned long long)image_ready.ref.handle.ticket_id, + (unsigned long long)image_ready.image.image_id))); if (stage_result) return; rollback_result = cluster_gcs_block_dedup_pcm_x_unmark_staged_exact(worker_id, &work->key, - &work->tag, &work->binding); + &work->tag, &work->binding); if (diagnostic) - ereport(LOG, - (errmsg_internal("PCM-X IMAGE_READY stage boundary: rollback"), - errdetail("rollback_result=%d worker=%d request_id=%llu ticket=%llu " - "image_id=%llu", - (int)rollback_result, worker_id, - (unsigned long long)image_ready.ref.identity.request_id, - (unsigned long long)image_ready.ref.handle.ticket_id, - (unsigned long long)image_ready.image.image_id))); + ereport(LOG, (errmsg_internal("PCM-X IMAGE_READY stage boundary: rollback"), + errdetail("rollback_result=%d worker=%d request_id=%llu ticket=%llu " + "image_id=%llu", + (int)rollback_result, worker_id, + (unsigned long long)image_ready.ref.identity.request_id, + (unsigned long long)image_ready.ref.handle.ticket_id, + (unsigned long long)image_ready.image.image_id))); if (rollback_result != GCS_BLOCK_PCM_X_IMAGE_REARMED && rollback_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) cluster_pcm_x_runtime_fail_closed(); @@ -12283,9 +12246,8 @@ gcs_block_pcm_x_stage_ready_work(int worker_id, const GcsBlockPcmXImageWork *wor static void gcs_block_pcm_x_revoke_refusal_note_exact_diagnosed( - const char *site, int own_result, const ClusterPcmOwnSnapshot *snap, - const PcmXTicketRef *ref, uint64 image_id, uint64 source_generation, - const ClusterPcmOwnFinishRefusal *finish_refusal); + const char *site, int own_result, const ClusterPcmOwnSnapshot *snap, const PcmXTicketRef *ref, + uint64 image_id, uint64 source_generation, const ClusterPcmOwnFinishRefusal *finish_refusal); static void gcs_block_pcm_x_revoke_refusal_note_work(const char *site, int own_result, const ClusterPcmOwnSnapshot *snap, @@ -12298,9 +12260,9 @@ gcs_block_pcm_x_revoke_refusal_note_work(const char *site, int own_result, } static void -gcs_block_pcm_x_revoke_refusal_note_finish_work( - int own_result, const ClusterPcmOwnSnapshot *snap, const GcsBlockPcmXImageWork *work, - const ClusterPcmOwnFinishRefusal *finish_refusal) +gcs_block_pcm_x_revoke_refusal_note_finish_work(int own_result, const ClusterPcmOwnSnapshot *snap, + const GcsBlockPcmXImageWork *work, + const ClusterPcmOwnFinishRefusal *finish_refusal) { const char *site = "materialized-finish-other"; @@ -12319,9 +12281,10 @@ gcs_block_pcm_x_revoke_refusal_note_finish_work( } static void -gcs_block_pcm_x_revoke_refusal_note_source_prepare_work( - int own_result, const ClusterPcmOwnSnapshot *snap, const GcsBlockPcmXImageWork *work, - ClusterPcmOwnSourcePrepareRefusal refusal) +gcs_block_pcm_x_revoke_refusal_note_source_prepare_work(int own_result, + const ClusterPcmOwnSnapshot *snap, + const GcsBlockPcmXImageWork *work, + ClusterPcmOwnSourcePrepareRefusal refusal) { const char *site = "materialize-begin-s-other"; @@ -12340,9 +12303,8 @@ gcs_block_pcm_x_revoke_refusal_note_source_prepare_work( static void -gcs_block_pcm_x_image_ready_arm_refusal_note_work(int result, - const GcsBlockPcmXImageWork *work, - PcmXLocalImageReadyRefusal refusal) +gcs_block_pcm_x_image_ready_arm_refusal_note_work(int result, const GcsBlockPcmXImageWork *work, + PcmXLocalImageReadyRefusal refusal) { const char *site = "image-ready-arm-other"; @@ -12396,8 +12358,8 @@ gcs_block_pcm_x_finish_revoke_retain(int worker_id, const GcsBlockPcmXImageWork PG_TRY(); { - result = cluster_bufmgr_pcm_own_finish_revoke_retain( - buf, revoking, page_lsn, retained, finish_refusal); + result = cluster_bufmgr_pcm_own_finish_revoke_retain(buf, revoking, page_lsn, retained, + finish_refusal); } PG_CATCH(); { @@ -12412,26 +12374,26 @@ gcs_block_pcm_x_finish_revoke_retain(int worker_id, const GcsBlockPcmXImageWork revoking->pcm_state); if (preserve_result != GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING) gcs_block_pcm_x_revoke_refusal_note_work("finish-error-evidence", (int)preserve_result, - revoking, work); - ereport(LOG, - (errmsg_internal("PCM-X finish-error evidence exact"), - errdetail("preserve_result=%d retained=%s worker=%d tag=%u/%u/%u/%d/%u " - "requester=%u backend=%d request_id=%llu ticket=%llu " - "queue_generation=%llu grant_generation=%llu image_id=%llu " - "reservation_token=%llu source_state=%u", - (int)preserve_result, - preserve_result == GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING ? "true" - : "false", - worker_id, work->tag.spcOid, work->tag.dbOid, work->tag.relNumber, - (int)work->tag.forkNum, work->tag.blockNum, - work->key.origin_node_id, work->key.requester_backend_id, - (unsigned long long)work->binding.identity.ref.identity.request_id, - (unsigned long long)work->binding.identity.ref.handle.ticket_id, - (unsigned long long)work->binding.identity.ref.handle.queue_generation, - (unsigned long long)work->binding.identity.ref.grant_generation, - (unsigned long long)work->binding.identity.image.image_id, - (unsigned long long)revoking->reservation_token, - (unsigned)revoking->pcm_state))); + revoking, work); + ereport( + LOG, + (errmsg_internal("PCM-X finish-error evidence exact"), + errdetail("preserve_result=%d retained=%s worker=%d tag=%u/%u/%u/%d/%u " + "requester=%u backend=%d request_id=%llu ticket=%llu " + "queue_generation=%llu grant_generation=%llu image_id=%llu " + "reservation_token=%llu source_state=%u", + (int)preserve_result, + preserve_result == GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING ? "true" : "false", + worker_id, work->tag.spcOid, work->tag.dbOid, work->tag.relNumber, + (int)work->tag.forkNum, work->tag.blockNum, work->key.origin_node_id, + work->key.requester_backend_id, + (unsigned long long)work->binding.identity.ref.identity.request_id, + (unsigned long long)work->binding.identity.ref.handle.ticket_id, + (unsigned long long)work->binding.identity.ref.handle.queue_generation, + (unsigned long long)work->binding.identity.ref.grant_generation, + (unsigned long long)work->binding.identity.image.image_id, + (unsigned long long)revoking->reservation_token, + (unsigned)revoking->pcm_state))); cluster_pcm_x_runtime_fail_closed(); ereport(LOG, (errmsg_internal("cluster PCM-X retained-image finish FlushBuffer failed; " "preserved immutable evidence and blocked recovery: %s", @@ -12491,7 +12453,7 @@ gcs_block_pcm_x_finish_materialized_work(int worker_id, const GcsBlockPcmXImageW gcs_block_pcm_x_note_own_result(own_result); if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY) { gcs_block_pcm_x_revoke_refusal_note_finish_work((int)own_result, ¤t, work, - &finish_refusal); + &finish_refusal); return; } if (own_result != CLUSTER_PCM_OWN_OK) { @@ -12621,18 +12583,17 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage } PG_END_TRY(); own_result = (ClusterPcmOwnResult)n_prepare_result; - } - else + } else own_result = cluster_bufmgr_pcm_own_begin_x_revoke(buf, ¤t, &revoking); gcs_block_pcm_x_note_own_result(own_result); if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY || own_result == CLUSTER_PCM_OWN_STALE) { if (source_is_s) - gcs_block_pcm_x_revoke_refusal_note_source_prepare_work( - (int)own_result, ¤t, work, source_prepare_refusal); + gcs_block_pcm_x_revoke_refusal_note_source_prepare_work((int)own_result, ¤t, work, + source_prepare_refusal); else - gcs_block_pcm_x_revoke_refusal_note_work("materialize-begin", (int)own_result, - ¤t, work); + gcs_block_pcm_x_revoke_refusal_note_work("materialize-begin", (int)own_result, ¤t, + work); return; } if (own_result != CLUSTER_PCM_OWN_OK) { @@ -12663,8 +12624,7 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage } memcpy(&page_header, block_data, sizeof(page_header)); if ((uint64)page_lsn != (uint64)PageXLogRecPtrGet(page_header.pd_lsn) - || ((source_is_n || source_is_s) - && page_scn != (uint64)page_header.pd_block_scn)) { + || ((source_is_n || source_is_s) && page_scn != (uint64)page_header.pd_block_scn)) { gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &work->binding, buf, &revoking, true); return; @@ -12713,8 +12673,8 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage return; } } else { - own_result = gcs_block_pcm_x_finish_revoke_retain(worker_id, work, &ready_binding, buf, - &revoking, page_lsn, &retained, &finish_refusal); + own_result = gcs_block_pcm_x_finish_revoke_retain( + worker_id, work, &ready_binding, buf, &revoking, page_lsn, &retained, &finish_refusal); gcs_block_pcm_x_note_own_result(own_result); if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY) { gcs_block_pcm_x_revoke_refusal_note_work("materialize-finish", (int)own_result, @@ -12838,14 +12798,13 @@ gcs_block_pcm_x_revoke_refusal_note_exact(const char *site, int own_result, uint64 source_generation) { gcs_block_pcm_x_revoke_refusal_note_exact_diagnosed(site, own_result, snap, ref, image_id, - source_generation, NULL); + source_generation, NULL); } static void gcs_block_pcm_x_revoke_refusal_note_exact_diagnosed( - const char *site, int own_result, const ClusterPcmOwnSnapshot *snap, - const PcmXTicketRef *ref, uint64 image_id, uint64 source_generation, - const ClusterPcmOwnFinishRefusal *finish_refusal) + const char *site, int own_result, const ClusterPcmOwnSnapshot *snap, const PcmXTicketRef *ref, + uint64 image_id, uint64 source_generation, const ClusterPcmOwnFinishRefusal *finish_refusal) { static const char *last_site = NULL; static uint32 last_flags = 0; @@ -12916,10 +12875,8 @@ gcs_block_pcm_x_revoke_refusal_note_exact_diagnosed( (unsigned long long)(snap != NULL ? snap->generation : 0), (unsigned long long)(snap != NULL ? snap->reservation_token : 0), (unsigned int)flags, (unsigned int)finish_reason, - (unsigned int)shared_refcount, - bm_io_in_progress ? "true" : "false", - (unsigned int)live_flags, - (unsigned long long)live_token))); + (unsigned int)shared_refcount, bm_io_in_progress ? "true" : "false", + (unsigned int)live_flags, (unsigned long long)live_token))); } return; } @@ -13088,8 +13045,8 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi return; } } - result = cluster_pcm_x_local_holder_revoke_apply_floor_exact( - revoke, required_page_scn, source_node, source_session); + result = cluster_pcm_x_local_holder_revoke_apply_floor_exact(revoke, required_page_scn, + source_node, source_session); cluster_pcm_x_stats_note_queue_result(result); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { /* Ingress only installs/re-arms holder work. It does not prove the @@ -13134,44 +13091,39 @@ cluster_gcs_handle_pcm_x_image_ready_envelope(const ClusterICEnvelope *env, cons diagnostic = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); wire_valid = cluster_gcs_pcm_x_image_ready_ingress_valid( image_ready, env->payload_length, source_node, current_epoch, tag_master, cluster_node_id); - authorized - = wire_valid - && gcs_block_pcm_x_transfer_ingress_authorized( - &image_ready->ref.identity.tag, source_node, current_epoch, &source_session); + authorized = wire_valid + && gcs_block_pcm_x_transfer_ingress_authorized( + &image_ready->ref.identity.tag, source_node, current_epoch, &source_session); if (diagnostic) - ereport(LOG, - (errmsg_internal("PCM-X IMAGE_READY master boundary: ingress"), - errdetail("wire_valid=%s authorized=%s source=%d local=%d tag_master=%d " - "epoch=%llu source_session=%llu request_id=%llu ticket=%llu " - "grant_generation=%llu image_source=%d image_id=%llu", - wire_valid ? "true" : "false", authorized ? "true" : "false", - source_node, cluster_node_id, tag_master, - (unsigned long long)current_epoch, - (unsigned long long)(authorized ? source_session : 0), - (unsigned long long)image_ready->ref.identity.request_id, - (unsigned long long)image_ready->ref.handle.ticket_id, - (unsigned long long)image_ready->ref.grant_generation, - image_ready->image.source_node, - (unsigned long long)image_ready->image.image_id))); + ereport(LOG, (errmsg_internal("PCM-X IMAGE_READY master boundary: ingress"), + errdetail("wire_valid=%s authorized=%s source=%d local=%d tag_master=%d " + "epoch=%llu source_session=%llu request_id=%llu ticket=%llu " + "grant_generation=%llu image_source=%d image_id=%llu", + wire_valid ? "true" : "false", authorized ? "true" : "false", + source_node, cluster_node_id, tag_master, + (unsigned long long)current_epoch, + (unsigned long long)(authorized ? source_session : 0), + (unsigned long long)image_ready->ref.identity.request_id, + (unsigned long long)image_ready->ref.handle.ticket_id, + (unsigned long long)image_ready->ref.grant_generation, + image_ready->image.source_node, + (unsigned long long)image_ready->image.image_id))); if (!wire_valid || !authorized) return; - required_page_scn - = cluster_pcm_lock_pi_watermark_scn_query(image_ready->ref.identity.tag); + required_page_scn = cluster_pcm_lock_pi_watermark_scn_query(image_ready->ref.identity.tag); floor_verdict = gcs_block_lost_write_verdict(required_page_scn, (SCN)image_ready->image.page_scn); if (floor_verdict == GCS_LOST_WRITE_FAIL_STALE || floor_verdict == GCS_LOST_WRITE_FAIL_ANOMALY) { - ereport(LOG, - (errmsg_internal("PCM-X IMAGE_READY source stale before PREPARE: " - "source=%d request=%llu ticket=%llu image=%llu " - "image_scn=%llu required_scn=%llu verdict=%d", - source_node, - (unsigned long long)image_ready->ref.identity.request_id, - (unsigned long long)image_ready->ref.handle.ticket_id, - (unsigned long long)image_ready->image.image_id, - (unsigned long long)image_ready->image.page_scn, - (unsigned long long)required_page_scn, - (int)floor_verdict))); + ereport(LOG, (errmsg_internal("PCM-X IMAGE_READY source stale before PREPARE: " + "source=%d request=%llu ticket=%llu image=%llu " + "image_scn=%llu required_scn=%llu verdict=%d", + source_node, + (unsigned long long)image_ready->ref.identity.request_id, + (unsigned long long)image_ready->ref.handle.ticket_id, + (unsigned long long)image_ready->image.image_id, + (unsigned long long)image_ready->image.page_scn, + (unsigned long long)required_page_scn, (int)floor_verdict))); cluster_pcm_x_runtime_fail_closed(); return; } @@ -13180,24 +13132,23 @@ cluster_gcs_handle_pcm_x_image_ready_envelope(const ClusterICEnvelope *env, cons cluster_pcm_x_stats_note_queue_result(result); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { gcs_block_pcm_x_revoke_refusal_note_exact(NULL, 0, NULL, NULL, 0, 0); - prepare_stage_result - = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, - prepare.ref.identity.node_id, &prepare, sizeof(prepare)); + prepare_stage_result = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, + prepare.ref.identity.node_id, &prepare, + sizeof(prepare)); } else - gcs_block_pcm_x_revoke_refusal_note_exact( - "image-ready-master-consume", (int)result, NULL, &image_ready->ref, - image_ready->image.image_id, image_ready->image.source_own_generation); + gcs_block_pcm_x_revoke_refusal_note_exact("image-ready-master-consume", (int)result, NULL, + &image_ready->ref, image_ready->image.image_id, + image_ready->image.source_own_generation); if (diagnostic) - ereport(LOG, - (errmsg_internal("PCM-X IMAGE_READY master boundary: consume"), - errdetail("result=%d prepare_stage_result=%s requester=%d source=%d " - "request_id=%llu ticket=%llu grant_generation=%llu image_id=%llu", - (int)result, prepare_stage_result ? "true" : "false", - image_ready->ref.identity.node_id, source_node, - (unsigned long long)image_ready->ref.identity.request_id, - (unsigned long long)image_ready->ref.handle.ticket_id, - (unsigned long long)image_ready->ref.grant_generation, - (unsigned long long)image_ready->image.image_id))); + ereport(LOG, (errmsg_internal("PCM-X IMAGE_READY master boundary: consume"), + errdetail("result=%d prepare_stage_result=%s requester=%d source=%d " + "request_id=%llu ticket=%llu grant_generation=%llu image_id=%llu", + (int)result, prepare_stage_result ? "true" : "false", + image_ready->ref.identity.node_id, source_node, + (unsigned long long)image_ready->ref.identity.request_id, + (unsigned long long)image_ready->ref.handle.ticket_id, + (unsigned long long)image_ready->ref.grant_generation, + (unsigned long long)image_ready->image.image_id))); } @@ -13226,52 +13177,48 @@ cluster_gcs_handle_pcm_x_prepare_grant_envelope(const ClusterICEnvelope *env, co diagnostic = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); wire_valid = cluster_gcs_pcm_x_prepare_grant_ingress_valid( prepare, env->payload_length, source_node, current_epoch, tag_master, cluster_node_id); - authorized - = wire_valid - && gcs_block_pcm_x_transfer_ingress_authorized( - &prepare->ref.identity.tag, source_node, current_epoch, &source_session); + authorized = wire_valid + && gcs_block_pcm_x_transfer_ingress_authorized( + &prepare->ref.identity.tag, source_node, current_epoch, &source_session); if (diagnostic) - ereport(LOG, - (errmsg_internal("PCM-X PREPARE_GRANT requester boundary: ingress"), - errdetail("wire_valid=%s authorized=%s source=%d local=%d tag_master=%d " - "epoch=%llu source_session=%llu request_id=%llu ticket=%llu " - "grant_generation=%llu image_id=%llu", - wire_valid ? "true" : "false", authorized ? "true" : "false", - source_node, cluster_node_id, tag_master, - (unsigned long long)current_epoch, - (unsigned long long)(authorized ? source_session : 0), - (unsigned long long)prepare->ref.identity.request_id, - (unsigned long long)prepare->ref.handle.ticket_id, - (unsigned long long)prepare->ref.grant_generation, - (unsigned long long)prepare->image.image_id))); + ereport(LOG, (errmsg_internal("PCM-X PREPARE_GRANT requester boundary: ingress"), + errdetail("wire_valid=%s authorized=%s source=%d local=%d tag_master=%d " + "epoch=%llu source_session=%llu request_id=%llu ticket=%llu " + "grant_generation=%llu image_id=%llu", + wire_valid ? "true" : "false", authorized ? "true" : "false", + source_node, cluster_node_id, tag_master, + (unsigned long long)current_epoch, + (unsigned long long)(authorized ? source_session : 0), + (unsigned long long)prepare->ref.identity.request_id, + (unsigned long long)prepare->ref.handle.ticket_id, + (unsigned long long)prepare->ref.grant_generation, + (unsigned long long)prepare->image.image_id))); if (!wire_valid || !authorized) return; lookup_result = cluster_pcm_x_local_lookup_exact(&prepare->ref.identity, &leader); if (lookup_result != PCM_X_QUEUE_OK) { if (diagnostic) - ereport(LOG, - (errmsg_internal("PCM-X PREPARE_GRANT requester boundary: apply"), - errdetail("lookup_result=%d apply_result=-1 source=%d request_id=%llu " - "ticket=%llu grant_generation=%llu image_id=%llu", - (int)lookup_result, source_node, - (unsigned long long)prepare->ref.identity.request_id, - (unsigned long long)prepare->ref.handle.ticket_id, - (unsigned long long)prepare->ref.grant_generation, - (unsigned long long)prepare->image.image_id))); + ereport(LOG, (errmsg_internal("PCM-X PREPARE_GRANT requester boundary: apply"), + errdetail("lookup_result=%d apply_result=-1 source=%d request_id=%llu " + "ticket=%llu grant_generation=%llu image_id=%llu", + (int)lookup_result, source_node, + (unsigned long long)prepare->ref.identity.request_id, + (unsigned long long)prepare->ref.handle.ticket_id, + (unsigned long long)prepare->ref.grant_generation, + (unsigned long long)prepare->image.image_id))); return; } result = cluster_pcm_x_local_prepare_grant_exact(&leader, prepare, source_node, source_session); cluster_pcm_x_stats_note_queue_result(result); if (diagnostic) - ereport(LOG, - (errmsg_internal("PCM-X PREPARE_GRANT requester boundary: apply"), - errdetail("lookup_result=%d apply_result=%d source=%d request_id=%llu " - "ticket=%llu grant_generation=%llu image_id=%llu", - (int)lookup_result, (int)result, source_node, - (unsigned long long)prepare->ref.identity.request_id, - (unsigned long long)prepare->ref.handle.ticket_id, - (unsigned long long)prepare->ref.grant_generation, - (unsigned long long)prepare->image.image_id))); + ereport(LOG, (errmsg_internal("PCM-X PREPARE_GRANT requester boundary: apply"), + errdetail("lookup_result=%d apply_result=%d source=%d request_id=%llu " + "ticket=%llu grant_generation=%llu image_id=%llu", + (int)lookup_result, (int)result, source_node, + (unsigned long long)prepare->ref.identity.request_id, + (unsigned long long)prepare->ref.handle.ticket_id, + (unsigned long long)prepare->ref.grant_generation, + (unsigned long long)prepare->image.image_id))); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) gcs_block_pcm_x_wake_requester(&prepare->ref.identity); } @@ -13404,8 +13351,7 @@ cluster_gcs_handle_pcm_x_final_ack_envelope(const ClusterICEnvelope *env, const fail_stage = "prepare"; goto final_ack_fail_closed; } - authority_valid - = cluster_pcm_lock_authority_snapshot(final_ack->ref.identity.tag, &authority); + authority_valid = cluster_pcm_lock_authority_snapshot(final_ack->ref.identity.tag, &authority); if (!authority_valid) { fail_stage = "authority-snapshot"; goto final_ack_fail_closed; @@ -13433,10 +13379,8 @@ cluster_gcs_handle_pcm_x_final_ack_envelope(const ClusterICEnvelope *env, const return; final_ack_fail_closed: - watermark_scn - = cluster_pcm_lock_pi_watermark_scn_query(final_ack->ref.identity.tag); - wm_have - = cluster_pcm_lock_pi_watermark_prov_query(final_ack->ref.identity.tag, &wm_prov); + watermark_scn = cluster_pcm_lock_pi_watermark_scn_query(final_ack->ref.identity.tag); + wm_have = cluster_pcm_lock_pi_watermark_prov_query(final_ack->ref.identity.tag, &wm_prov); ereport( LOG, (errmsg("PCM-X FINAL_ACK fail-closed at %s", fail_stage), @@ -13462,9 +13406,8 @@ cluster_gcs_handle_pcm_x_final_ack_envelope(const ClusterICEnvelope *env, const authority.x_holder_node, authority.s_holders_bitmap, authority.master_holder.node_id, authority.pending_x_requester_node, (unsigned long long)authority.pending_x_since_lsn, token.image.source_node, (unsigned long long)handoff.page_scn, - (unsigned long long)watermark_scn, - cluster_pcm_wm_src_text(wm_prov.source), wm_have ? 1 : 0, - wm_prov.table_full ? 1 : 0, wm_prov.sender_node, + (unsigned long long)watermark_scn, cluster_pcm_wm_src_text(wm_prov.source), + wm_have ? 1 : 0, wm_prov.table_full ? 1 : 0, wm_prov.sender_node, (unsigned long long)wm_prov.request_id, (unsigned long long)wm_prov.epoch, (unsigned long long)wm_prov.old_scn, (unsigned long long)wm_prov.new_scn, wm_have && wm_prov.new_scn == watermark_scn ? 1 : 0))); @@ -14106,12 +14049,12 @@ gcs_block_pcm_x_master_drive_retry_tick(void) cursor_before = cursor; result = cluster_pcm_x_master_drive_work_next(&cursor, PCM_X_MASTER_DRIVE_SCAN_BUDGET, &tag, - &cluster_epoch); + &cluster_epoch); if (result == PCM_X_QUEUE_OK) gcs_block_pcm_x_master_drive_tag(&tag, cluster_epoch); else gcs_block_pcm_x_master_retry_observe("work-next", result, -1, cursor_before, cursor, NULL, - cluster_epoch); + cluster_epoch); } @@ -15264,16 +15207,14 @@ gcs_block_wake_local_pending_s_request(const GcsBlockInvalidatePayload *inv) int slot_idx; LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); - for (slot_idx = 0; slot_idx < MAX_OUTSTANDING_BLOCK_REQUESTS_PER_BACKEND; - slot_idx++) { + for (slot_idx = 0; slot_idx < MAX_OUTSTANDING_BLOCK_REQUESTS_PER_BACKEND; slot_idx++) { ClusterGcsBlockOutstandingSlot *slot = &blk->slots[slot_idx]; GcsBlockReplyHeader *hdr; if (!GcsBlockLocalPendingSDenialMatches( slot->in_use, slot->reply_received, slot->stale, slot->transition_id, - &slot->tag, slot->request_epoch, slot->expected_master_node, - slot->direct_state, slot->direct_target_prepared, &inv->tag, inv->epoch, - inv->master_node)) + &slot->tag, slot->request_epoch, slot->expected_master_node, slot->direct_state, + slot->direct_target_prepared, &inv->tag, inv->epoch, inv->master_node)) continue; hdr = &slot->reply_header; @@ -15284,8 +15225,7 @@ gcs_block_wake_local_pending_s_request(const GcsBlockInvalidatePayload *inv) hdr->requester_backend_id = backend_idx + 1; hdr->transition_id = slot->transition_id; hdr->status = (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X; - GcsBlockReplyHeaderSetForwardingMasterNode( - hdr, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + GcsBlockReplyHeaderSetForwardingMasterNode(hdr, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); memset(slot->reply_block_data, 0, sizeof(slot->reply_block_data)); hdr->checksum = gcs_block_compute_checksum(slot->reply_block_data); slot->reply_sf_dep_valid = false; @@ -15318,8 +15258,7 @@ gcs_block_wake_local_pending_s_request(const GcsBlockInvalidatePayload *inv) #define GCS_BLOCK_PENDING_OBS_DIRECT_STATE UINT32_C(0x080) #define GCS_BLOCK_PENDING_OBS_DIRECT_PREPARED UINT32_C(0x100) -typedef struct GcsBlockGrantPendingObservation -{ +typedef struct GcsBlockGrantPendingObservation { bool valid; uint64 invalidate_request_id; uint64 invalidate_epoch; @@ -15349,8 +15288,7 @@ typedef struct GcsBlockGrantPendingObservation * BUSY retries while retaining the first sighting and every identity/state * change for the exact INVALIDATE. */ static void -gcs_block_observe_grant_pending_invalidate(const GcsBlockInvalidatePayload *inv, - bool woke_local) +gcs_block_observe_grant_pending_invalidate(const GcsBlockInvalidatePayload *inv, bool woke_local) { static GcsBlockGrantPendingObservation cache[8]; static uint32 next_cache_slot = 0; @@ -15375,8 +15313,7 @@ gcs_block_observe_grant_pending_invalidate(const GcsBlockInvalidatePayload *inv, obs.slot_index = -1; obs.slot_master = -1; obs.woke_local = woke_local; - obs.own_result - = (int)cluster_bufmgr_pcm_own_snapshot_by_tag(&inv->tag, &obs.buffer_id, &own); + obs.own_result = (int)cluster_bufmgr_pcm_own_snapshot_by_tag(&inv->tag, &obs.buffer_id, &own); obs.own_state = own.pcm_state; obs.own_generation = own.generation; obs.own_token = own.reservation_token; @@ -15388,8 +15325,7 @@ gcs_block_observe_grant_pending_invalidate(const GcsBlockInvalidatePayload *inv, int slot_idx; LWLockAcquire(&blk->lock.lock, LW_SHARED); - for (slot_idx = 0; slot_idx < MAX_OUTSTANDING_BLOCK_REQUESTS_PER_BACKEND; - slot_idx++) { + for (slot_idx = 0; slot_idx < MAX_OUTSTANDING_BLOCK_REQUESTS_PER_BACKEND; slot_idx++) { ClusterGcsBlockOutstandingSlot *slot = &blk->slots[slot_idx]; int priority; @@ -15437,8 +15373,7 @@ gcs_block_observe_grant_pending_invalidate(const GcsBlockInvalidatePayload *inv, obs.reject_mask = GCS_BLOCK_PENDING_OBS_NO_SLOT; for (i = 0; i < lengthof(cache); i++) { - if (cache[i].valid - && cache[i].invalidate_request_id == obs.invalidate_request_id + if (cache[i].valid && cache[i].invalidate_request_id == obs.invalidate_request_id && cache[i].invalidate_epoch == obs.invalidate_epoch && cache[i].invalidate_master == obs.invalidate_master && BufferTagsEqual(&cache[i].tag, &obs.tag)) { @@ -15461,16 +15396,15 @@ gcs_block_observe_grant_pending_invalidate(const GcsBlockInvalidatePayload *inv, "woke_local=%d slot_backend=%d slot_index=%d slot_request_id=%llu " "slot_epoch=%llu slot_master=%d slot_transition=%u reply_received=%d stale=%d " "direct_state=%d direct_prepared=%d reject_mask=0x%x", - (unsigned long long)obs.invalidate_request_id, - (unsigned long long)obs.invalidate_epoch, obs.invalidate_master, - (unsigned int)inv->invalidating_for_x_node, inv->tag.relNumber, - (int)inv->tag.forkNum, inv->tag.blockNum, obs.own_result, obs.buffer_id, - obs.own_state, (unsigned long long)obs.own_generation, - (unsigned long long)obs.own_token, obs.own_flags, obs.woke_local ? 1 : 0, - obs.slot_backend, obs.slot_index, (unsigned long long)obs.slot_request_id, - (unsigned long long)obs.slot_epoch, obs.slot_master, obs.slot_transition, - obs.slot_reply_received ? 1 : 0, obs.slot_stale ? 1 : 0, - obs.slot_direct_state, obs.slot_direct_prepared ? 1 : 0, obs.reject_mask); + (unsigned long long)obs.invalidate_request_id, (unsigned long long)obs.invalidate_epoch, + obs.invalidate_master, (unsigned int)inv->invalidating_for_x_node, inv->tag.relNumber, + (int)inv->tag.forkNum, inv->tag.blockNum, obs.own_result, obs.buffer_id, obs.own_state, + (unsigned long long)obs.own_generation, (unsigned long long)obs.own_token, obs.own_flags, + obs.woke_local ? 1 : 0, obs.slot_backend, obs.slot_index, + (unsigned long long)obs.slot_request_id, (unsigned long long)obs.slot_epoch, + obs.slot_master, obs.slot_transition, obs.slot_reply_received ? 1 : 0, + obs.slot_stale ? 1 : 0, obs.slot_direct_state, obs.slot_direct_prepared ? 1 : 0, + obs.reject_mask); } /* @@ -15638,15 +15572,15 @@ gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) if (!pcm_x_queue_invalidate_refusal_logged) { pcm_x_queue_invalidate_refusal_logged = true; - ereport(LOG, - (errmsg_internal("PCM-X queue INVALIDATE passive-S release refused"), - errdetail_internal("epoch=%llu tag=%u/%u/%u/%d/%u requester=%u " - "master=%d request_id=%llu queue_result=%d own_result=%d", - (unsigned long long)inv->epoch, inv->tag.spcOid, - inv->tag.dbOid, inv->tag.relNumber, (int)inv->tag.forkNum, - inv->tag.blockNum, (unsigned int)inv->invalidating_for_x_node, - inv->master_node, (unsigned long long)inv->request_id, - (int)queue_result, (int)own_result))); + ereport(LOG, (errmsg_internal("PCM-X queue INVALIDATE passive-S release refused"), + errdetail_internal( + "epoch=%llu tag=%u/%u/%u/%d/%u requester=%u " + "master=%d request_id=%llu queue_result=%d own_result=%d", + (unsigned long long)inv->epoch, inv->tag.spcOid, inv->tag.dbOid, + inv->tag.relNumber, (int)inv->tag.forkNum, inv->tag.blockNum, + (unsigned int)inv->invalidating_for_x_node, inv->master_node, + (unsigned long long)inv->request_id, (int)queue_result, + (int)own_result))); } } /* GCS serve-stall round-5 (A2): nothing changed, no ACK — the @@ -16066,8 +16000,7 @@ cluster_gcs_handle_block_invalidate_ack_envelope(const ClusterICEnvelope *env, c queue_positive = true; else if (queue_result == PCM_X_QUEUE_BUSY) { uint64 now_ms = (uint64)(GetCurrentTimestamp() / 1000); - uint64 retry_delay_ms - = gcs_block_pcm_x_invalidate_busy_retry_delay_ms(&queue_snapshot); + uint64 retry_delay_ms = gcs_block_pcm_x_invalidate_busy_retry_delay_ms(&queue_snapshot); queue_result = cluster_pcm_x_master_invalidate_busy_backoff_exact( &queue_snapshot, ack->sender_node, now_ms, retry_delay_ms, &queue_updated); @@ -16077,10 +16010,9 @@ cluster_gcs_handle_block_invalidate_ack_envelope(const ClusterICEnvelope *env, c else gcs_block_pcm_x_master_drive_fail_closed(queue_result); return; - } - else if (queue_result == PCM_X_QUEUE_BAD_STATE || queue_result == PCM_X_QUEUE_CORRUPT - || queue_result == PCM_X_QUEUE_COUNTER_EXHAUSTED - || queue_result == PCM_X_QUEUE_INVALID) + } else if (queue_result == PCM_X_QUEUE_BAD_STATE || queue_result == PCM_X_QUEUE_CORRUPT + || queue_result == PCM_X_QUEUE_COUNTER_EXHAUSTED + || queue_result == PCM_X_QUEUE_INVALID) gcs_block_pcm_x_master_drive_fail_closed(queue_result); if (!queue_positive) return; diff --git a/src/backend/cluster/cluster_gcs_block_dedup.c b/src/backend/cluster/cluster_gcs_block_dedup.c index 513c078d27..6600794a45 100644 --- a/src/backend/cluster/cluster_gcs_block_dedup.c +++ b/src/backend/cluster/cluster_gcs_block_dedup.c @@ -488,8 +488,7 @@ dedup_pcm_x_required_page_scn_get(const GcsBlockDedupEntry *entry) } static void -dedup_pcm_x_binding_from_entry(const GcsBlockDedupEntry *entry, - GcsBlockPcmXImageBinding *binding) +dedup_pcm_x_binding_from_entry(const GcsBlockDedupEntry *entry, GcsBlockPcmXImageBinding *binding) { binding->identity = entry->payload_meta.pcm_x_identity; binding->master_session = entry->pcm_x_master_session; @@ -752,15 +751,13 @@ dedup_pending_x_denial_is_exact(const GcsBlockDedupEntry *entry) return entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_GENERIC && entry->transition_id == (uint8)PCM_TRANS_N_TO_S && entry->status == (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X - && entry->completed_at_ts != 0 - && header->request_id == entry->key.request_id - && header->epoch == entry->key.cluster_epoch - && header->sender_node == cluster_node_id + && entry->completed_at_ts != 0 && header->request_id == entry->key.request_id + && header->epoch == entry->key.cluster_epoch && header->sender_node == cluster_node_id && header->requester_backend_id == entry->key.requester_backend_id && header->transition_id == (uint8)PCM_TRANS_N_TO_S && header->status == (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X && GcsBlockReplyHeaderGetForwardingMasterNode(header) - == GCS_BLOCK_REPLY_NO_FORWARDING_MASTER; + == GCS_BLOCK_REPLY_NO_FORWARDING_MASTER; } static bool @@ -771,8 +768,7 @@ dedup_pending_x_entry_has_legacy_s_right(const GcsBlockDedupEntry *entry) if (entry->completed_at_ts == 0) return true; status = (GcsBlockReplyStatus)entry->status; - return status == GCS_BLOCK_REPLY_GRANTED - || status == GCS_BLOCK_REPLY_GRANTED_STORAGE_FALLBACK + return status == GCS_BLOCK_REPLY_GRANTED || status == GCS_BLOCK_REPLY_GRANTED_STORAGE_FALLBACK || status == GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER || status == GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER || status == GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE; @@ -864,8 +860,7 @@ cluster_gcs_block_dedup_pending_x_deny_next(int worker_id, const BufferTag *tag, if (have_replay) *denied_out = replay; LWLockRelease(&shard->lock.lock); - return have_replay ? GCS_BLOCK_PENDING_X_DENY_REPLAY - : GCS_BLOCK_PENDING_X_DENY_NOT_FOUND; + return have_replay ? GCS_BLOCK_PENDING_X_DENY_REPLAY : GCS_BLOCK_PENDING_X_DENY_NOT_FOUND; } GcsBlockPendingXDenyResult @@ -892,8 +887,7 @@ cluster_gcs_block_dedup_pending_x_deny_exact(int worker_id, const GcsBlockDedupK LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); if (!found || entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC - || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 - || entry->transition_id != transition_id) { + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 || entry->transition_id != transition_id) { if (found && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC) dedup_pcm_x_note_failclosed(shard); LWLockRelease(&shard->lock.lock); @@ -904,8 +898,7 @@ cluster_gcs_block_dedup_pending_x_deny_exact(int worker_id, const GcsBlockDedupK LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PENDING_X_DENY_REPLAY; } - if (entry->status == (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X - && entry->completed_at_ts != 0) { + if (entry->status == (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X && entry->completed_at_ts != 0) { dedup_pcm_x_note_failclosed(shard); LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PENDING_X_DENY_INVALID; @@ -919,8 +912,8 @@ cluster_gcs_block_dedup_pending_x_deny_exact(int worker_id, const GcsBlockDedupK bool cluster_gcs_block_dedup_set_request_flags_exact(int worker_id, const GcsBlockDedupKey *key, - const BufferTag *tag, uint8 transition_id, - uint8 request_flags) + const BufferTag *tag, uint8 transition_id, + uint8 request_flags) { ClusterGcsBlockDedupShard *shard; HTAB *htab = NULL; @@ -930,8 +923,7 @@ cluster_gcs_block_dedup_set_request_flags_exact(int worker_id, const GcsBlockDed Assert(key != NULL); Assert(tag != NULL); - if (key == NULL || tag == NULL - || (request_flags & ~GCS_BLOCK_DEDUP_REQUEST_F_VALID_MASK) != 0) + if (key == NULL || tag == NULL || (request_flags & ~GCS_BLOCK_DEDUP_REQUEST_F_VALID_MASK) != 0) return false; shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); if (shard == NULL) @@ -939,8 +931,7 @@ cluster_gcs_block_dedup_set_request_flags_exact(int worker_id, const GcsBlockDed LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); if (found && entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_GENERIC - && memcmp(&entry->tag, tag, sizeof(*tag)) == 0 - && entry->transition_id == transition_id) { + && memcmp(&entry->tag, tag, sizeof(*tag)) == 0 && entry->transition_id == transition_id) { uint8 pinned_flags = GCS_BLOCK_DEDUP_REQUEST_F_PINNED | request_flags; if (entry->request_flags == 0) diff --git a/src/backend/cluster/cluster_gcs_block_shard.c b/src/backend/cluster/cluster_gcs_block_shard.c index fc56439f53..e957e8df4c 100644 --- a/src/backend/cluster/cluster_gcs_block_shard.c +++ b/src/backend/cluster/cluster_gcs_block_shard.c @@ -143,8 +143,7 @@ cluster_gcs_block_payload_shard(uint8 msg_type, const void *payload, uint16 payl break; case PGRAC_IC_MSG_PCM_X_REVOKE: /* Source-floor V2 appends one SCN to the byte-identical V1 prefix. */ - if (payload_len != sizeof(PcmXRevokePayload) - && payload_len != sizeof(PcmXRevokePayloadV2)) + if (payload_len != sizeof(PcmXRevokePayload) && payload_len != sizeof(PcmXRevokePayloadV2)) return -1; memcpy(&pcm_x_tag, payload, sizeof(pcm_x_tag)); tag = &pcm_x_tag; diff --git a/src/backend/cluster/cluster_lms.c b/src/backend/cluster/cluster_lms.c index d9d0bdf080..a42628e9e1 100644 --- a/src/backend/cluster/cluster_lms.c +++ b/src/backend/cluster/cluster_lms.c @@ -61,8 +61,8 @@ #include "cluster/cluster_cr_server.h" /* spec-6.12b CR work slots */ #include "cluster/cluster_conf.h" #include "cluster/cluster_cssd.h" -#include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ -#include "cluster/cluster_gcs_block.h" /* PGRAC: spec-7.2 D4 plane probe + pi drain */ +#include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ +#include "cluster/cluster_gcs_block.h" /* PGRAC: spec-7.2 D4 plane probe + pi drain */ #include "cluster/cluster_inject.h" #include "cluster/cluster_write_fence.h" /* PGRAC: spec-7.2 D5 fence linkage */ #include "cluster/cluster_ges.h" @@ -724,22 +724,23 @@ lms_note_pcm_x_finish_flush_injection_reload(int worker_id) void -cluster_lms_note_pcm_x_image_ready_boundary( - uint8 msg_type, const char *boundary, int result, int runtime_state, bool fence_enforcing, - bool fence_allowed, uint32 dest_node_id, uint64 request_id, uint64 ticket_id, uint64 grant_generation, - uint64 image_id) +cluster_lms_note_pcm_x_image_ready_boundary(uint8 msg_type, const char *boundary, int result, + int runtime_state, bool fence_enforcing, + bool fence_allowed, uint32 dest_node_id, + uint64 request_id, uint64 ticket_id, + uint64 grant_generation, uint64 image_id) { - if (boundary == NULL - || !cluster_injection_is_armed("cluster-pcm-x-retain-flush-error")) + if (boundary == NULL || !cluster_injection_is_armed("cluster-pcm-x-retain-flush-error")) return; - ereport(LOG, - (errmsg_internal("PCM-X IMAGE_READY transport boundary: %s", boundary), - errdetail("msg_type=%u result=%d runtime=%d fence_enforcing=%s fence_allowed=%s dest=%u " - "request_id=%llu ticket=%llu grant_generation=%llu image_id=%llu", - (unsigned)msg_type, result, runtime_state, fence_enforcing ? "true" : "false", - fence_allowed ? "true" : "false", dest_node_id, - (unsigned long long)request_id, (unsigned long long)ticket_id, - (unsigned long long)grant_generation, (unsigned long long)image_id))); + ereport( + LOG, + (errmsg_internal("PCM-X IMAGE_READY transport boundary: %s", boundary), + errdetail("msg_type=%u result=%d runtime=%d fence_enforcing=%s fence_allowed=%s dest=%u " + "request_id=%llu ticket=%llu grant_generation=%llu image_id=%llu", + (unsigned)msg_type, result, runtime_state, fence_enforcing ? "true" : "false", + fence_allowed ? "true" : "false", dest_node_id, (unsigned long long)request_id, + (unsigned long long)ticket_id, (unsigned long long)grant_generation, + (unsigned long long)image_id))); } void diff --git a/src/backend/cluster/cluster_lms_outbound.c b/src/backend/cluster/cluster_lms_outbound.c index 1b1186cf9c..a6bff9ad08 100644 --- a/src/backend/cluster/cluster_lms_outbound.c +++ b/src/backend/cluster/cluster_lms_outbound.c @@ -204,8 +204,8 @@ cluster_lms_outbound_request_lwlocks(void) */ static bool lms_outbound_enqueue_internal(int worker_id, uint8 msg_type, uint32 dest_node_id, - const void *payload, uint16 payload_len, - uint32 required_capability, uint32 connection_generation) + const void *payload, uint16 payload_len, uint32 required_capability, + uint32 connection_generation) { ClusterLmsOutboundState *ring; LWLock *lock; @@ -247,8 +247,8 @@ bool cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, const void *payload, uint16 payload_len) { - return lms_outbound_enqueue_internal(worker_id, msg_type, dest_node_id, payload, payload_len, - 0, 0); + return lms_outbound_enqueue_internal(worker_id, msg_type, dest_node_id, payload, payload_len, 0, + 0); } /* Stage a wire-version-sensitive frame for one exact HELLO-authenticated @@ -258,13 +258,12 @@ cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, bool cluster_lms_outbound_enqueue_cap_bound(int worker_id, uint8 msg_type, uint32 dest_node_id, const void *payload, uint16 payload_len, - uint32 required_capability, - uint32 connection_generation) + uint32 required_capability, uint32 connection_generation) { if (required_capability == 0 || dest_node_id >= CLUSTER_MAX_NODES) return false; return lms_outbound_enqueue_internal(worker_id, msg_type, dest_node_id, payload, payload_len, - required_capability, connection_generation); + required_capability, connection_generation); } /* @@ -276,15 +275,14 @@ cluster_lms_outbound_enqueue_cap_bound(int worker_id, uint8 msg_type, uint32 des */ bool cluster_lms_outbound_enqueue_zero_block_reply(int worker_id, uint32 dest_node_id, - const GcsBlockReplyHeader *header, bool direct_land) + const GcsBlockReplyHeader *header, bool direct_land) { ClusterLmsOutboundState *ring; LWLock *lock; ClusterLmsOutboundSlot *slot; - if (worker_id < 0 || worker_id >= CLUSTER_LMS_MAX_WORKERS - || dest_node_id >= CLUSTER_MAX_NODES || header == NULL - || (direct_land && (int32)dest_node_id == cluster_node_id) + if (worker_id < 0 || worker_id >= CLUSTER_LMS_MAX_WORKERS || dest_node_id >= CLUSTER_MAX_NODES + || header == NULL || (direct_land && (int32)dest_node_id == cluster_node_id) || header->status != (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X) return false; if (cluster_lms_outbound_rings == NULL || OB_LOCK(worker_id) == NULL) @@ -391,8 +389,7 @@ cluster_lms_outbound_drain_send(int worker_id) * no protocol ACK is generated, so the armed reliable leg retries. */ if (slot.required_capability != 0 && !cluster_sf_peer_capability_generation_matches( - (int32)slot.dest_node_id, slot.required_capability, - slot.connection_generation)) { + (int32)slot.dest_node_id, slot.required_capability, slot.connection_generation)) { lms_outbound_pcm_x_image_ready_note(&slot, "capability-guard", -1); cluster_lms_obs_note_outbound_cap_guard_drop(worker_id); continue; @@ -412,8 +409,7 @@ cluster_lms_outbound_drain_send(int worker_id) rc = CLUSTER_IC_SEND_HARD_ERROR; goto handle_send_result; } - zero_reply.header.checksum - = cluster_gcs_block_compute_checksum(zero_reply.block_data); + zero_reply.header.checksum = cluster_gcs_block_compute_checksum(zero_reply.block_data); send_payload = &zero_reply; send_payload_len = sizeof(zero_reply); } else if (slot.kind != (uint8)CLUSTER_LMS_OUTBOUND_FRAME) { @@ -459,23 +455,21 @@ cluster_lms_outbound_drain_send(int worker_id) */ if (slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_DIRECT_ZERO_BLOCK_REPLY) rc = cluster_gcs_block_send_direct_zero_reply((int32)slot.dest_node_id, - &zero_reply.header); + &zero_reply.header); else if ((int32)slot.dest_node_id == cluster_node_id) { ClusterICEnvelope env; - if (cluster_ic_envelope_build( - &env, slot.msg_type, (uint32)cluster_node_id, slot.dest_node_id, - send_payload, send_payload_len) - && cluster_ic_dispatch_envelope(&env, send_payload, - cluster_node_id)) + if (cluster_ic_envelope_build(&env, slot.msg_type, (uint32)cluster_node_id, + slot.dest_node_id, send_payload, send_payload_len) + && cluster_ic_dispatch_envelope(&env, send_payload, cluster_node_id)) rc = CLUSTER_IC_SEND_DONE; else rc = CLUSTER_IC_SEND_HARD_ERROR; } else - rc = cluster_ic_send_envelope(slot.msg_type, (int32)slot.dest_node_id, - send_payload, send_payload_len); + rc = cluster_ic_send_envelope(slot.msg_type, (int32)slot.dest_node_id, send_payload, + send_payload_len); -handle_send_result: + handle_send_result: lms_outbound_pcm_x_image_ready_note(&slot, "send-result", (int)rc); if (slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_ZERO_BLOCK_REPLY || slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_DIRECT_ZERO_BLOCK_REPLY) diff --git a/src/backend/cluster/cluster_pcm_lock.c b/src/backend/cluster/cluster_pcm_lock.c index 8f25bb7827..d80efcac8f 100644 --- a/src/backend/cluster/cluster_pcm_lock.c +++ b/src/backend/cluster/cluster_pcm_lock.c @@ -2852,9 +2852,8 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode, bool *out_ret Assert(out_retry_denied != NULL); if (out_retry_denied == NULL) - ereport(ERROR, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("cluster_pcm_lock_acquire_buffer: NULL retry result"))); + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cluster_pcm_lock_acquire_buffer: NULL retry result"))); *out_retry_denied = false; if (buf == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), @@ -2983,12 +2982,10 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode, bool *out_ret PcmAuthoritySnapshot authority; bool have_authority = cluster_pcm_lock_authority_snapshot(tag, &authority); - if (have_authority && authority.state == PCM_STATE_X - && authority.x_holder_node >= 0 && authority.x_holder_node != cluster_node_id - && authority.s_holders_bitmap == 0 + if (have_authority && authority.state == PCM_STATE_X && authority.x_holder_node >= 0 + && authority.x_holder_node != cluster_node_id && authority.s_holders_bitmap == 0 && authority.master_holder.node_id == (uint32)authority.x_holder_node) - return cluster_gcs_local_master_read_image_and_wait(buf, &authority, - out_retry_denied); + return cluster_gcs_local_master_read_image_and_wait(buf, &authority, out_retry_denied); } else /* mode == PCM_LOCK_MODE_X */ { /* @@ -3090,7 +3087,7 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode, bool *out_ret if (!pcm_lock_acquire_local(tag, mode, &raced_remote_x)) { if (mode == PCM_LOCK_MODE_S) return cluster_gcs_local_master_read_image_and_wait(buf, &raced_remote_x, - out_retry_denied); + out_retry_denied); return cluster_gcs_local_master_x_transfer_and_wait(buf, &raced_remote_x, clean_eligible, out_retry_denied); } diff --git a/src/backend/cluster/cluster_pcm_own.c b/src/backend/cluster/cluster_pcm_own.c index 320da62ae8..22777b7cd1 100644 --- a/src/backend/cluster/cluster_pcm_own.c +++ b/src/backend/cluster/cluster_pcm_own.c @@ -245,8 +245,8 @@ cluster_pcm_own_grant_commit_exact(int buf_id, uint64 expected_generation, uint6 ClusterPcmOwnResult cluster_pcm_own_writer_grant_commit_exact(int buf_id, uint64 expected_generation, - uint64 reservation_token, - uint64 *out_committed_generation) + uint64 reservation_token, + uint64 *out_committed_generation) { ClusterPcmOwnEntry *entry; ClusterPcmOwnResult live_result; @@ -290,7 +290,7 @@ cluster_pcm_own_writer_grant_commit_exact(int buf_id, uint64 expected_generation ClusterPcmOwnResult cluster_pcm_own_writer_activation_clear_exact(int buf_id, uint64 expected_generation, - uint64 reservation_token) + uint64 reservation_token) { ClusterPcmOwnEntry *entry; uint64 live_activation; diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index e5ffc8620f..712f81e855 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -1000,7 +1000,7 @@ cluster_pcm_x_requester_wait_once(PcmXPreSleepRevalidateCallback revalidate, Pcm PcmXQueueResult cluster_pcm_x_requester_wait_once_result(PcmXPreSleepResultCallback revalidate, - PcmXWaitCallback wait, void *callback_arg) + PcmXWaitCallback wait, void *callback_arg) { PcmXQueueResult result; @@ -1328,16 +1328,17 @@ cluster_pcm_x_local_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, memcpy(©, locked, sizeof(copy)); LWLockRelease(&header->local_locks[partition].lock); snprintf(buf, buflen, - "rel=%u fork=%d blk=%u flags=0x%x round=%u master=%d ref_node=%d ref_ticket=" UINT64_FORMAT - " ref_grant=" UINT64_FORMAT " holder_ticket=" UINT64_FORMAT - " blocker_ticket=" UINT64_FORMAT " blocker_gen=" UINT64_FORMAT + "rel=%u fork=%d blk=%u flags=0x%x round=%u master=%d ref_node=%d " + "ref_ticket=" UINT64_FORMAT " ref_grant=" UINT64_FORMAT + " holder_ticket=" UINT64_FORMAT " blocker_ticket=" UINT64_FORMAT + " blocker_gen=" UINT64_FORMAT " blocker_op=%u blocker_phase=%u blocker_last=%u blocker_last_node=%u" " blocker_tomb=0x%llx blocker_count=%u blocker_head=%ld blocker_next=%u" " blocker_crc=%u drain_gen=" UINT64_FORMAT " holder_drain_gen=" UINT64_FORMAT " members=%zu head=%ld leader=%ld active_writer=%ld", - copy.tag.relNumber, (int)copy.tag.forkNum, copy.tag.blockNum, pcm_x_slot_flags_read(©.slot), - copy.local_round, copy.master_node, copy.ref.identity.node_id, - copy.ref.handle.ticket_id, copy.ref.grant_generation, + copy.tag.relNumber, (int)copy.tag.forkNum, copy.tag.blockNum, + pcm_x_slot_flags_read(©.slot), copy.local_round, copy.master_node, + copy.ref.identity.node_id, copy.ref.handle.ticket_id, copy.ref.grant_generation, copy.holder_ref.handle.ticket_id, copy.blocker_snapshot_ref.handle.ticket_id, copy.blocker_set_generation, copy.blocker_snapshot_reliable.pending_opcode, copy.blocker_snapshot_reliable.phase, @@ -4851,10 +4852,11 @@ pcm_x_master_drive_capture_locked(PcmXRuntimeSnapshot runtime, PcmXMasterTagSlot * reclaimed in that gap. Those monotone successors consume the old drive * token; they are not evidence that the live queue is corrupt. */ static PcmXQueueResult -pcm_x_master_drive_expected_capture_locked( - PcmXRuntimeSnapshot runtime, PcmXMasterTagSlot *tag_slot, PcmXSlotRef tag_ref, - PcmXMasterTicketSlot *ticket, PcmXSlotRef ticket_ref, const PcmXTicketRef *expected_ref, - PcmXMasterDriveSnapshot *snapshot) +pcm_x_master_drive_expected_capture_locked(PcmXRuntimeSnapshot runtime, PcmXMasterTagSlot *tag_slot, + PcmXSlotRef tag_ref, PcmXMasterTicketSlot *ticket, + PcmXSlotRef ticket_ref, + const PcmXTicketRef *expected_ref, + PcmXMasterDriveSnapshot *snapshot) { uint32 state; @@ -4863,11 +4865,10 @@ pcm_x_master_drive_expected_capture_locked( if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, expected_ref)) return PCM_X_QUEUE_STALE; state = pcm_x_slot_state_read(&ticket->slot); - if (state == PCM_XT_COMPLETE || state == PCM_XT_CANCELLED - || state == PCM_XT_RETIRE_CREDIT) + if (state == PCM_XT_COMPLETE || state == PCM_XT_CANCELLED || state == PCM_XT_RETIRE_CREDIT) return PCM_X_QUEUE_NOT_READY; return pcm_x_master_drive_capture_locked(runtime, tag_slot, tag_ref, ticket, ticket_ref, - snapshot); + snapshot); } @@ -5010,8 +5011,8 @@ cluster_pcm_x_master_commit_retry_exact(const PcmXMasterDriveSnapshot *expected, ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, &expected->ref.identity.tag, PCM_X_MASTER_TICKET_DOMAIN_STATES); - result = pcm_x_master_drive_expected_capture_locked( - runtime, tag_slot, tag_ref, ticket, ticket_ref, &expected->ref, ¤t); + result = pcm_x_master_drive_expected_capture_locked(runtime, tag_slot, tag_ref, ticket, + ticket_ref, &expected->ref, ¤t); if (result != PCM_X_QUEUE_OK) goto retry_done; if (!pcm_x_master_drive_snapshot_equal(¤t, expected)) { @@ -5107,8 +5108,8 @@ cluster_pcm_x_master_invalidate_busy_backoff_exact(const PcmXMasterDriveSnapshot ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, &expected->ref.identity.tag, PCM_X_MASTER_TICKET_DOMAIN_STATES); - result = pcm_x_master_drive_expected_capture_locked( - runtime, tag_slot, tag_ref, ticket, ticket_ref, &expected->ref, ¤t); + result = pcm_x_master_drive_expected_capture_locked(runtime, tag_slot, tag_ref, ticket, + ticket_ref, &expected->ref, ¤t); if (result != PCM_X_QUEUE_OK) goto busy_done; if (!pcm_x_master_drive_snapshot_equal(¤t, expected)) { @@ -5119,12 +5120,11 @@ cluster_pcm_x_master_invalidate_busy_backoff_exact(const PcmXMasterDriveSnapshot ticket->reliable.expected_responder_node, ticket->reliable.expected_responder_session) || !pcm_x_transfer_peer_exact(ticket, ticket->reliable.expected_responder_node, - ticket->reliable.expected_responder_session)) { + ticket->reliable.expected_responder_session)) { result = PCM_X_QUEUE_STALE; goto busy_done; } - if (ticket->reliable.retry_deadline_ms != 0 - && now_ms < ticket->reliable.retry_deadline_ms) { + if (ticket->reliable.retry_deadline_ms != 0 && now_ms < ticket->reliable.retry_deadline_ms) { *snapshot_out = current; result = PCM_X_QUEUE_DUPLICATE; goto busy_done; @@ -5135,7 +5135,7 @@ cluster_pcm_x_master_invalidate_busy_backoff_exact(const PcmXMasterDriveSnapshot ticket->reliable.retry_deadline_ms = next_retry_deadline_ms; pg_write_barrier(); result = pcm_x_master_drive_capture_locked(runtime, tag_slot, tag_ref, ticket, ticket_ref, - snapshot_out); + snapshot_out); busy_done: LWLockRelease(&header->master_locks[partition].lock); @@ -5217,8 +5217,8 @@ cluster_pcm_x_master_drive_bitmap_replace_exact(const PcmXMasterDriveSnapshot *e ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, &expected->ref.identity.tag, PCM_X_MASTER_TICKET_DOMAIN_STATES); - result = pcm_x_master_drive_expected_capture_locked( - runtime, tag_slot, tag_ref, ticket, ticket_ref, &expected->ref, ¤t); + result = pcm_x_master_drive_expected_capture_locked(runtime, tag_slot, tag_ref, ticket, + ticket_ref, &expected->ref, ¤t); if (result != PCM_X_QUEUE_OK) goto replace_done; *snapshot_out = current; @@ -12503,17 +12503,16 @@ pcm_x_local_blocker_ack_tombstone_exact(const PcmXLocalTagSlot *tag_slot, * master/session COMMIT, with a fully empty canonical payload, is eligible. */ static bool pcm_x_local_empty_blocker_commit_exact(const PcmXLocalTagSlot *tag_slot, - int32 authenticated_master_node, - uint64 authenticated_master_session) + int32 authenticated_master_node, + uint64 authenticated_master_session) { return tag_slot != NULL && tag_slot->blocker_set_generation != 0 && tag_slot->blocker_set_generation != UINT64_MAX && tag_slot->blocker_snapshot_reliable.state_sequence != 0 && tag_slot->blocker_snapshot_reliable.response_tombstone_mask == 0 && pcm_x_transfer_leg_exact(&tag_slot->blocker_snapshot_reliable, - PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, - authenticated_master_node, - authenticated_master_session) + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, + authenticated_master_node, authenticated_master_session) && tag_slot->blocker_snapshot_head_index == PCM_X_INVALID_SLOT_INDEX && tag_slot->blocker_snapshot_count == 0 && tag_slot->blocker_snapshot_crc32c == 0 && tag_slot->blocker_snapshot_next_chunk == 0; @@ -13075,9 +13074,9 @@ cluster_pcm_x_local_blocker_ack_exact(const PcmXTicketRef *ref, uint64 set_gener PcmXQueueResult cluster_pcm_x_local_holder_revoke_apply_floor_exact(const PcmXRevokePayload *revoke, - uint64 required_page_scn, - int32 authenticated_master_node, - uint64 authenticated_master_session) + uint64 required_page_scn, + int32 authenticated_master_node, + uint64 authenticated_master_session) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; PcmXRuntimeSnapshot runtime; @@ -13186,20 +13185,20 @@ cluster_pcm_x_local_holder_revoke_apply_floor_exact(const PcmXRevokePayload *rev * ACK handler can nevertheless observe a transient local admission gate and * leave the exact COMMIT armed. Consume only that canonical empty set (for * the first or a replay generation), since no blocker edge can be omitted. */ - if (!pcm_x_local_empty_blocker_commit_exact( - tag_slot, authenticated_master_node, authenticated_master_session)) { + if (!pcm_x_local_empty_blocker_commit_exact(tag_slot, authenticated_master_node, + authenticated_master_session)) { result = PCM_X_QUEUE_NOT_READY; goto revoke_release_gate; } } else if (tag_slot->blocker_set_generation == 0 - || tag_slot->blocker_snapshot_reliable.state_sequence == 0 - || tag_slot->blocker_snapshot_reliable.last_response_opcode - != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK - || tag_slot->blocker_snapshot_reliable.last_responder_node - != (uint32)authenticated_master_node - || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX - || tag_slot->blocker_snapshot_count != 0 || tag_slot->blocker_snapshot_crc32c != 0 - || tag_slot->blocker_snapshot_next_chunk != 0) { + || tag_slot->blocker_snapshot_reliable.state_sequence == 0 + || tag_slot->blocker_snapshot_reliable.last_response_opcode + != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK + || tag_slot->blocker_snapshot_reliable.last_responder_node + != (uint32)authenticated_master_node + || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_count != 0 || tag_slot->blocker_snapshot_crc32c != 0 + || tag_slot->blocker_snapshot_next_chunk != 0) { result = PCM_X_QUEUE_CORRUPT; fail_closed = true; goto revoke_release_gate; @@ -13253,8 +13252,8 @@ cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, int32 authenticated_master_node, uint64 authenticated_master_session) { - return cluster_pcm_x_local_holder_revoke_apply_floor_exact( - revoke, 0, authenticated_master_node, authenticated_master_session); + return cluster_pcm_x_local_holder_revoke_apply_floor_exact(revoke, 0, authenticated_master_node, + authenticated_master_session); } @@ -13404,12 +13403,12 @@ cluster_pcm_x_local_queue_invalidate_authorize_exact(const BufferTag *tag, uint6 * passive pinned S become N+PI instead of waiting forever for an ACK * the advanced master is no longer required to replay. */ if (!pcm_x_transfer_leg_exact(&tag_slot->blocker_snapshot_reliable, - PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, - authenticated_master_node, authenticated_master_session) + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, + authenticated_master_node, authenticated_master_session) || tag_slot->blocker_snapshot_count != 0) result = PCM_X_QUEUE_NOT_READY; - else if (!pcm_x_local_empty_blocker_commit_exact( - tag_slot, authenticated_master_node, authenticated_master_session) + else if (!pcm_x_local_empty_blocker_commit_exact(tag_slot, authenticated_master_node, + authenticated_master_session) || (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 || tag_slot->holder_terminal_drain_generation != 0) { result = PCM_X_QUEUE_CORRUPT; @@ -13417,16 +13416,17 @@ cluster_pcm_x_local_queue_invalidate_authorize_exact(const BufferTag *tag, uint6 } else result = PCM_X_QUEUE_OK; } else if (tag_slot->blocker_snapshot_reliable.state_sequence == 0 - || tag_slot->blocker_snapshot_reliable.last_response_opcode - != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK - || tag_slot->blocker_snapshot_reliable.last_responder_node - != (uint32)authenticated_master_node - || tag_slot->blocker_snapshot_reliable.response_tombstone_mask != 0 - || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX - || tag_slot->blocker_snapshot_count != 0 || tag_slot->blocker_snapshot_crc32c != 0 - || tag_slot->blocker_snapshot_next_chunk != 0 - || (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 - || tag_slot->holder_terminal_drain_generation != 0) { + || tag_slot->blocker_snapshot_reliable.last_response_opcode + != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK + || tag_slot->blocker_snapshot_reliable.last_responder_node + != (uint32)authenticated_master_node + || tag_slot->blocker_snapshot_reliable.response_tombstone_mask != 0 + || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_count != 0 + || tag_slot->blocker_snapshot_crc32c != 0 + || tag_slot->blocker_snapshot_next_chunk != 0 + || (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 + || tag_slot->holder_terminal_drain_generation != 0) { result = PCM_X_QUEUE_CORRUPT; fail_closed = true; } else @@ -13440,10 +13440,11 @@ cluster_pcm_x_local_queue_invalidate_authorize_exact(const BufferTag *tag, uint6 PcmXQueueResult -cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( - const PcmXGrantPayload *image_ready, int32 authenticated_master_node, - uint64 authenticated_master_session, PcmXGrantPayload *replay_out, - PcmXLocalImageReadyRefusal *refusal_out) +cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed(const PcmXGrantPayload *image_ready, + int32 authenticated_master_node, + uint64 authenticated_master_session, + PcmXGrantPayload *replay_out, + PcmXLocalImageReadyRefusal *refusal_out) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; PcmXRuntimeSnapshot runtime; @@ -13518,7 +13519,7 @@ cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( goto ready_release_gate; } if (!pcm_x_local_holder_transfer_peer_exact(tag_slot, authenticated_master_node, - authenticated_master_session) + authenticated_master_session) || !pcm_x_ticket_ref_equal(&tag_slot->holder_ref, &image_ready->ref) || tag_slot->holder_image.image_id != image_ready->image.image_id) { if (refusal_out != NULL) @@ -18698,8 +18699,8 @@ pcm_x_local_blocker_participant_drain_locked(PcmXLocalTagSlot *tag_slot, PcmXSlo || tag_slot->blocker_snapshot_ref.identity.cluster_epoch != tag_slot->cluster_epoch) return PCM_X_QUEUE_CORRUPT; if ((!pcm_x_local_blocker_ack_tombstone_exact(tag_slot, authenticated_master_node) - && !pcm_x_local_empty_blocker_commit_exact( - tag_slot, authenticated_master_node, authenticated_master_session)) + && !pcm_x_local_empty_blocker_commit_exact(tag_slot, authenticated_master_node, + authenticated_master_session)) || tag_slot->blocker_snapshot_reliable.response_tombstone_mask != 0) { blocker_state = pcm_x_local_blocker_lane_retire_state(tag_slot); return blocker_state == PCM_X_QUEUE_CORRUPT ? blocker_state : PCM_X_QUEUE_NOT_READY; @@ -18781,9 +18782,8 @@ pcm_x_local_holder_drain_poll_exact(const PcmXDrainPollPayload *poll, goto holder_drain_done; } if (pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref)) { - result = pcm_x_local_blocker_participant_drain_locked(tag_slot, tag_ref, poll, - authenticated_master_node, - authenticated_master_session); + result = pcm_x_local_blocker_participant_drain_locked( + tag_slot, tag_ref, poll, authenticated_master_node, authenticated_master_session); fail_closed = result == PCM_X_QUEUE_CORRUPT; goto holder_drain_done; } diff --git a/src/backend/cluster/cluster_pcm_x_image_fetch.c b/src/backend/cluster/cluster_pcm_x_image_fetch.c index 3812cf3211..53ce06d958 100644 --- a/src/backend/cluster/cluster_pcm_x_image_fetch.c +++ b/src/backend/cluster/cluster_pcm_x_image_fetch.c @@ -86,10 +86,12 @@ cluster_pcm_x_image_fetch_build_request(const PcmXLocalProgress *progress, int32 bool -cluster_pcm_x_image_fetch_request_exact_diagnosed( - const ClusterICEnvelope *env, const GcsBlockRequestPayload *request, - const PcmXLocalHolderProgress *holder, int32 holder_node, int32 current_master_node, - uint64 current_epoch, PcmXImageFetchRequestRefusal *refusal_out) +cluster_pcm_x_image_fetch_request_exact_diagnosed(const ClusterICEnvelope *env, + const GcsBlockRequestPayload *request, + const PcmXLocalHolderProgress *holder, + int32 holder_node, int32 current_master_node, + uint64 current_epoch, + PcmXImageFetchRequestRefusal *refusal_out) { static const uint8 zero_reserved[sizeof(request->reserved_0)] = { 0 }; int32 decoded_backend_id; @@ -148,8 +150,8 @@ cluster_pcm_x_image_fetch_request_exact_diagnosed( *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_IMAGE; return false; } - if (!cluster_gcs_requester_id_decode(holder->ref.identity.request_id, - &decoded_requester_node, &decoded_backend_id, NULL) + if (!cluster_gcs_requester_id_decode(holder->ref.identity.request_id, &decoded_requester_node, + &decoded_backend_id, NULL) || decoded_requester_node != request->sender_node || decoded_backend_id != request->requester_backend_id) { if (refusal_out != NULL) diff --git a/src/backend/cluster/cluster_sf_dep.c b/src/backend/cluster/cluster_sf_dep.c index 1cea155b8f..f1193388f3 100644 --- a/src/backend/cluster/cluster_sf_dep.c +++ b/src/backend/cluster/cluster_sf_dep.c @@ -407,7 +407,7 @@ cluster_sf_peer_supports_pcm_x_source_floor(int32 peer_id) * reconnect. */ bool cluster_sf_peer_pcm_x_source_floor_sample(int32 peer_id, bool *source_floor_out, - uint32 *generation_out) + uint32 *generation_out) { bool supported; @@ -429,7 +429,7 @@ cluster_sf_peer_pcm_x_source_floor_sample(int32 peer_id, bool *source_floor_out, /* Drain-side exact fence for a capability-bound LMS slot. */ bool cluster_sf_peer_capability_generation_matches(int32 peer_id, uint32 required_capabilities, - uint32 expected_generation) + uint32 expected_generation) { bool matches; @@ -438,8 +438,7 @@ cluster_sf_peer_capability_generation_matches(int32 peer_id, uint32 required_cap return false; LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); matches = cluster_sf_peer_cap_generation_matches_exact( - &ClusterSfDep->peer_capabilities[peer_id], required_capabilities, - expected_generation); + &ClusterSfDep->peer_capabilities[peer_id], required_capabilities, expected_generation); LWLockRelease(&ClusterSfDep->lock); return matches; } diff --git a/src/backend/cluster/cluster_tt_local.c b/src/backend/cluster/cluster_tt_local.c index 34f02775fa..088d8c9c89 100644 --- a/src/backend/cluster/cluster_tt_local.c +++ b/src/backend/cluster/cluster_tt_local.c @@ -57,7 +57,7 @@ #include "cluster/cluster_undo_record_api.h" /* spec-3.12 D2b cluster_undo_tt_rollover_locked */ #include "cluster/cluster_tt_status.h" #include "cluster/cluster_tt_status_hint.h" /* spec-3.2 D4 wire emit append */ -#include "cluster/cluster_uba.h" /* P0-33 exact data-ref alias */ +#include "cluster/cluster_uba.h" /* P0-33 exact data-ref alias */ #include "cluster/storage/cluster_undo_alloc.h" /* cluster_undo_active_segment_for_node_or_create */ #ifdef USE_PGRAC_CLUSTER @@ -477,12 +477,10 @@ cluster_tt_local_finish_bindings(bool committed, SCN commit_scn) /* Compose either the canonical key or one page-ref segment alias. */ static bool -build_binding_key(const ClusterTTLocalBinding *binding, uint16 segment_id, - ClusterTTStatusKey *out) +build_binding_key(const ClusterTTLocalBinding *binding, uint16 segment_id, ClusterTTStatusKey *out) { memset(out, 0, sizeof(*out)); - if (binding == NULL || !TransactionIdIsValid(binding->top_xid) - || segment_id == 0) + if (binding == NULL || !TransactionIdIsValid(binding->top_xid) || segment_id == 0) return false; out->origin_node_id = (uint16)cluster_node_id; @@ -498,13 +496,12 @@ build_local_key(TransactionId xid, ClusterTTStatusKey *out) { int idx = cluster_tt_local_find_binding(xid); - if (idx < 0) - { + if (idx < 0) { memset(out, 0, sizeof(*out)); return false; } return build_binding_key(&cluster_tt_local_bindings[idx], - (uint16)cluster_tt_local_bindings[idx].segment_id, out); + (uint16)cluster_tt_local_bindings[idx].segment_id, out); } /* Install and emit one already-minted exact key. */ @@ -522,8 +519,7 @@ install_key(const ClusterTTStatusKey *key, ClusterTTStatus status, SCN commit_sc bool hit = false; bool epoch_stable = ((uint32)cluster_epoch_get_current() == key->cluster_epoch); - if (epoch_stable) - { + if (epoch_stable) { hit = cluster_tt_status_lookup_exact(key, &res); epoch_stable = ((uint32)cluster_epoch_get_current() == key->cluster_epoch); } @@ -538,13 +534,12 @@ install_key(const ClusterTTStatusKey *key, ClusterTTStatus status, SCN commit_sc } static void -install_binding_aliases(const ClusterTTLocalBinding *binding, - ClusterTTStatus status, SCN commit_scn) +install_binding_aliases(const ClusterTTLocalBinding *binding, ClusterTTStatus status, + SCN commit_scn) { uint16 i; - for (i = 0; i < binding->active_alias_count; i++) - { + for (i = 0; i < binding->active_alias_count; i++) { ClusterTTStatusKey key; if (build_binding_key(binding, binding->active_alias_segments[i], &key)) @@ -727,41 +722,34 @@ cluster_tt_local_record_data_active(TransactionId xid, UBA uba) * If that invariant is ever broken, publish no guessed alias. */ if (!uba_decode(uba, &record_segment, &block_no, &slot_offset, &row_offset) || uba_origin_node_id(uba) != (NodeId)cluster_node_id - || slot_offset != binding->slot_offset) - { + || slot_offset != binding->slot_offset) { Assert(false); return; } (void)block_no; (void)row_offset; - if (record_segment != binding->segment_id) - { + if (record_segment != binding->segment_id) { for (i = 0; i < binding->active_alias_count; i++) if (binding->active_alias_segments[i] == (uint16)record_segment) break; - if (i == binding->active_alias_count) - { + if (i == binding->active_alias_count) { MemoryContext oldcxt; oldcxt = MemoryContextSwitchTo(TopTransactionContext); - if (binding->active_alias_segments == NULL) - { + if (binding->active_alias_segments == NULL) { binding->active_alias_capacity = 4; - binding->active_alias_segments = (uint16 *)palloc( - sizeof(uint16) * binding->active_alias_capacity); - } - else if (binding->active_alias_count == binding->active_alias_capacity) - { + binding->active_alias_segments + = (uint16 *)palloc(sizeof(uint16) * binding->active_alias_capacity); + } else if (binding->active_alias_count == binding->active_alias_capacity) { binding->active_alias_capacity *= 2; - binding->active_alias_segments = (uint16 *)repalloc( - binding->active_alias_segments, - sizeof(uint16) * binding->active_alias_capacity); + binding->active_alias_segments + = (uint16 *)repalloc(binding->active_alias_segments, + sizeof(uint16) * binding->active_alias_capacity); } MemoryContextSwitchTo(oldcxt); - binding->active_alias_segments[binding->active_alias_count++] - = (uint16)record_segment; + binding->active_alias_segments[binding->active_alias_count++] = (uint16)record_segment; } } diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index e2aaed2997..12ea0e7533 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -953,9 +953,9 @@ cluster_gcs_pcm_x_revoke_ingress_valid(const PcmXRevokePayload *request, Size pa { return request != NULL && (payload_length == sizeof(*request) || payload_length == sizeof(PcmXRevokePayloadV2)) - && authenticated_node >= 0 - && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 - && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == authenticated_node + && authenticated_node >= 0 && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT + && local_node >= 0 && local_node < PCM_X_PROTOCOL_NODE_LIMIT + && tag_master == authenticated_node && cluster_gcs_pcm_x_transfer_ref_wire_valid(&request->ref, current_epoch) && cluster_gcs_pcm_x_image_id_master_wire_valid(request->image_id, authenticated_node); } @@ -1314,14 +1314,13 @@ GcsBlockLocalPendingSDenialMatches(bool in_use, bool reply_received, bool stale, uint8 transition_id, const BufferTag *slot_tag, uint64 request_epoch, int32 expected_master_node, ClusterGcsBlockDirectState direct_state, - bool direct_target_prepared, - const BufferTag *invalidate_tag, uint64 invalidate_epoch, - int32 invalidate_master_node) + bool direct_target_prepared, const BufferTag *invalidate_tag, + uint64 invalidate_epoch, int32 invalidate_master_node) { return in_use && !reply_received && !stale && slot_tag != NULL && invalidate_tag != NULL - && transition_id == (uint8)PCM_TRANS_N_TO_S - && BufferTagsEqual(slot_tag, invalidate_tag) && request_epoch == invalidate_epoch - && expected_master_node == invalidate_master_node && !direct_target_prepared + && transition_id == (uint8)PCM_TRANS_N_TO_S && BufferTagsEqual(slot_tag, invalidate_tag) + && request_epoch == invalidate_epoch && expected_master_node == invalidate_master_node + && !direct_target_prepared && (direct_state == GCS_BLOCK_DIRECT_UNARMED || direct_state == GCS_BLOCK_DIRECT_ABORTED); } @@ -3280,10 +3279,8 @@ typedef enum ClusterBufmgrGcsDowngradeOutcome { CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY } ClusterBufmgrGcsDowngradeOutcome; extern bool cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag); -extern ClusterBufmgrGcsDowngradeOutcome -cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image( - BufferTag tag, XLogRecPtr *out_page_lsn, char *dst, - ClusterBufmgrGcsCopyRefusal *out_refusal); +extern ClusterBufmgrGcsDowngradeOutcome cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image( + BufferTag tag, XLogRecPtr *out_page_lsn, char *dst, ClusterBufmgrGcsCopyRefusal *out_refusal); extern bool cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node); extern ClusterBufmgrGcsDowngradeOutcome cluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image( @@ -3383,9 +3380,9 @@ extern int cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, * Terminal denials ereport(ERROR) and do not return. */ extern bool cluster_gcs_send_block_request_and_wait(BufferDesc *buf, - PcmLockTransition transition_id, - int master_node, bool clean_eligible, - bool *out_retry_denied); + PcmLockTransition transition_id, + int master_node, bool clean_eligible, + bool *out_retry_denied); /* * spec-5.2 D2 (sub-case B) — local-master read-image forward. Used by @@ -3398,8 +3395,8 @@ extern bool cluster_gcs_send_block_request_and_wait(BufferDesc *buf, * bufmgr aborts/rearms GRANT_PENDING and selects a fresh holder identity. */ extern bool cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, - const PcmAuthoritySnapshot *expected, - bool *out_retry_denied); + const PcmAuthoritySnapshot *expected, + bool *out_retry_denied); /* PGRAC: spec-5.2 D11 — local-master writer-transfer (revoke); durable X grant. * spec-5.2a D2/D3: clean_eligible routes a clean (sequence) page through the * flush-data-before-drop holder path + stale-holder storage-fallback recovery. @@ -3535,8 +3532,8 @@ typedef enum GcsBlockSendFamily { } GcsBlockSendFamily; extern void cluster_gcs_block_note_send_outcome(GcsBlockSendFamily family, ClusterICSendResult rc); -extern ClusterICSendResult cluster_gcs_block_send_direct_zero_reply( - int32 dest_node, const GcsBlockReplyHeader *header); +extern ClusterICSendResult +cluster_gcs_block_send_direct_zero_reply(int32 dest_node, const GcsBlockReplyHeader *header); extern uint64 cluster_gcs_get_reply_send_queued_count(void); extern uint64 cluster_gcs_get_reply_send_not_admitted_count(void); diff --git a/src/include/cluster/cluster_gcs_block_dedup.h b/src/include/cluster/cluster_gcs_block_dedup.h index 62c7ab29dc..92624eefd3 100644 --- a/src/include/cluster/cluster_gcs_block_dedup.h +++ b/src/include/cluster/cluster_gcs_block_dedup.h @@ -489,9 +489,11 @@ extern GcsBlockPendingXDenyResult cluster_gcs_block_dedup_pending_x_deny_exact(int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, uint8 transition_id, GcsBlockDedupEntry *denied_out); -extern bool cluster_gcs_block_dedup_set_request_flags_exact( - int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, uint8 transition_id, - uint8 request_flags); +extern bool cluster_gcs_block_dedup_set_request_flags_exact(int worker_id, + const GcsBlockDedupKey *key, + const BufferTag *tag, + uint8 transition_id, + uint8 request_flags); /* Dedicated PCM-X image storage over the existing dedup entry pool. Reserve * claims capacity before the revoke lifecycle starts. Materialize publishes diff --git a/src/include/cluster/cluster_lms.h b/src/include/cluster/cluster_lms.h index b79242c125..ba694f242d 100644 --- a/src/include/cluster/cluster_lms.h +++ b/src/include/cluster/cluster_lms.h @@ -405,18 +405,21 @@ extern void cluster_lms_outbound_request_lwlocks(void); extern bool cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, const void *payload, uint16 payload_len); extern bool cluster_lms_outbound_enqueue_cap_bound(int worker_id, uint8 msg_type, - uint32 dest_node_id, const void *payload, - uint16 payload_len, uint32 required_capability, - uint32 connection_generation); + uint32 dest_node_id, const void *payload, + uint16 payload_len, uint32 required_capability, + uint32 connection_generation); struct GcsBlockReplyHeader; -extern bool cluster_lms_outbound_enqueue_zero_block_reply( - int worker_id, uint32 dest_node_id, const struct GcsBlockReplyHeader *header, bool direct_land); +extern bool cluster_lms_outbound_enqueue_zero_block_reply(int worker_id, uint32 dest_node_id, + const struct GcsBlockReplyHeader *header, + bool direct_land); extern int cluster_lms_outbound_drain_send(int worker_id); extern uint32 cluster_lms_outbound_depth(int worker_id); -extern void cluster_lms_note_pcm_x_image_ready_boundary( - uint8 msg_type, const char *boundary, int result, int runtime_state, bool fence_enforcing, - bool fence_allowed, uint32 dest_node_id, uint64 request_id, uint64 ticket_id, uint64 grant_generation, - uint64 image_id); +extern void cluster_lms_note_pcm_x_image_ready_boundary(uint8 msg_type, const char *boundary, + int result, int runtime_state, + bool fence_enforcing, bool fence_allowed, + uint32 dest_node_id, uint64 request_id, + uint64 ticket_id, uint64 grant_generation, + uint64 image_id); /* * Read-only accessors for SQL view + diagnostics. diff --git a/src/include/cluster/cluster_pcm_own.h b/src/include/cluster/cluster_pcm_own.h index cfe8bbc14c..ac7300730b 100644 --- a/src/include/cluster/cluster_pcm_own.h +++ b/src/include/cluster/cluster_pcm_own.h @@ -61,11 +61,11 @@ #define PCM_OWN_FLAG_REVOKING ((uint32)0x2) /* a revoke (downgrade/invalidate) started */ typedef struct ClusterPcmOwnEntry { - pg_atomic_uint64 generation; /* monotone; bumped on every committed transition */ - pg_atomic_uint64 reservation_token; /* monotone; active iff a transient flag is set */ + pg_atomic_uint64 generation; /* monotone; bumped on every committed transition */ + pg_atomic_uint64 reservation_token; /* monotone; active iff a transient flag is set */ pg_atomic_uint64 writer_activation_token; /* committed X not yet activated under content X */ - pg_atomic_uint32 flags; /* PCM_OWN_FLAG_* */ - uint32 _pad; /* keep 32B aligned */ + pg_atomic_uint32 flags; /* PCM_OWN_FLAG_* */ + uint32 _pad; /* keep 32B aligned */ } ClusterPcmOwnEntry; StaticAssertDecl(sizeof(ClusterPcmOwnEntry) == 32, "ClusterPcmOwnEntry must remain 32 bytes"); @@ -136,13 +136,15 @@ extern ClusterPcmOwnResult cluster_pcm_own_revoke_commit_exact(int buf_id, /* X-only combined linearization: clear the exact GRANT_PENDING lifecycle, * bump its ownership generation, and publish the writer activation fence * before the caller releases the BufferDesc header lock. */ -extern ClusterPcmOwnResult cluster_pcm_own_writer_grant_commit_exact( - int buf_id, uint64 expected_generation, uint64 reservation_token, - uint64 *out_committed_generation); +extern ClusterPcmOwnResult +cluster_pcm_own_writer_grant_commit_exact(int buf_id, uint64 expected_generation, + uint64 reservation_token, + uint64 *out_committed_generation); /* Clear only the exact committed generation/token after content-EXCLUSIVE * activation proof or after exact queue-claim cleanup. */ -extern ClusterPcmOwnResult cluster_pcm_own_writer_activation_clear_exact( - int buf_id, uint64 expected_generation, uint64 reservation_token); +extern ClusterPcmOwnResult cluster_pcm_own_writer_activation_clear_exact(int buf_id, + uint64 expected_generation, + uint64 reservation_token); /* PCM-X retained-image revoke: commit advances the ownership generation but * deliberately leaves the exact REVOKING token live until DRAIN proves the * immutable image record is no longer needed. Release clears only that diff --git a/src/include/cluster/cluster_pcm_x_bufmgr.h b/src/include/cluster/cluster_pcm_x_bufmgr.h index 15e544e526..65306ef698 100644 --- a/src/include/cluster/cluster_pcm_x_bufmgr.h +++ b/src/include/cluster/cluster_pcm_x_bufmgr.h @@ -86,8 +86,7 @@ cluster_pcm_x_holder_register_retry_action(PcmXQueueResult result, bool runtime_ if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) return CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE; if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BARRIER_CLOSED - || (runtime_active - && (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY))) + || (runtime_active && (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY))) return CLUSTER_PCM_X_HOLDER_RETRY_WAIT; return CLUSTER_PCM_X_HOLDER_RETRY_FAIL; } @@ -233,8 +232,8 @@ cluster_pcm_x_cached_cover_bypasses_queue(bool local_cache, bool requested_x, ui * ownership round. */ static inline bool cluster_pcm_x_cached_cover_reverify_accepts(uint8 requested_state, uint64 captured_generation, - uint64 current_generation, uint8 current_state, - uint32 current_flags) + uint64 current_generation, uint8 current_state, + uint32 current_flags) { bool covers; @@ -243,11 +242,9 @@ cluster_pcm_x_cached_cover_reverify_accepts(uint8 requested_state, uint64 captur if (current_flags != 0) return false; covers = current_state == (uint8)PCM_STATE_X - || (requested_state == (uint8)PCM_STATE_S - && current_state == (uint8)PCM_STATE_S); + || (requested_state == (uint8)PCM_STATE_S && current_state == (uint8)PCM_STATE_S); return covers - && (requested_state == (uint8)PCM_STATE_S - || current_generation == captured_generation); + && (requested_state == (uint8)PCM_STATE_S || current_generation == captured_generation); } /* ConditionalLockBuffer cannot initiate a PCM conversion. Preserve native @@ -484,9 +481,8 @@ cluster_bufmgr_pcm_own_begin_s_revoke(BufferDesc *buf, const ClusterPcmOwnSnapsh * hard floor; V1's zero floor still prefers the newer observed copy. */ extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_prepare_s_source_image( BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_s, SCN required_page_scn, - ClusterPcmOwnSnapshot *out_revoking, char block_data[BLCKSZ], - XLogRecPtr *out_page_lsn, uint64 *out_page_scn, - ClusterPcmOwnSourcePrepareRefusal *out_refusal); + ClusterPcmOwnSnapshot *out_revoking, char block_data[BLCKSZ], XLogRecPtr *out_page_lsn, + uint64 *out_page_scn, ClusterPcmOwnSourcePrepareRefusal *out_refusal); extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_abort_s_revoke(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking); diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h index 7f4d836324..35360cd2fe 100644 --- a/src/include/cluster/cluster_pcm_x_convert.h +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -1382,7 +1382,7 @@ extern bool cluster_pcm_x_requester_wait_once(PcmXPreSleepRevalidateCallback rev PcmXWaitCallback wait, void *callback_arg); extern PcmXQueueResult cluster_pcm_x_requester_wait_once_result(PcmXPreSleepResultCallback revalidate, - PcmXWaitCallback wait, void *callback_arg); + PcmXWaitCallback wait, void *callback_arg); extern void cluster_pcm_x_stats_note_queue_result(PcmXQueueResult result); extern void cluster_pcm_x_stats_note_own_begin(void); extern void cluster_pcm_x_stats_note_own_commit(void); @@ -1466,8 +1466,8 @@ cluster_pcm_x_master_commit_retry_exact(const PcmXMasterDriveSnapshot *expected, uint64 next_retry_deadline_ms, PcmXPhasePayload *commit_out, PcmXMasterDriveSnapshot *snapshot_out); extern PcmXQueueResult cluster_pcm_x_master_invalidate_busy_backoff_exact( - const PcmXMasterDriveSnapshot *expected, int32 busy_node, uint64 now_ms, - uint64 retry_delay_ms, PcmXMasterDriveSnapshot *snapshot_out); + const PcmXMasterDriveSnapshot *expected, int32 busy_node, uint64 now_ms, uint64 retry_delay_ms, + PcmXMasterDriveSnapshot *snapshot_out); typedef bool (*PcmXStageFrameCallback)(uint8 msg_type, int32 dest_node_id, const void *payload, Size payload_len, void *callback_arg); extern PcmXQueueResult @@ -1639,11 +1639,9 @@ extern PcmXQueueResult cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, int32 authenticated_master_node, uint64 authenticated_master_session); -extern PcmXQueueResult -cluster_pcm_x_local_holder_revoke_apply_floor_exact(const PcmXRevokePayload *revoke, - uint64 required_page_scn, - int32 authenticated_master_node, - uint64 authenticated_master_session); +extern PcmXQueueResult cluster_pcm_x_local_holder_revoke_apply_floor_exact( + const PcmXRevokePayload *revoke, uint64 required_page_scn, int32 authenticated_master_node, + uint64 authenticated_master_session); extern PcmXQueueResult cluster_pcm_x_local_holder_image_ready_arm_exact( const PcmXGrantPayload *image_ready, int32 authenticated_master_node, uint64 authenticated_master_session, PcmXGrantPayload *replay_out); diff --git a/src/include/cluster/cluster_sf_dep.h b/src/include/cluster/cluster_sf_dep.h index 286bd470fa..24acea633d 100644 --- a/src/include/cluster/cluster_sf_dep.h +++ b/src/include/cluster/cluster_sf_dep.h @@ -273,10 +273,10 @@ extern bool cluster_sf_peer_supports_pcm_x_convert(int32 peer_id); extern bool cluster_sf_peer_supports_pcm_x_rebase(int32 peer_id); extern bool cluster_sf_peer_supports_pcm_x_source_floor(int32 peer_id); extern bool cluster_sf_peer_pcm_x_source_floor_sample(int32 peer_id, bool *source_floor_out, - uint32 *generation_out); + uint32 *generation_out); extern bool cluster_sf_peer_capability_generation_matches(int32 peer_id, - uint32 required_capabilities, - uint32 expected_generation); + uint32 required_capabilities, + uint32 expected_generation); /* review P0-2: lock-coherent (CONVERT supported, REBASE bit, record * generation) triple for the formation collector's double sample. */ extern bool cluster_sf_peer_pcm_x_capability_sample(int32 peer_id, bool *rebase_out, diff --git a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c index 62959db442..985770101f 100644 --- a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c +++ b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c @@ -379,7 +379,8 @@ cluster_gcs_send_block_request_and_wait(struct BufferDesc *buf pg_attribute_unus /* spec-5.2 D2 sub-case B stub: local-master read-image forward unreachable. */ bool cluster_gcs_local_master_read_image_and_wait(struct BufferDesc *buf pg_attribute_unused(), - const PcmAuthoritySnapshot *expected pg_attribute_unused(), + const PcmAuthoritySnapshot *expected + pg_attribute_unused(), bool *out_retry_denied pg_attribute_unused()) { abort(); diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index e7261acec3..7b2c86f6bf 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -2042,9 +2042,9 @@ UT_TEST(test_pcm_x_invalidate_busy_routes_to_exact_ticket_backoff) received = backoff != NULL ? strstr(backoff, "invalidate_busy_received_count") : NULL; queue_return = received != NULL ? strstr(received, "if (!queue_positive)") : NULL; legacy_busy = queue_return != NULL - ? strstr(queue_return, - "ack->ack_status == GCS_BLOCK_INVALIDATE_ACK_STATUS_RETRYABLE_BUSY") - : NULL; + ? strstr(queue_return, + "ack->ack_status == GCS_BLOCK_INVALIDATE_ACK_STATUS_RETRYABLE_BUSY") + : NULL; UT_ASSERT_NOT_NULL(handler); UT_ASSERT_NOT_NULL(queue_busy); UT_ASSERT_NOT_NULL(backoff); @@ -2071,8 +2071,7 @@ UT_TEST(test_pcm_x_invalidate_busy_routes_to_exact_ticket_backoff) * retry at 25s and phase-lock the denied reader's GRANT_PENDING window. */ delay_helper = strstr(source, "\ngcs_block_pcm_x_invalidate_busy_retry_delay_ms("); delay_end = delay_helper != NULL ? strstr(delay_helper, "\n}") : NULL; - leg_retry_delay - = delay_helper != NULL ? strstr(delay_helper, "snapshot->retry_count") : NULL; + leg_retry_delay = delay_helper != NULL ? strstr(delay_helper, "snapshot->retry_count") : NULL; UT_ASSERT_NOT_NULL(delay_helper); UT_ASSERT_NOT_NULL(delay_end); if (delay_helper != NULL && delay_end != NULL) @@ -2084,9 +2083,9 @@ UT_TEST(test_pcm_x_invalidate_busy_routes_to_exact_ticket_backoff) execute_end = execute != NULL ? strstr(execute, "\n}\n") : NULL; self_capable_first = execute != NULL ? strstr(execute, "inv->master_node == cluster_node_id") : NULL; - self_capable_second = self_capable_first != NULL - ? strstr(self_capable_first + 1, "inv->master_node == cluster_node_id") - : NULL; + self_capable_second = self_capable_first != NULL ? strstr(self_capable_first + 1, + "inv->master_node == cluster_node_id") + : NULL; UT_ASSERT_NOT_NULL(execute); UT_ASSERT_NOT_NULL(execute_end); UT_ASSERT_NOT_NULL(self_capable_first); @@ -2169,28 +2168,25 @@ UT_TEST(test_pcm_x_grant_pending_invalidate_wakes_local_s_before_busy) if (source == NULL) return; direct_prepare = strstr(source, "\ngcs_block_direct_prepare_attempt("); - direct_prepare_end - = direct_prepare != NULL ? strstr(direct_prepare, "\n}\n\n\n") : NULL; + direct_prepare_end = direct_prepare != NULL ? strstr(direct_prepare, "\n}\n\n\n") : NULL; direct_first_reply_check = direct_prepare != NULL ? strstr(direct_prepare, "slot->reply_received") : NULL; direct_target_prepare = direct_first_reply_check != NULL - ? strstr(direct_first_reply_check, - "cluster_bufmgr_prepare_direct_land_target_for_gcs(") - : NULL; - direct_second_reply_check = direct_target_prepare != NULL - ? strstr(direct_target_prepare, "slot->reply_received") + ? strstr(direct_first_reply_check, + "cluster_bufmgr_prepare_direct_land_target_for_gcs(") : NULL; - direct_target_cleanup = direct_second_reply_check != NULL - ? strstr(direct_second_reply_check, - "gcs_block_direct_finish_target(buf, true, false") - : NULL; + direct_second_reply_check = direct_target_prepare != NULL + ? strstr(direct_target_prepare, "slot->reply_received") + : NULL; + direct_target_cleanup + = direct_second_reply_check != NULL + ? strstr(direct_second_reply_check, "gcs_block_direct_finish_target(buf, true, false") + : NULL; helper = strstr(source, "\ngcs_block_wake_local_pending_s_request("); execute = strstr(source, "\ngcs_block_invalidate_execute("); - pending = execute != NULL ? strstr(execute, "cluster_bufmgr_block_grant_pending(inv->tag)") - : NULL; - wake = pending != NULL - ? strstr(pending, "gcs_block_wake_local_pending_s_request(inv)") - : NULL; + pending + = execute != NULL ? strstr(execute, "cluster_bufmgr_block_grant_pending(inv->tag)") : NULL; + wake = pending != NULL ? strstr(pending, "gcs_block_wake_local_pending_s_request(inv)") : NULL; busy = wake != NULL ? strstr(wake, "invalidate_busy_sent_count") : NULL; UT_ASSERT_NOT_NULL(helper); UT_ASSERT_NOT_NULL(direct_prepare); @@ -2203,9 +2199,9 @@ UT_TEST(test_pcm_x_grant_pending_invalidate_wakes_local_s_before_busy) UT_ASSERT_NOT_NULL(pending); UT_ASSERT_NOT_NULL(wake); UT_ASSERT_NOT_NULL(busy); - if (direct_prepare != NULL && direct_prepare_end != NULL - && direct_first_reply_check != NULL && direct_target_prepare != NULL - && direct_second_reply_check != NULL && direct_target_cleanup != NULL) + if (direct_prepare != NULL && direct_prepare_end != NULL && direct_first_reply_check != NULL + && direct_target_prepare != NULL && direct_second_reply_check != NULL + && direct_target_cleanup != NULL) UT_ASSERT(direct_prepare < direct_first_reply_check && direct_first_reply_check < direct_target_prepare && direct_target_prepare < direct_second_reply_check @@ -2246,9 +2242,8 @@ UT_TEST(test_pcm_x_grant_pending_orphan_observation_is_identity_exact) } execute = strstr(gcs_source, "\ngcs_block_invalidate_execute("); - pending = execute != NULL - ? strstr(execute, "cluster_bufmgr_block_grant_pending(inv->tag)") - : NULL; + pending + = execute != NULL ? strstr(execute, "cluster_bufmgr_block_grant_pending(inv->tag)") : NULL; wake = pending != NULL ? strstr(pending, "woke_local = gcs_block_wake_local_pending_s_request(inv)") : NULL; @@ -2273,11 +2268,11 @@ UT_TEST(test_pcm_x_grant_pending_orphan_observation_is_identity_exact) retry = strstr(bufmgr_source, "\ncluster_bufmgr_pcm_retry_denied_rearm("); abort = retry != NULL - ? strstr(retry, "cluster_pcm_own_abort_grant_reservation(buf, base, *reservation_token)") + ? strstr(retry, + "cluster_pcm_own_abort_grant_reservation(buf, base, *reservation_token)") : NULL; - abort_observe = abort != NULL - ? strstr(abort, "cluster PCM pending-X exact abort observation:") - : NULL; + abort_observe + = abort != NULL ? strstr(abort, "cluster PCM pending-X exact abort observation:") : NULL; UT_ASSERT_NOT_NULL(retry); UT_ASSERT_NOT_NULL(abort); UT_ASSERT_NOT_NULL(abort_observe); @@ -2286,10 +2281,11 @@ UT_TEST(test_pcm_x_grant_pending_orphan_observation_is_identity_exact) release = strstr(gcs_source, "\ngcs_block_release_slot("); release_end = release != NULL ? strstr(release, "\n}\n") : NULL; - release_live = release != NULL - ? strstr(release, - "cluster GCS block slot released with live direct target observation:") - : NULL; + release_live + = release != NULL + ? strstr(release, + "cluster GCS block slot released with live direct target observation:") + : NULL; UT_ASSERT_NOT_NULL(release); UT_ASSERT_NOT_NULL(release_end); UT_ASSERT_NOT_NULL(release_live); @@ -2298,14 +2294,14 @@ UT_TEST(test_pcm_x_grant_pending_orphan_observation_is_identity_exact) direct_fail = strstr(gcs_source, "\ngcs_block_direct_fail_slot("); direct_fail_end = direct_fail != NULL ? strstr(direct_fail, "\n}\n") : NULL; - direct_finish = direct_fail != NULL - ? strstr(direct_fail, - "gcs_block_direct_finish_target(target_buf, prepared, false") - : NULL; - direct_abort_observe = direct_finish != NULL - ? strstr(direct_finish, - "cluster GCS block direct abort observation:") - : NULL; + direct_finish + = direct_fail != NULL + ? strstr(direct_fail, "gcs_block_direct_finish_target(target_buf, prepared, false") + : NULL; + direct_abort_observe + = direct_finish != NULL + ? strstr(direct_finish, "cluster GCS block direct abort observation:") + : NULL; UT_ASSERT_NOT_NULL(direct_fail); UT_ASSERT_NOT_NULL(direct_fail_end); UT_ASSERT_NOT_NULL(direct_finish); @@ -2371,31 +2367,22 @@ UT_TEST(test_pcm_x_final_ack_fail_closed_names_exact_handoff_stage) const char *handler = source != NULL ? strstr(source, "\ncluster_gcs_handle_pcm_x_final_ack_envelope(") : NULL; const char *handler_end - = handler != NULL - ? strstr(handler, "\ncluster_gcs_handle_pcm_x_final_commit_ack_envelope(") - : NULL; + = handler != NULL ? strstr(handler, "\ncluster_gcs_handle_pcm_x_final_commit_ack_envelope(") + : NULL; const char *stage_log = handler != NULL ? strstr(handler, "PCM-X FINAL_ACK fail-closed at %s") : NULL; const char *canonical = handler != NULL ? strstr(handler, "cluster_pcm_x_runtime_fail_closed()") : NULL; const char *direct = handler != NULL ? strstr(handler, "cluster_pcm_x_runtime_transition(") : NULL; - const char *master_holder - = handler != NULL ? strstr(handler, "master_holder=%u") : NULL; - const char *image_page_scn - = handler != NULL ? strstr(handler, "image_page_scn=%llu") : NULL; - const char *watermark_scn - = handler != NULL ? strstr(handler, "watermark_scn=%llu") : NULL; - const char *watermark_source - = handler != NULL ? strstr(handler, "wm_src=%s") : NULL; - const char *watermark_sender - = handler != NULL ? strstr(handler, "wm_sender=%d") : NULL; - const char *watermark_request - = handler != NULL ? strstr(handler, "wm_request_id=%llu") : NULL; - const char *watermark_old - = handler != NULL ? strstr(handler, "wm_old_scn=%llu") : NULL; - const char *watermark_new - = handler != NULL ? strstr(handler, "wm_new_scn=%llu") : NULL; + const char *master_holder = handler != NULL ? strstr(handler, "master_holder=%u") : NULL; + const char *image_page_scn = handler != NULL ? strstr(handler, "image_page_scn=%llu") : NULL; + const char *watermark_scn = handler != NULL ? strstr(handler, "watermark_scn=%llu") : NULL; + const char *watermark_source = handler != NULL ? strstr(handler, "wm_src=%s") : NULL; + const char *watermark_sender = handler != NULL ? strstr(handler, "wm_sender=%d") : NULL; + const char *watermark_request = handler != NULL ? strstr(handler, "wm_request_id=%llu") : NULL; + const char *watermark_old = handler != NULL ? strstr(handler, "wm_old_scn=%llu") : NULL; + const char *watermark_new = handler != NULL ? strstr(handler, "wm_new_scn=%llu") : NULL; UT_ASSERT_NOT_NULL(handler); UT_ASSERT_NOT_NULL(handler_end); @@ -2584,14 +2571,10 @@ UT_TEST(test_pcm_x_ready_materializes_exact_n_s_or_x_source_without_wire_change) UT_ASSERT_NOT_NULL(strstr(begin, "cluster_bufmgr_pcm_own_prepare_s_source_image(")); UT_ASSERT_NOT_NULL(strstr(begin, "binding.required_page_scn")); UT_ASSERT_NOT_NULL(strstr(begin, "&source_prepare_refusal")); - UT_ASSERT_NOT_NULL( - strstr(source, "materialize-begin-s-content-lock")); - UT_ASSERT_NOT_NULL( - strstr(source, "materialize-begin-s-dirty-flushed")); - UT_ASSERT_NOT_NULL( - strstr(source, "materialize-begin-s-dirty-raced")); - UT_ASSERT_NOT_NULL( - strstr(source, "materialize-begin-s-io-in-progress")); + UT_ASSERT_NOT_NULL(strstr(source, "materialize-begin-s-content-lock")); + UT_ASSERT_NOT_NULL(strstr(source, "materialize-begin-s-dirty-flushed")); + UT_ASSERT_NOT_NULL(strstr(source, "materialize-begin-s-dirty-raced")); + UT_ASSERT_NOT_NULL(strstr(source, "materialize-begin-s-io-in-progress")); UT_ASSERT_NOT_NULL(strstr(begin, "cluster_bufmgr_pcm_own_begin_x_revoke(")); UT_ASSERT_NOT_NULL(strstr(begin, "cluster_pcm_x_revoke_finish_mode(")); UT_ASSERT_NOT_NULL(strstr(begin, "CLUSTER_PCM_X_REVOKE_FINISH_DROP")); @@ -2681,16 +2664,14 @@ UT_TEST(test_pcm_x_s_source_hard_failure_observation_is_reason_exact) const char *observe; const char *prepare; const char *prepare_end; - static const char *const reasons[] = { - "CLUSTER_PCM_S_SOURCE_HARD_INITIAL_CURRENT_IMAGE", - "CLUSTER_PCM_S_SOURCE_HARD_INITIAL_IO_ERROR", - "CLUSTER_PCM_S_SOURCE_HARD_BEGIN_REVOKE_CORRUPT", - "CLUSTER_PCM_S_SOURCE_HARD_STORAGE_VERIFY", - "CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_CURRENT_IMAGE", - "CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_IO_ERROR", - "CLUSTER_PCM_S_SOURCE_HARD_NO_COVER", - "CLUSTER_PCM_S_SOURCE_HARD_ABORT_FAILURE" - }; + static const char *const reasons[] = { "CLUSTER_PCM_S_SOURCE_HARD_INITIAL_CURRENT_IMAGE", + "CLUSTER_PCM_S_SOURCE_HARD_INITIAL_IO_ERROR", + "CLUSTER_PCM_S_SOURCE_HARD_BEGIN_REVOKE_CORRUPT", + "CLUSTER_PCM_S_SOURCE_HARD_STORAGE_VERIFY", + "CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_CURRENT_IMAGE", + "CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_IO_ERROR", + "CLUSTER_PCM_S_SOURCE_HARD_NO_COVER", + "CLUSTER_PCM_S_SOURCE_HARD_ABORT_FAILURE" }; int i; UT_ASSERT_NOT_NULL(source); @@ -2698,9 +2679,10 @@ UT_TEST(test_pcm_x_s_source_hard_failure_observation_is_reason_exact) return; observe = strstr(source, "\ncluster_bufmgr_pcm_own_observe_s_source_hard_failure("); prepare = strstr(source, "\ncluster_bufmgr_pcm_own_prepare_s_source_image("); - prepare_end = prepare != NULL - ? strstr(prepare, "\n/* Abort only the matching S-source staging reservation. */") - : NULL; + prepare_end + = prepare != NULL + ? strstr(prepare, "\n/* Abort only the matching S-source staging reservation. */") + : NULL; UT_ASSERT_NOT_NULL(observe); UT_ASSERT_NOT_NULL(prepare); UT_ASSERT_NOT_NULL(prepare_end); @@ -2722,7 +2704,8 @@ UT_TEST(test_pcm_x_s_source_hard_failure_observation_is_reason_exact) UT_ASSERT(observe < prepare); if (prepare != NULL && prepare_end != NULL) { UT_ASSERT_NOT_NULL(strstr(prepare, "hard_failure_reason =")); - UT_ASSERT_NOT_NULL(strstr(prepare, "cluster_bufmgr_pcm_own_observe_s_source_hard_failure(")); + UT_ASSERT_NOT_NULL( + strstr(prepare, "cluster_bufmgr_pcm_own_observe_s_source_hard_failure(")); UT_ASSERT(strstr(prepare, "cluster_bufmgr_pcm_own_observe_s_source_hard_failure(") < prepare_end); } @@ -2829,8 +2812,7 @@ UT_TEST(test_pcm_x_ready_admission_marks_before_send_and_rolls_back_refusal) UT_ASSERT_NOT_NULL(strstr(begin, "PCM-X IMAGE_READY stage boundary")); UT_ASSERT_NOT_NULL(strstr(begin, "mark_result")); UT_ASSERT_NOT_NULL(strstr(begin, "stage_result")); - UT_ASSERT_NOT_NULL( - strstr(outbound_source, "cluster_lms_note_pcm_x_image_ready_boundary(")); + UT_ASSERT_NOT_NULL(strstr(outbound_source, "cluster_lms_note_pcm_x_image_ready_boundary(")); handler = strstr(source, "\ncluster_gcs_handle_pcm_x_image_ready_envelope("); UT_ASSERT_NOT_NULL(handler); if (handler != NULL) { @@ -2985,15 +2967,15 @@ UT_TEST(test_pcm_x_destructive_finish_fault_times_out_in_sql_before_harness) UT_ASSERT_NOT_NULL(leg); UT_ASSERT_NOT_NULL(destructive_leg); statement_timeout = leg != NULL ? strstr(leg, "statement_timeout") : NULL; - insert = leg != NULL - ? strstr(leg, "INSERT INTO pcm_xq_flush_error(id, v) VALUES (2, 1)") - : NULL; + insert + = leg != NULL ? strstr(leg, "INSERT INTO pcm_xq_flush_error(id, v) VALUES (2, 1)") : NULL; harness_timeout = leg != NULL ? strstr(leg, "timeout => 30") : NULL; assertion = leg != NULL ? strstr(leg, "L5F remote writer failed") : NULL; - relfilenode = destructive_leg != NULL ? strstr(destructive_leg, "flush_error_relfilenode") : NULL; - exact_oracle - = destructive_leg != NULL ? strstr(destructive_leg, "PCM-X finish-error evidence exact") - : NULL; + relfilenode + = destructive_leg != NULL ? strstr(destructive_leg, "flush_error_relfilenode") : NULL; + exact_oracle = destructive_leg != NULL + ? strstr(destructive_leg, "PCM-X finish-error evidence exact") + : NULL; global_occupancy = destructive_leg != NULL ? strstr(destructive_leg, "dedup_entry_count") : NULL; UT_ASSERT_NOT_NULL(statement_timeout); @@ -3465,8 +3447,7 @@ UT_TEST(test_pcm_x_requester_driver_owns_fifo_and_transfer_lifecycles) wait_exact = strstr(source, "\ngcs_block_pcm_x_requester_wait_exact("); wait_exact_end = wait_exact != NULL ? strstr(wait_exact, "\n}\n") : NULL; wait_revalidate = strstr(source, "\ngcs_block_pcm_x_requester_pre_sleep_revalidate("); - wait_revalidate_end - = wait_revalidate != NULL ? strstr(wait_revalidate, "\n}\n") : NULL; + wait_revalidate_end = wait_revalidate != NULL ? strstr(wait_revalidate, "\n}\n") : NULL; UT_ASSERT_NOT_NULL(driver); UT_ASSERT_NOT_NULL(driver_end); UT_ASSERT_NOT_NULL(rekey_helper); @@ -4403,8 +4384,8 @@ UT_TEST(test_revoke_handler_silent_refusal_arms_all_note) UT_ASSERT_NOT_NULL(strstr(materialize, "\"materialize-finish\"")); } publish = materialize != NULL - ? strstr(materialize, "cluster_gcs_block_dedup_pcm_x_publish_ready_exact(") - : NULL; + ? strstr(materialize, "cluster_gcs_block_dedup_pcm_x_publish_ready_exact(") + : NULL; reset = publish != NULL ? strstr(publish, "gcs_block_pcm_x_revoke_refusal_note(NULL, 0, NULL)") : NULL; if (reset == NULL && publish != NULL) @@ -4455,16 +4436,16 @@ UT_TEST(test_local_master_read_image_retries_holder_busy_with_fresh_identity) budget = strstr(read_image, "cluster_gcs_block_retransmit_max_retries"); loop = budget != NULL ? strstr(budget, "for (retry_attempt = 0;") : NULL; backoff = loop != NULL ? strstr(loop, "gcs_block_backoff_ms_for_retry(retry_attempt)") : NULL; - fresh_id = backoff != NULL ? strstr(backoff, "gcs_block_pcm_x_next_request_id(&request_id)") : NULL; + fresh_id + = backoff != NULL ? strstr(backoff, "gcs_block_pcm_x_next_request_id(&request_id)") : NULL; slot_id = fresh_id != NULL ? strstr(fresh_id, "slot->request_id = request_id") : NULL; forward_id = slot_id != NULL ? strstr(slot_id, "fwd.request_id = request_id") : NULL; - send = forward_id != NULL - ? strstr(forward_id, "cluster_grd_outbound_enqueue_backend_msg(") - : NULL; - retryable_deny - = send != NULL ? strstr(send, "GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER") : NULL; + send = forward_id != NULL ? strstr(forward_id, "cluster_grd_outbound_enqueue_backend_msg(") + : NULL; + retryable_deny = send != NULL ? strstr(send, "GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER") : NULL; retry = retryable_deny != NULL ? strstr(retryable_deny, "continue;") : NULL; - terminal_error = retry != NULL ? strstr(retry, "could not obtain read image from X holder") : NULL; + terminal_error + = retry != NULL ? strstr(retry, "could not obtain read image from X holder") : NULL; UT_ASSERT_NOT_NULL(budget); UT_ASSERT_NOT_NULL(loop); UT_ASSERT_NOT_NULL(backoff); @@ -4522,17 +4503,15 @@ UT_TEST(test_local_master_read_image_stops_retrying_displaced_holder_exactly) ? strstr(retry_arg, "if (!cluster_pcm_lock_authority_matches(tag, expected))") : NULL; reserve = precheck != NULL ? strstr(precheck, "gcs_block_reserve_slot(") : NULL; - backoff = reserve != NULL ? strstr(reserve, "gcs_block_backoff_ms_for_retry(retry_attempt)") - : NULL; + backoff + = reserve != NULL ? strstr(reserve, "gcs_block_backoff_ms_for_retry(retry_attempt)") : NULL; backoff_check = backoff != NULL ? strstr(backoff, "if (!cluster_pcm_lock_authority_matches(tag, expected))") : NULL; fresh_id = backoff_check != NULL ? strstr(backoff_check, "gcs_block_pcm_x_next_request_id(&request_id)") : NULL; - denial = fresh_id != NULL - ? strstr(fresh_id, "GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER") - : NULL; + denial = fresh_id != NULL ? strstr(fresh_id, "GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER") : NULL; denial_check = denial != NULL ? strstr(denial, "if (!cluster_pcm_lock_authority_matches(tag, expected))") : NULL; @@ -4540,8 +4519,8 @@ UT_TEST(test_local_master_read_image_stops_retrying_displaced_holder_exactly) release = drift_retry != NULL ? strstr(drift_retry, "gcs_block_release_slot(slot)") : NULL; drift_return = release != NULL ? strstr(release, "if (*out_retry_denied)") : NULL; terminal_error = drift_return != NULL - ? strstr(drift_return, "could not obtain read image from X holder") - : NULL; + ? strstr(drift_return, "could not obtain read image from X holder") + : NULL; UT_ASSERT_NOT_NULL(expected_arg); UT_ASSERT_NOT_NULL(retry_arg); @@ -4558,8 +4537,8 @@ UT_TEST(test_local_master_read_image_stops_retrying_displaced_holder_exactly) UT_ASSERT_NOT_NULL(terminal_error); if (expected_arg != NULL && retry_arg != NULL && precheck != NULL && reserve != NULL && backoff != NULL && backoff_check != NULL && fresh_id != NULL && denial != NULL - && denial_check != NULL && drift_retry != NULL && release != NULL - && drift_return != NULL && terminal_error != NULL) + && denial_check != NULL && drift_retry != NULL && release != NULL && drift_return != NULL + && terminal_error != NULL) UT_ASSERT(read_image < expected_arg && expected_arg < retry_arg && retry_arg < precheck && precheck < reserve && reserve < backoff && backoff < backoff_check && backoff_check < fresh_id && fresh_id < denial && denial < denial_check @@ -4584,8 +4563,7 @@ UT_TEST(test_local_master_read_image_refusal_evidence_is_attempt_exact) = forward != NULL ? strstr(forward, "cluster_bufmgr_gcs_copy_refusal_name") : NULL; const char *refusal_log = refusal_name != NULL ? strstr(refusal_name, "holder ship image refused") : NULL; - const char *request_id - = refusal_log != NULL ? strstr(refusal_log, "request_id=") : NULL; + const char *request_id = refusal_log != NULL ? strstr(refusal_log, "request_id=") : NULL; /* P0-21 observation only: the requester terminal error reports its exact * bounded-attempt result, while the holder process log binds each copy @@ -4602,7 +4580,8 @@ UT_TEST(test_local_master_read_image_refusal_evidence_is_attempt_exact) UT_ASSERT_NOT_NULL(refusal_log); UT_ASSERT_NOT_NULL(request_id); if (forward_end != NULL && refusal_name != NULL && refusal_log != NULL && request_id != NULL) - UT_ASSERT(refusal_name < refusal_log && refusal_log < request_id && request_id < forward_end); + UT_ASSERT(refusal_name < refusal_log && refusal_log < request_id + && request_id < forward_end); UT_ASSERT_NOT_NULL(t400); if (t400 != NULL) { const char *holder_before = strstr(t400, "holder_evicted_before_by_node"); @@ -4610,8 +4589,8 @@ UT_TEST(test_local_master_read_image_refusal_evidence_is_attempt_exact) = holder_before != NULL ? strstr(holder_before, "holder_evicted_after_by_node") : NULL; const char *holder_diag = holder_after != NULL ? strstr(holder_after, "holder copy refusal baseline") : NULL; - const char *holder_final - = holder_diag != NULL ? strstr(holder_diag, "'block_forward_holder_evicted_count'") + const char *holder_final = holder_diag != NULL + ? strstr(holder_diag, "'block_forward_holder_evicted_count'") : NULL; UT_ASSERT_NOT_NULL(holder_before); @@ -4704,55 +4683,71 @@ UT_TEST(test_remote_downgrade_prepares_exact_image_before_notify_and_reply) char *gcs_source = read_gcs_block_source(); char *bufmgr_source = read_source_path("../../backend/storage/buffer/bufmgr.c"); const char *forward = gcs_source != NULL - ? strstr(gcs_source, "\ncluster_gcs_handle_block_forward_envelope(") : NULL; + ? strstr(gcs_source, "\ncluster_gcs_handle_block_forward_envelope(") + : NULL; const char *forward_end = forward != NULL ? strstr(forward, "\n}\n") : NULL; - const char *inject = forward != NULL - ? strstr(forward, "cluster-gcs-block-evict-holder-before-ship") : NULL; - const char *downgrade = inject != NULL - ? strstr(inject, "cluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image(") : NULL; - const char *reuse = downgrade != NULL ? strstr(downgrade, "holder_ship_ok = remote_downgraded") - : NULL; - const char *post_notify_guard = downgrade != NULL - ? strstr(downgrade, - "remote_downgrade_outcome == CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY") - : NULL; + const char *inject + = forward != NULL ? strstr(forward, "cluster-gcs-block-evict-holder-before-ship") : NULL; + const char *downgrade + = inject != NULL + ? strstr(inject, "cluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image(") + : NULL; + const char *reuse + = downgrade != NULL ? strstr(downgrade, "holder_ship_ok = remote_downgraded") : NULL; + const char *post_notify_guard + = downgrade != NULL + ? strstr(downgrade, "remote_downgrade_outcome == " + "CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY") + : NULL; const char *post_notify_return = post_notify_guard != NULL ? strstr(post_notify_guard, "return;") : NULL; const char *copy = reuse != NULL ? strstr(reuse, "gcs_block_get_ship_image(") : NULL; - const char *helper = bufmgr_source != NULL - ? strstr(bufmgr_source, "\ncluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image(") - : NULL; + const char *helper + = bufmgr_source != NULL + ? strstr(bufmgr_source, + "\ncluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image(") + : NULL; const char *helper_end = helper != NULL ? strstr(helper, "\n}\n") : NULL; - const char *reserve = helper != NULL ? strstr(helper, "cluster_bufmgr_pcm_own_begin_x_revoke(") - : NULL; + const char *reserve + = helper != NULL ? strstr(helper, "cluster_bufmgr_pcm_own_begin_x_revoke(") : NULL; const char *precopy = reserve != NULL ? strstr(reserve, "memcpy(dst,") : NULL; - const char *notify = precopy != NULL - ? strstr(precopy, "cluster_gcs_send_transition_nowait(") : NULL; - const char *commit = notify != NULL - ? strstr(notify, "cluster_bufmgr_pcm_own_finish_x_to_s_downgrade(") : NULL; - const char *abort = helper != NULL - ? strstr(helper, "cluster_bufmgr_pcm_own_abort_x_revoke(") : NULL; - const char *local_helper = bufmgr_source != NULL - ? strstr(bufmgr_source, "\ncluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image(") - : NULL; + const char *notify + = precopy != NULL ? strstr(precopy, "cluster_gcs_send_transition_nowait(") : NULL; + const char *commit + = notify != NULL ? strstr(notify, "cluster_bufmgr_pcm_own_finish_x_to_s_downgrade(") : NULL; + const char *abort + = helper != NULL ? strstr(helper, "cluster_bufmgr_pcm_own_abort_x_revoke(") : NULL; + const char *local_helper + = bufmgr_source != NULL + ? strstr(bufmgr_source, "\ncluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image(") + : NULL; const char *local_helper_end = local_helper != NULL ? strstr(local_helper, "\n}\n") : NULL; const char *local_reserve = local_helper != NULL - ? strstr(local_helper, "cluster_bufmgr_pcm_own_begin_x_revoke(") : NULL; + ? strstr(local_helper, "cluster_bufmgr_pcm_own_begin_x_revoke(") + : NULL; const char *local_copy = local_reserve != NULL ? strstr(local_reserve, "memcpy(dst,") : NULL; - const char *local_master = local_copy != NULL - ? strstr(local_copy, "cluster_pcm_lock_apply_gcs_transition(") : NULL; - const char *local_commit = local_master != NULL - ? strstr(local_master, "cluster_bufmgr_pcm_own_finish_x_to_s_downgrade(") : NULL; - const char *local_call = gcs_source != NULL - ? strstr(gcs_source, "cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image(") : NULL; - const char *prepared = local_call != NULL ? strstr(local_call, "scache_image_prepared = true") - : NULL; - const char *produce = prepared != NULL - ? strstr(prepared, "gcs_block_produce_reply(req, block_buf, scache_image_prepared") : NULL; - const char *produce_def = gcs_source != NULL ? strstr(gcs_source, "\ngcs_block_produce_reply(") - : NULL; - const char *skip_second_copy = produce_def != NULL - ? strstr(produce_def, "if (!preprepared_image\n\t\t&& !gcs_block_get_ship_image(") : NULL; + const char *local_master + = local_copy != NULL ? strstr(local_copy, "cluster_pcm_lock_apply_gcs_transition(") : NULL; + const char *local_commit + = local_master != NULL + ? strstr(local_master, "cluster_bufmgr_pcm_own_finish_x_to_s_downgrade(") + : NULL; + const char *local_call + = gcs_source != NULL + ? strstr(gcs_source, "cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image(") + : NULL; + const char *prepared + = local_call != NULL ? strstr(local_call, "scache_image_prepared = true") : NULL; + const char *produce + = prepared != NULL + ? strstr(prepared, "gcs_block_produce_reply(req, block_buf, scache_image_prepared") + : NULL; + const char *produce_def + = gcs_source != NULL ? strstr(gcs_source, "\ngcs_block_produce_reply(") : NULL; + const char *skip_second_copy + = produce_def != NULL + ? strstr(produce_def, "if (!preprepared_image\n\t\t&& !gcs_block_get_ship_image(") + : NULL; /* P0-32: injection is decided before any irreversible downgrade. The * successful arm reuses bytes proven under the same content EXCLUSIVE @@ -4782,11 +4777,13 @@ UT_TEST(test_remote_downgrade_prepares_exact_image_before_notify_and_reply) UT_ASSERT_NOT_NULL(notify); UT_ASSERT_NOT_NULL(commit); UT_ASSERT_NOT_NULL(abort); - if (helper_end != NULL && reserve != NULL && precopy != NULL && notify != NULL && commit != NULL) + if (helper_end != NULL && reserve != NULL && precopy != NULL && notify != NULL + && commit != NULL) UT_ASSERT(reserve < precopy && precopy < notify && notify < commit && commit < helper_end); if (helper_end != NULL && abort != NULL) UT_ASSERT(abort < notify && abort < helper_end); - UT_ASSERT_NOT_NULL(helper != NULL ? strstr(helper, "cluster_pcm_x_runtime_fail_closed()") : NULL); + UT_ASSERT_NOT_NULL(helper != NULL ? strstr(helper, "cluster_pcm_x_runtime_fail_closed()") + : NULL); UT_ASSERT_NOT_NULL(local_helper); UT_ASSERT_NOT_NULL(local_helper_end); UT_ASSERT_NOT_NULL(local_reserve); @@ -4868,18 +4865,15 @@ UT_TEST(test_gcs_apply_state_drift_restarts_with_fresh_request_identity) * N->S/N->X must abandon this attempt and re-enter bufmgr with a fresh * request/token identity. Other transition incompatibilities remain * structural terminal denials. */ - UT_ASSERT_EQ((int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_PENDING_X, - PCM_TRANS_N_TO_S), + UT_ASSERT_EQ((int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_PENDING_X, PCM_TRANS_N_TO_S), (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); - UT_ASSERT_EQ((int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_INCOMPATIBLE, - PCM_TRANS_N_TO_S), + UT_ASSERT_EQ((int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_INCOMPATIBLE, PCM_TRANS_N_TO_S), (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); - UT_ASSERT_EQ((int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_INCOMPATIBLE, - PCM_TRANS_N_TO_X), + UT_ASSERT_EQ((int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_INCOMPATIBLE, PCM_TRANS_N_TO_X), (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); - UT_ASSERT_EQ((int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_INCOMPATIBLE, - PCM_TRANS_S_TO_X_UPGRADE), - (int)GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE); + UT_ASSERT_EQ( + (int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_INCOMPATIBLE, PCM_TRANS_S_TO_X_UPGRADE), + (int)GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE); UT_ASSERT_NOT_NULL(produce); UT_ASSERT_NOT_NULL(produce_end); @@ -4909,18 +4903,14 @@ UT_TEST(test_master_direct_copy_busy_uses_only_fresh_identity_retry_boundary) GcsBlockReplyStatus expected; }; static const struct CopyRefusalCase cases[] = { - { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE, - GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INVALID_ARGUMENT, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, - { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT, - GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, - { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST, - GCS_BLOCK_REPLY_DENIED_PENDING_X }, - { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_SECOND, - GCS_BLOCK_REPLY_DENIED_PENDING_X }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST, GCS_BLOCK_REPLY_DENIED_PENDING_X }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_SECOND, GCS_BLOCK_REPLY_DENIED_PENDING_X }, { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_HC89_LSN_DRIFT, @@ -4965,12 +4955,12 @@ UT_TEST(test_master_direct_copy_busy_uses_only_fresh_identity_retry_boundary) copy_refusal = strstr(produce, "produce-copy-refused:%s"); copy_refusal_end = copy_refusal != NULL ? strstr(copy_refusal, "return true;") : NULL; mapping = copy_refusal != NULL - ? strstr(copy_refusal, "GcsBlockMasterDirectCopyRefusalStatus(copy_refusal)") + ? strstr(copy_refusal, "GcsBlockMasterDirectCopyRefusalStatus(copy_refusal)") + : NULL; + terminal_literal + = copy_refusal != NULL + ? strstr(copy_refusal, "*out_status = GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER") : NULL; - terminal_literal = copy_refusal != NULL - ? strstr(copy_refusal, - "*out_status = GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER") - : NULL; UT_ASSERT_NOT_NULL(copy_refusal); UT_ASSERT_NOT_NULL(copy_refusal_end); UT_ASSERT_NOT_NULL(mapping); @@ -4983,8 +4973,7 @@ UT_TEST(test_master_direct_copy_busy_uses_only_fresh_identity_retry_boundary) * the same cached request id. Bufmgr then aborts the exact old reservation, * waits, and rearms a fresh token/request identity (covered by * test_pending_x_denied_retry_leaves_master_invalidate_gap). */ - requester_retry - = strstr(source, "if (final_status == GCS_BLOCK_REPLY_DENIED_PENDING_X)"); + requester_retry = strstr(source, "if (final_status == GCS_BLOCK_REPLY_DENIED_PENDING_X)"); retry_flag = requester_retry != NULL ? strstr(requester_retry, "retry_denied = true;") : NULL; retry_break = retry_flag != NULL ? strstr(retry_flag, "break;") : NULL; same_id_continue = retry_flag != NULL ? strstr(retry_flag, "continue;") : NULL; @@ -5024,20 +5013,13 @@ UT_TEST(test_master_not_holder_producers_log_one_coherent_authority_snapshot) "authority.transition_count", }; static const char *const required_reasons[] = { - "direct-land-nonsendable", - "direct-land-forward-rearm", - "pcm-x-image-not-ready", - "produce-no-resident-authority", - "produce-copy-refused", - "clean-third-party-master", - "live-x-other-holder", - "pending-x-reserve-failed", - "x-forward-send-failed", - "x-state-holder-unroutable", - "holder-immediate-deny", - "holder-drop-pinned", - "holder-drop-stale", - "holder-copy-refused", + "direct-land-nonsendable", "direct-land-forward-rearm", + "pcm-x-image-not-ready", "produce-no-resident-authority", + "produce-copy-refused", "clean-third-party-master", + "live-x-other-holder", "pending-x-reserve-failed", + "x-forward-send-failed", "x-state-holder-unroutable", + "holder-immediate-deny", "holder-drop-pinned", + "holder-drop-stale", "holder-copy-refused", }; size_t i; @@ -5060,8 +5042,9 @@ UT_TEST(test_master_not_holder_producers_log_one_coherent_authority_snapshot) UT_ASSERT(field < helper_end); } snapshot = strstr(helper, "cluster_pcm_lock_authority_snapshot("); - duplicate_snapshot - = snapshot != NULL ? strstr(snapshot + 1, "cluster_pcm_lock_authority_snapshot(") : NULL; + duplicate_snapshot = snapshot != NULL + ? strstr(snapshot + 1, "cluster_pcm_lock_authority_snapshot(") + : NULL; UT_ASSERT_NOT_NULL(snapshot); UT_ASSERT(snapshot != NULL && snapshot < helper_end); UT_ASSERT(duplicate_snapshot == NULL || duplicate_snapshot >= helper_end); @@ -5215,21 +5198,20 @@ UT_TEST(test_pcm_x_source_floor_v2_is_connection_bound_until_lms_drain) transfer = strstr(source, "\ngcs_block_pcm_x_master_drive_transfer("); transfer_end = transfer != NULL ? strstr(transfer, "\n}\n") : NULL; - auth = transfer != NULL - ? strstr(transfer, "gcs_block_pcm_x_authenticated_session_result(") - : NULL; + auth = transfer != NULL ? strstr(transfer, "gcs_block_pcm_x_authenticated_session_result(") + : NULL; arm = auth != NULL ? strstr(auth, "cluster_pcm_x_master_revoke_arm_exact(") : NULL; sample = arm != NULL ? strstr(arm, "cluster_sf_peer_pcm_x_source_floor_sample(") : NULL; - generation - = sample != NULL ? strstr(sample, "auth_sample.connection_generation_before") : NULL; + generation = sample != NULL ? strstr(sample, "auth_sample.connection_generation_before") : NULL; guarded_stage = generation != NULL ? strstr(generation, "gcs_block_pcm_x_stage_frame_cap_bound(") : NULL; - v1_fallback = guarded_stage != NULL - ? strstr(guarded_stage, "cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_REVOKE") - : NULL; - stray_simple_query = transfer != NULL - ? strstr(transfer, "cluster_sf_peer_supports_pcm_x_source_floor(source)") - : NULL; + v1_fallback + = guarded_stage != NULL + ? strstr(guarded_stage, "cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_REVOKE") + : NULL; + stray_simple_query + = transfer != NULL ? strstr(transfer, "cluster_sf_peer_supports_pcm_x_source_floor(source)") + : NULL; UT_ASSERT_NOT_NULL(transfer); UT_ASSERT_NOT_NULL(transfer_end); UT_ASSERT_NOT_NULL(auth); diff --git a/src/test/cluster_unit/test_cluster_gcs_block_dedup.c b/src/test/cluster_unit/test_cluster_gcs_block_dedup.c index d0014fef40..a1255861a1 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_dedup.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_dedup.c @@ -1428,13 +1428,13 @@ UT_TEST(u25_pcm_x_work_prefers_reserved_and_marks_ready_staged) reserved_binding.identity.image.page_checksum = 0; reserved_binding.required_page_scn = UINT64_C(72057594037950810); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &reserved_key, &reserved_tag, - &reserved_binding), + &reserved_binding), (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); wrong_floor = reserved_binding; wrong_floor.required_page_scn++; - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &reserved_key, &reserved_tag, - &wrong_floor), - (int)GCS_BLOCK_PCM_X_IMAGE_STALE); + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pcm_x_reserve(0, &reserved_key, &reserved_tag, &wrong_floor), + (int)GCS_BLOCK_PCM_X_IMAGE_STALE); memset(&work, 0, sizeof(work)); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), @@ -1808,15 +1808,15 @@ UT_TEST(u33_pending_x_arm_terminates_inflight_legacy_s_exactly) forward_hdr.transition_id = (uint8)PCM_TRANS_N_TO_S; forward_hdr.status = (uint8)GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER; GcsBlockReplyHeaderSetForwardingMasterNode(&forward_hdr, cluster_node_id); - cluster_gcs_block_dedup_install_reply(0, &forwarded, - GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER, &forward_hdr, NULL); + cluster_gcs_block_dedup_install_reply(0, &forwarded, GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER, + &forward_hdr, NULL); /* Different tag and same-tag writer identities are not legacy-S victims. */ UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( 0, &unrelated, other_tag, PCM_TRANS_N_TO_S, 1000, true, &cached), (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( - 0, &writer, tag, PCM_TRANS_N_TO_X, 1000, true, &cached), + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register(0, &writer, tag, PCM_TRANS_N_TO_X, + 1000, true, &cached), (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); for (i = 0; i < 2; i++) { @@ -1826,12 +1826,10 @@ UT_TEST(u33_pending_x_arm_terminates_inflight_legacy_s_exactly) UT_ASSERT_EQ((int)denied.entry_kind, (int)GCS_BLOCK_DEDUP_ENTRY_GENERIC); UT_ASSERT_EQ((int)denied.transition_id, (int)PCM_TRANS_N_TO_S); UT_ASSERT_EQ((int)denied.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); - UT_ASSERT_EQ((int)denied.reply_header.status, - (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ((int)denied.reply_header.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); UT_ASSERT_EQ(denied.reply_header.request_id, denied.key.request_id); UT_ASSERT_EQ(denied.reply_header.epoch, denied.key.cluster_epoch); - UT_ASSERT_EQ(denied.reply_header.requester_backend_id, - denied.key.requester_backend_id); + UT_ASSERT_EQ(denied.reply_header.requester_backend_id, denied.key.requester_backend_id); UT_ASSERT_EQ((int)denied.reply_header.transition_id, (int)PCM_TRANS_N_TO_S); UT_ASSERT_EQ(denied.reply_header.sender_node, cluster_node_id); UT_ASSERT_EQ(GcsBlockReplyHeaderGetForwardingMasterNode(&denied.reply_header), @@ -1856,8 +1854,8 @@ UT_TEST(u33_pending_x_arm_terminates_inflight_legacy_s_exactly) late_granted.requester_backend_id = denied.key.requester_backend_id; late_granted.transition_id = (uint8)PCM_TRANS_N_TO_S; late_granted.status = (uint8)GCS_BLOCK_REPLY_GRANTED; - cluster_gcs_block_dedup_install_reply(0, &denied.key, GCS_BLOCK_REPLY_GRANTED, - &late_granted, page); + cluster_gcs_block_dedup_install_reply(0, &denied.key, GCS_BLOCK_REPLY_GRANTED, &late_granted, + page); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( 0, &denied.key, tag, PCM_TRANS_N_TO_S, 1000, true, &cached), (int)GCS_BLOCK_DEDUP_CACHED_REPLY); @@ -1867,24 +1865,22 @@ UT_TEST(u33_pending_x_arm_terminates_inflight_legacy_s_exactly) /* Duplicate DONE is idempotent and removes both old identities from the * denial replay set; unrelated entries remain in their original states. */ for (i = 0; i < 2; i++) { - UT_ASSERT(cluster_gcs_block_dedup_mark_done(0, &denied_keys[i], &tag, - PCM_TRANS_N_TO_S)); - UT_ASSERT(cluster_gcs_block_dedup_mark_done(0, &denied_keys[i], &tag, - PCM_TRANS_N_TO_S)); + UT_ASSERT(cluster_gcs_block_dedup_mark_done(0, &denied_keys[i], &tag, PCM_TRANS_N_TO_S)); + UT_ASSERT(cluster_gcs_block_dedup_mark_done(0, &denied_keys[i], &tag, PCM_TRANS_N_TO_S)); } UT_ASSERT_EQ(cluster_gcs_block_dedup_pending_x_deny_next(0, &tag, &denied), GCS_BLOCK_PENDING_X_DENY_NOT_FOUND); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( 0, &unrelated, other_tag, PCM_TRANS_N_TO_S, 1000, true, &cached), (int)GCS_BLOCK_DEDUP_IN_FLIGHT_DUPLICATE); - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( - 0, &writer, tag, PCM_TRANS_N_TO_X, 1000, true, &cached), + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register(0, &writer, tag, PCM_TRANS_N_TO_X, + 1000, true, &cached), (int)GCS_BLOCK_DEDUP_IN_FLIGHT_DUPLICATE); /* Once the queue round is over, a reader with a fresh identity is admitted * normally; the two denied identities cannot poison the new request. */ - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( - 0, &fresh, tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register(0, &fresh, tag, PCM_TRANS_N_TO_S, + 1000, true, &cached), (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); } @@ -1905,8 +1901,8 @@ UT_TEST(u34_pending_x_new_reader_exact_deny_precedes_cached_shortcut) reset_fake_dedup(2, FAKE_DEDUP_CAP); memset(&granted, 0, sizeof(granted)); memset(page, 0x71, sizeof(page)); - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( - 0, &key, tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register(0, &key, tag, PCM_TRANS_N_TO_S, + 1000, true, &cached), (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); UT_ASSERT(cluster_gcs_block_dedup_set_request_flags_exact( 0, &key, &tag, PCM_TRANS_N_TO_S, GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND)); @@ -1917,37 +1913,34 @@ UT_TEST(u34_pending_x_new_reader_exact_deny_precedes_cached_shortcut) granted.requester_backend_id = key.requester_backend_id; granted.transition_id = (uint8)PCM_TRANS_N_TO_S; granted.status = (uint8)GCS_BLOCK_REPLY_GRANTED; - GcsBlockReplyHeaderSetForwardingMasterNode(&granted, - GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + GcsBlockReplyHeaderSetForwardingMasterNode(&granted, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); cluster_gcs_block_dedup_install_reply(0, &key, GCS_BLOCK_REPLY_GRANTED, &granted, page); - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pending_x_deny_exact( - 0, &key, &tag, PCM_TRANS_N_TO_S, &denied), - (int)GCS_BLOCK_PENDING_X_DENY_NEW); + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pending_x_deny_exact(0, &key, &tag, PCM_TRANS_N_TO_S, &denied), + (int)GCS_BLOCK_PENDING_X_DENY_NEW); UT_ASSERT_EQ((int)denied.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); UT_ASSERT_EQ((int)denied.request_flags, - (int)(GCS_BLOCK_DEDUP_REQUEST_F_PINNED - | GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND)); - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pending_x_deny_exact( - 0, &key, &tag, PCM_TRANS_N_TO_S, &denied), - (int)GCS_BLOCK_PENDING_X_DENY_REPLAY); - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( - 0, &key, tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + (int)(GCS_BLOCK_DEDUP_REQUEST_F_PINNED | GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND)); + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pending_x_deny_exact(0, &key, &tag, PCM_TRANS_N_TO_S, &denied), + (int)GCS_BLOCK_PENDING_X_DENY_REPLAY); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register(0, &key, tag, PCM_TRANS_N_TO_S, + 1000, true, &cached), (int)GCS_BLOCK_DEDUP_CACHED_REPLY); UT_ASSERT_EQ((int)cached.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); /* Missing and mismatched identities are zero-mutation invalid results. */ - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pending_x_deny_exact( - 0, &absent, &tag, PCM_TRANS_N_TO_S, &denied), + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pending_x_deny_exact(0, &absent, &tag, + PCM_TRANS_N_TO_S, &denied), (int)GCS_BLOCK_PENDING_X_DENY_INVALID); UT_ASSERT(!cluster_gcs_block_dedup_set_request_flags_exact( 0, &key, &tag, PCM_TRANS_N_TO_X, GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND)); - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( - 0, &key, tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register(0, &key, tag, PCM_TRANS_N_TO_S, + 1000, true, &cached), (int)GCS_BLOCK_DEDUP_CACHED_REPLY); UT_ASSERT_EQ((int)cached.request_flags, - (int)(GCS_BLOCK_DEDUP_REQUEST_F_PINNED - | GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND)); + (int)(GCS_BLOCK_DEDUP_REQUEST_F_PINNED | GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND)); } diff --git a/src/test/cluster_unit/test_cluster_ic.c b/src/test/cluster_unit/test_cluster_ic.c index 5a88a437d5..5ae45dba80 100644 --- a/src/test/cluster_unit/test_cluster_ic.c +++ b/src/test/cluster_unit/test_cluster_ic.c @@ -735,8 +735,7 @@ UT_TEST(test_hello_smart_fusion_capability_gate) | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 - | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1 - | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1); + | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1 | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1); cluster_smart_fusion = false; cluster_interconnect_tier = CLUSTER_IC_TIER_STUB; diff --git a/src/test/cluster_unit/test_cluster_lms_outbound.c b/src/test/cluster_unit/test_cluster_lms_outbound.c index 1146beb6e2..cec1575537 100644 --- a/src/test/cluster_unit/test_cluster_lms_outbound.c +++ b/src/test/cluster_unit/test_cluster_lms_outbound.c @@ -82,10 +82,11 @@ static int ut_pcm_x_boundary_note_count = 0; static uint8 ut_pcm_x_boundary_msg_types[8]; void -cluster_lms_note_pcm_x_image_ready_boundary( - uint8 msg_type, const char *boundary, int result, int runtime_state, bool fence_enforcing, - bool fence_allowed, uint32 dest_node_id, uint64 request_id, uint64 ticket_id, uint64 grant_generation, - uint64 image_id) +cluster_lms_note_pcm_x_image_ready_boundary(uint8 msg_type, const char *boundary, int result, + int runtime_state, bool fence_enforcing, + bool fence_allowed, uint32 dest_node_id, + uint64 request_id, uint64 ticket_id, + uint64 grant_generation, uint64 image_id) { if (ut_pcm_x_boundary_note_count < (int)lengthof(ut_pcm_x_boundary_msg_types)) ut_pcm_x_boundary_msg_types[ut_pcm_x_boundary_note_count] = msg_type; @@ -104,7 +105,7 @@ cluster_lms_note_pcm_x_image_ready_boundary( bool cluster_sf_peer_capability_generation_matches(int32 peer_id, uint32 required_capabilities, - uint32 expected_generation) + uint32 expected_generation) { if (peer_id < 0 || peer_id >= CLUSTER_MAX_NODES || required_capabilities == 0) return false; @@ -312,8 +313,7 @@ cluster_ic_send_envelope(uint8 msg_type, int32 dest_node_id, const void *payload ut_sent_log[ut_sent_n].dest = dest_node_id; ut_sent_log[ut_sent_n].marker = payload_len > 0 ? *(const uint8 *)payload : 0; ut_sent_log[ut_sent_n].payload_len = payload_len; - if (msg_type == PGRAC_IC_MSG_GCS_BLOCK_REPLY - && payload_len >= sizeof(GcsBlockReplyHeader)) + if (msg_type == PGRAC_IC_MSG_GCS_BLOCK_REPLY && payload_len >= sizeof(GcsBlockReplyHeader)) memcpy(&ut_sent_log[ut_sent_n].reply_header, payload, sizeof(GcsBlockReplyHeader)); } ut_sent_n++; @@ -590,10 +590,10 @@ UT_TEST(test_pcm_x_image_ready_and_prepare_transport_boundaries_are_observable) ut_reset_log(); memset(&payload, 0, sizeof(payload)); - UT_ASSERT(cluster_lms_outbound_enqueue( - 0, PGRAC_IC_MSG_PCM_X_IMAGE_READY, UT_PEER_X, &payload, sizeof(payload))); - UT_ASSERT(cluster_lms_outbound_enqueue( - 0, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, UT_PEER_Y, &payload, sizeof(payload))); + UT_ASSERT(cluster_lms_outbound_enqueue(0, PGRAC_IC_MSG_PCM_X_IMAGE_READY, UT_PEER_X, &payload, + sizeof(payload))); + UT_ASSERT(cluster_lms_outbound_enqueue(0, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, UT_PEER_Y, &payload, + sizeof(payload))); UT_ASSERT_EQ(cluster_lms_outbound_drain_send(0), 2); UT_ASSERT_EQ(ut_pcm_x_boundary_note_count, 2); UT_ASSERT_EQ((int)ut_pcm_x_boundary_msg_types[0], (int)PGRAC_IC_MSG_PCM_X_IMAGE_READY); @@ -635,8 +635,7 @@ UT_TEST(test_zero_block_reply_is_expanded_by_data_owner) UT_ASSERT_EQ((int)ut_sent_log[1].msg_type, (int)PGRAC_IC_MSG_GCS_BLOCK_REPLY); UT_ASSERT_EQ((int)ut_sent_log[1].payload_len, (int)GCS_BLOCK_REPLY_PAYLOAD_TOTAL_SIZE); UT_ASSERT_EQ(ut_sent_log[1].reply_header.request_id, hdr.request_id); - UT_ASSERT_EQ((int)ut_sent_log[1].reply_header.status, - (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ((int)ut_sent_log[1].reply_header.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); UT_ASSERT_EQ(ut_sent_log[1].reply_header.checksum, UINT32_C(0xA55A7E11)); } @@ -695,8 +694,8 @@ UT_TEST(test_cap_bound_frame_drops_on_connection_generation_drift) ut_reset_log(); ut_peer_capabilities[UT_PEER_X] = cap; ut_peer_cap_generation[UT_PEER_X] = 18; - UT_ASSERT(cluster_lms_outbound_enqueue_cap_bound( - 0, PGRAC_IC_MSG_PCM_X_REVOKE, UT_PEER_X, &marker, sizeof(marker), cap, 17)); + UT_ASSERT(cluster_lms_outbound_enqueue_cap_bound(0, PGRAC_IC_MSG_PCM_X_REVOKE, UT_PEER_X, + &marker, sizeof(marker), cap, 17)); UT_ASSERT_EQ(cluster_lms_outbound_drain_send(0), 0); UT_ASSERT_EQ(cluster_lms_outbound_depth(0), 0); UT_ASSERT_EQ(ut_count_marker(marker), 0); @@ -710,8 +709,8 @@ UT_TEST(test_cap_bound_frame_drops_on_capability_downgrade) ut_reset_log(); ut_peer_cap_generation[UT_PEER_X] = 21; - UT_ASSERT(cluster_lms_outbound_enqueue_cap_bound( - 0, PGRAC_IC_MSG_PCM_X_REVOKE, UT_PEER_X, &marker, sizeof(marker), cap, 21)); + UT_ASSERT(cluster_lms_outbound_enqueue_cap_bound(0, PGRAC_IC_MSG_PCM_X_REVOKE, UT_PEER_X, + &marker, sizeof(marker), cap, 21)); UT_ASSERT_EQ(cluster_lms_outbound_drain_send(0), 0); UT_ASSERT_EQ(cluster_lms_outbound_depth(0), 0); UT_ASSERT_EQ(ut_count_marker(marker), 0); @@ -726,8 +725,8 @@ UT_TEST(test_cap_bound_frame_sends_on_exact_connection_capability) ut_reset_log(); ut_peer_capabilities[UT_PEER_X] = cap; ut_peer_cap_generation[UT_PEER_X] = 34; - UT_ASSERT(cluster_lms_outbound_enqueue_cap_bound( - 0, PGRAC_IC_MSG_PCM_X_REVOKE, UT_PEER_X, &marker, sizeof(marker), cap, 34)); + UT_ASSERT(cluster_lms_outbound_enqueue_cap_bound(0, PGRAC_IC_MSG_PCM_X_REVOKE, UT_PEER_X, + &marker, sizeof(marker), cap, 34)); UT_ASSERT_EQ(cluster_lms_outbound_drain_send(0), 1); UT_ASSERT_EQ(cluster_lms_outbound_depth(0), 0); UT_ASSERT_EQ(ut_count_marker(marker), 1); diff --git a/src/test/cluster_unit/test_cluster_pcm_direct_init.c b/src/test/cluster_unit/test_cluster_pcm_direct_init.c index a2e26eb5c1..b7403f33fd 100644 --- a/src/test/cluster_unit/test_cluster_pcm_direct_init.c +++ b/src/test/cluster_unit/test_cluster_pcm_direct_init.c @@ -441,16 +441,21 @@ UT_TEST(test_precrit_vm_barrier_refusal_unwinds_to_caller) free(bufmgr); } if (heapam != NULL) { - static const char *const pretoast_order[] - = { "PGRAC: vm barrier unwind (update pre-toast)", - "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", "cluster_heap_vm_barrier_warm", - "ReleaseBuffer(vmbuffer)", "vmbuffer = InvalidBuffer", - "cluster_heap_lock_with_vm_repin", "goto l2;" }; + static const char *const pretoast_order[] = { "PGRAC: vm barrier unwind (update pre-toast)", + "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", + "cluster_heap_vm_barrier_warm", + "ReleaseBuffer(vmbuffer)", + "vmbuffer = InvalidBuffer", + "cluster_heap_lock_with_vm_repin", + "goto l2;" }; static const char *const requalify_order[] = { "PGRAC: vm barrier unwind (update requalify)", - "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", "cluster_heap_vm_barrier_warm", - "ReleaseBuffer(vmbuffer)", "vmbuffer = InvalidBuffer", - "cluster_heap_lock_with_vm_repin", "goto l2;" }; + "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", + "cluster_heap_vm_barrier_warm", + "ReleaseBuffer(vmbuffer)", + "vmbuffer = InvalidBuffer", + "cluster_heap_lock_with_vm_repin", + "goto l2;" }; static const char *const reacquire_order[] = { "PGRAC: vm barrier unwind (update reacquire)", "LockBuffer(newbuf, BUFFER_LOCK_UNLOCK)", @@ -458,11 +463,13 @@ UT_TEST(test_precrit_vm_barrier_refusal_unwinds_to_caller) "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", "cluster_heap_vm_barrier_warm", "goto l_pgrac_reacquire;" }; - static const char *const delete_order[] - = { "PGRAC: vm barrier unwind (delete requalify)", - "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", "cluster_heap_vm_barrier_warm", - "ReleaseBuffer(vmbuffer)", "vmbuffer = InvalidBuffer", - "cluster_heap_lock_with_vm_repin", "goto l1;" }; + static const char *const delete_order[] = { "PGRAC: vm barrier unwind (delete requalify)", + "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", + "cluster_heap_vm_barrier_warm", + "ReleaseBuffer(vmbuffer)", + "vmbuffer = InvalidBuffer", + "cluster_heap_lock_with_vm_repin", + "goto l1;" }; /* The warm helper itself may only run with no content lock held: * it must take and drop the map-page lock, nothing else. */ @@ -497,26 +504,27 @@ UT_TEST(test_heap_update_drops_vm_pin_across_heap_pcm_wait) const char *update = strstr(heapam, "\nheap_update("); const char *update_lock = update != NULL ? strstr(update, "cluster_heap_lock_with_vm_repin(") : NULL; - const char *repin_branch = update != NULL - ? strstr(update, "if (vmbuffer == InvalidBuffer && PageIsAllVisible(page))") - : NULL; + const char *repin_branch + = update != NULL + ? strstr(update, "if (vmbuffer == InvalidBuffer && PageIsAllVisible(page))") + : NULL; const char *repin_branch_end = repin_branch != NULL ? strstr(repin_branch, "goto l2;") : NULL; - const char *repin_branch_helper = repin_branch != NULL - ? strstr(repin_branch, "cluster_heap_lock_with_vm_repin(") : NULL; - const char *success_tail = update != NULL - ? strstr(update, "recptr = log_heap_update(") - : NULL; - const char *vm_unlock = success_tail != NULL - ? strstr(success_tail, - "if (vm_locked)\n\t\tLockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);") - : NULL; - const char *vm_release = vm_unlock != NULL - ? strstr(vm_unlock, "ReleaseBuffer(vmbuffer);") - : NULL; + const char *repin_branch_helper + = repin_branch != NULL ? strstr(repin_branch, "cluster_heap_lock_with_vm_repin(") + : NULL; + const char *success_tail + = update != NULL ? strstr(update, "recptr = log_heap_update(") : NULL; + const char *vm_unlock + = success_tail != NULL + ? strstr(success_tail, + "if (vm_locked)\n\t\tLockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);") + : NULL; + const char *vm_release + = vm_unlock != NULL ? strstr(vm_unlock, "ReleaseBuffer(vmbuffer);") : NULL; const char *heap_unlock = vm_unlock != NULL - ? strstr(vm_unlock, "LockBuffer(buffer, BUFFER_LOCK_UNLOCK);") - : NULL; + ? strstr(vm_unlock, "LockBuffer(buffer, BUFFER_LOCK_UNLOCK);") + : NULL; static const char *const helper_order[] = { "visibilitymap_pin(relation, heap_block, vmbuffer)", "ReleaseBuffer(*vmbuffer)", @@ -545,8 +553,8 @@ UT_TEST(test_heap_update_drops_vm_pin_across_heap_pcm_wait) const char *repin = strstr(visibilitymap, "\nvisibilitymap_pin_recent("); const char *repin_end = repin != NULL ? strstr(repin, "\n}\n") : NULL; static const char *const repin_order[] - = { "HEAPBLK_TO_MAPBLOCK(heapBlk)", "ReadRecentBuffer(", - "VISIBILITYMAP_FORKNUM", "recent_buffer" }; + = { "HEAPBLK_TO_MAPBLOCK(heapBlk)", "ReadRecentBuffer(", "VISIBILITYMAP_FORKNUM", + "recent_buffer" }; UT_ASSERT(repin != NULL); UT_ASSERT(repin_end != NULL); @@ -595,9 +603,11 @@ UT_TEST(test_cross_page_heap_pair_barrier_refusal_retries_unlocked) relation = strstr(hio, "\nRelationGetBufferForTuple("); relation_end = relation != NULL ? strstr(relation, "\n}\n") : NULL; lower_branch = relation != NULL ? strstr(relation, "else if (otherBlock < targetBlock)") : NULL; - lower_call = lower_branch != NULL ? strstr(lower_branch, "cluster_hio_lock_buffer_pair(") : NULL; + lower_call + = lower_branch != NULL ? strstr(lower_branch, "cluster_hio_lock_buffer_pair(") : NULL; upper_branch = lower_branch != NULL ? strstr(lower_branch, "\n\t\telse\n\t\t{") : NULL; - upper_call = upper_branch != NULL ? strstr(upper_branch, "cluster_hio_lock_buffer_pair(") : NULL; + upper_call + = upper_branch != NULL ? strstr(upper_branch, "cluster_hio_lock_buffer_pair(") : NULL; extension = relation != NULL ? strstr(relation, "Reacquire locks if necessary") : NULL; extension_call = extension != NULL ? strstr(extension, "cluster_hio_lock_buffer_pair(") : NULL; @@ -609,8 +619,7 @@ UT_TEST(test_cross_page_heap_pair_barrier_refusal_retries_unlocked) UT_ASSERT(relation_end != NULL); if (helper != NULL && helper_end != NULL) assert_ordered(helper, helper_order, lengthof(helper_order)); - UT_ASSERT(pins != NULL && pins_end != NULL && pins_call != NULL - && pins_call < pins_end); + UT_ASSERT(pins != NULL && pins_end != NULL && pins_call != NULL && pins_call < pins_end); UT_ASSERT(lower_branch != NULL && lower_call != NULL && upper_branch != NULL && lower_call < upper_branch); UT_ASSERT(upper_branch != NULL && upper_call != NULL && relation_end != NULL diff --git a/src/test/cluster_unit/test_cluster_pcm_lock.c b/src/test/cluster_unit/test_cluster_pcm_lock.c index 590858ffe6..8ddf14e20b 100644 --- a/src/test/cluster_unit/test_cluster_pcm_lock.c +++ b/src/test/cluster_unit/test_cluster_pcm_lock.c @@ -531,17 +531,16 @@ cluster_shmem_register_region(const ClusterShmemRegion *region pg_attribute_unus void cluster_injection_run(const char *name) { - if (fake_acquire_entry_handoff_armed - && strcmp(name, "cluster-pcm-acquire-entry") == 0) { + if (fake_acquire_entry_handoff_armed && strcmp(name, "cluster-pcm-acquire-entry") == 0) { fake_acquire_entry_handoff_armed = false; cluster_injection_armed_count = 0; - UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition( - fake_acquire_entry_handoff_tag, fake_acquire_entry_handoff_release, - fake_acquire_entry_handoff_source), + UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition(fake_acquire_entry_handoff_tag, + fake_acquire_entry_handoff_release, + fake_acquire_entry_handoff_source), 1); - UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition( - fake_acquire_entry_handoff_tag, PCM_TRANS_N_TO_X, - fake_acquire_entry_handoff_target), + UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition(fake_acquire_entry_handoff_tag, + PCM_TRANS_N_TO_X, + fake_acquire_entry_handoff_target), 1); } } @@ -2297,15 +2296,14 @@ UT_TEST(test_pcm_x_slotless_ack_floor_fences_stale_source_before_prepare) cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); cluster_node_id = 1; cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); - UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9011), - PCM_PENDING_X_RESERVE_OK); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9011), PCM_PENDING_X_RESERVE_OK); /* Source A (node 0) is pre-acked by the transfer driver. Apply the two * production-equivalent state effects of non-source B's ACK (node 1), then * use the bounded source contract below to prove their real handler order. */ UT_ASSERT(cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_S_TO_N_INVALIDATE, 1)); - cluster_pcm_lock_pi_watermark_scn_advance(tag, watermark_scn, - CLUSTER_PCM_WM_SRC_ACK_SLOTLESS, 1, 9011, 17); + cluster_pcm_lock_pi_watermark_scn_advance(tag, watermark_scn, CLUSTER_PCM_WM_SRC_ACK_SLOTLESS, + 1, 9011, 17); UT_ASSERT(cluster_pcm_lock_pi_watermark_prov_query(tag, &provenance)); UT_ASSERT_EQ((int)provenance.source, (int)CLUSTER_PCM_WM_SRC_ACK_SLOTLESS); UT_ASSERT_EQ(provenance.sender_node, 1); @@ -2327,8 +2325,7 @@ UT_TEST(test_pcm_x_slotless_ack_floor_fences_stale_source_before_prepare) UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_S); UT_ASSERT_EQ(after.s_holders_bitmap, UINT32_C(1) << 0); UT_ASSERT_EQ(after.pending_x_requester_node, 3); - UT_ASSERT_EQ((uint64)cluster_pcm_lock_pi_watermark_scn_query(tag), - (uint64)watermark_scn); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_pi_watermark_scn_query(tag), (uint64)watermark_scn); /* Pin the production ordering: B's exact ACK must publish W before the * bitmap can drive type 49. IMAGE_READY must then consume that current @@ -2337,25 +2334,21 @@ UT_TEST(test_pcm_x_slotless_ack_floor_fences_stale_source_before_prepare) if (source == NULL) return; ack_handler = strstr(source, "\ncluster_gcs_handle_block_invalidate_ack_envelope("); - ack_end = ack_handler != NULL - ? strstr(ack_handler, "\n/* PGRAC: spec-7.2 flip") - : NULL; + ack_end = ack_handler != NULL ? strstr(ack_handler, "\n/* PGRAC: spec-7.2 flip") : NULL; ack_match = ack_handler != NULL ? strstr(ack_handler, "gcs_block_pcm_x_queue_invalidate_ack_match(") : NULL; - holder_remove = ack_match != NULL - ? strstr(ack_match, "cluster_pcm_lock_apply_gcs_transition(") - : NULL; + holder_remove + = ack_match != NULL ? strstr(ack_match, "cluster_pcm_lock_apply_gcs_transition(") : NULL; watermark_advance = holder_remove != NULL ? strstr(holder_remove, "cluster_pcm_lock_pi_watermark_scn_advance(") : NULL; - bitmap_replace = watermark_advance != NULL - ? strstr(watermark_advance, - "cluster_pcm_x_master_drive_bitmap_replace_exact(") - : NULL; - drive = bitmap_replace != NULL - ? strstr(bitmap_replace, "gcs_block_pcm_x_master_drive_tag(") - : NULL; + bitmap_replace + = watermark_advance != NULL + ? strstr(watermark_advance, "cluster_pcm_x_master_drive_bitmap_replace_exact(") + : NULL; + drive = bitmap_replace != NULL ? strstr(bitmap_replace, "gcs_block_pcm_x_master_drive_tag(") + : NULL; UT_ASSERT_NOT_NULL(ack_handler); UT_ASSERT_NOT_NULL(ack_end); UT_ASSERT_NOT_NULL(ack_match); @@ -2371,29 +2364,26 @@ UT_TEST(test_pcm_x_slotless_ack_floor_fences_stale_source_before_prepare) ready_handler = strstr(source, "\ncluster_gcs_handle_pcm_x_image_ready_envelope("); ready_end = ready_handler != NULL - ? strstr(ready_handler, "\ncluster_gcs_handle_pcm_x_prepare_grant_envelope(") - : NULL; - floor_query = ready_handler != NULL - ? strstr(ready_handler, "cluster_pcm_lock_pi_watermark_scn_query(") - : NULL; - floor_verdict = floor_query != NULL - ? strstr(floor_query, "gcs_block_lost_write_verdict(") + ? strstr(ready_handler, "\ncluster_gcs_handle_pcm_x_prepare_grant_envelope(") : NULL; + floor_query = ready_handler != NULL + ? strstr(ready_handler, "cluster_pcm_lock_pi_watermark_scn_query(") + : NULL; + floor_verdict + = floor_query != NULL ? strstr(floor_query, "gcs_block_lost_write_verdict(") : NULL; image_ready = ready_handler != NULL - ? strstr(ready_handler, "cluster_pcm_x_master_image_ready_exact(") - : NULL; - prepare = image_ready != NULL - ? strstr(image_ready, "PGRAC_IC_MSG_PCM_X_PREPARE_GRANT") - : NULL; + ? strstr(ready_handler, "cluster_pcm_x_master_image_ready_exact(") + : NULL; + prepare = image_ready != NULL ? strstr(image_ready, "PGRAC_IC_MSG_PCM_X_PREPARE_GRANT") : NULL; UT_ASSERT_NOT_NULL(ready_handler); UT_ASSERT_NOT_NULL(ready_end); UT_ASSERT_NOT_NULL(image_ready); UT_ASSERT_NOT_NULL(prepare); - source_floor_gate - = ready_handler != NULL && ready_end != NULL && floor_query != NULL - && floor_verdict != NULL && image_ready != NULL && prepare != NULL - && ready_handler < floor_query && floor_query < floor_verdict - && floor_verdict < image_ready && image_ready < prepare && prepare < ready_end; + source_floor_gate = ready_handler != NULL && ready_end != NULL && floor_query != NULL + && floor_verdict != NULL && image_ready != NULL && prepare != NULL + && ready_handler < floor_query && floor_query < floor_verdict + && floor_verdict < image_ready && image_ready < prepare + && prepare < ready_end; UT_ASSERT(source_floor_gate); free(source); } diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c index 406614ee41..6f0ad76d9f 100644 --- a/src/test/cluster_unit/test_cluster_pcm_own.c +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -112,30 +112,30 @@ UT_TEST(test_writer_activation_fence_blocks_revoke_until_exact_clear) uint64 writer_token = 0; reset_fixture(); - UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact( - 0, 0, PCM_OWN_FLAG_GRANT_PENDING, &writer_token), - CLUSTER_PCM_OWN_OK); - UT_ASSERT_EQ(cluster_pcm_own_writer_grant_commit_exact( - 0, 0, writer_token, &committed_generation), - CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ( + cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_GRANT_PENDING, &writer_token), + CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ( + cluster_pcm_own_writer_grant_commit_exact(0, 0, writer_token, &committed_generation), + CLUSTER_PCM_OWN_OK); UT_ASSERT_EQ(committed_generation, 1); assert_entry(1, writer_token, 0); assert_writer_activation(writer_token); /* The committed X and its not-yet-activated writer are one shared * linearization. A downgrade cannot reserve the same tuple. */ - UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact( - 0, 1, PCM_OWN_FLAG_REVOKING, &revoke_token), - CLUSTER_PCM_OWN_BUSY); + UT_ASSERT_EQ( + cluster_pcm_own_reservation_begin_exact(0, 1, PCM_OWN_FLAG_REVOKING, &revoke_token), + CLUSTER_PCM_OWN_BUSY); UT_ASSERT_EQ(revoke_token, 0); assert_entry(1, writer_token, 0); assert_writer_activation(writer_token); UT_ASSERT(!cluster_pcm_own_gen_bump_checked(0, NULL)); pg_atomic_write_u64(&ClusterPcmOwnArray[0].writer_activation_token, writer_token + 1); - UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact( - 0, 1, PCM_OWN_FLAG_REVOKING, &revoke_token), - CLUSTER_PCM_OWN_CORRUPT); + UT_ASSERT_EQ( + cluster_pcm_own_reservation_begin_exact(0, 1, PCM_OWN_FLAG_REVOKING, &revoke_token), + CLUSTER_PCM_OWN_CORRUPT); pg_atomic_write_u64(&ClusterPcmOwnArray[0].writer_activation_token, writer_token); /* A delayed or wrong clear is an exact no-op. */ @@ -148,12 +148,11 @@ UT_TEST(test_writer_activation_fence_blocks_revoke_until_exact_clear) UT_ASSERT_EQ(cluster_pcm_own_writer_activation_clear_exact(0, 1, writer_token), CLUSTER_PCM_OWN_OK); assert_writer_activation(0); - UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact( - 0, 1, PCM_OWN_FLAG_REVOKING, &revoke_token), - CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ( + cluster_pcm_own_reservation_begin_exact(0, 1, PCM_OWN_FLAG_REVOKING, &revoke_token), + CLUSTER_PCM_OWN_OK); UT_ASSERT_EQ(revoke_token, writer_token + 1); - UT_ASSERT_EQ(cluster_pcm_own_reservation_abort_exact( - 0, 1, revoke_token, PCM_OWN_FLAG_REVOKING), + UT_ASSERT_EQ(cluster_pcm_own_reservation_abort_exact(0, 1, revoke_token, PCM_OWN_FLAG_REVOKING), CLUSTER_PCM_OWN_OK); UT_ASSERT(cluster_pcm_own_gen_bump_checked(0, &committed_generation)); UT_ASSERT_EQ(committed_generation, 2); @@ -425,28 +424,28 @@ UT_TEST(test_share_cover_reverify_accepts_stable_successor_grant) /* Once content authority is held, a stable current S/X successor is the * exact node-level grant for a read. Generation drift alone must not open * a fresh legacy reservation from S (the forbidden S_NEW shape). */ - UT_ASSERT(cluster_pcm_x_cached_cover_reverify_accepts( - (uint8)PCM_LOCK_MODE_S, UINT64_C(13), UINT64_C(14), (uint8)PCM_STATE_S, 0)); - UT_ASSERT(cluster_pcm_x_cached_cover_reverify_accepts( - (uint8)PCM_LOCK_MODE_S, UINT64_C(13), UINT64_C(14), (uint8)PCM_STATE_X, 0)); + UT_ASSERT(cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_S, UINT64_C(13), + UINT64_C(14), (uint8)PCM_STATE_S, 0)); + UT_ASSERT(cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_S, UINT64_C(13), + UINT64_C(14), (uint8)PCM_STATE_X, 0)); /* A writer keeps the stricter generation-exact gate and must re-enter the * convert queue after any ownership round. A non-covering or live * lifecycle remains closed for both modes. */ - UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts( - (uint8)PCM_LOCK_MODE_X, UINT64_C(13), UINT64_C(14), (uint8)PCM_STATE_X, 0)); - UT_ASSERT(cluster_pcm_x_cached_cover_reverify_accepts( - (uint8)PCM_LOCK_MODE_X, UINT64_C(14), UINT64_C(14), (uint8)PCM_STATE_X, 0)); - UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts( - (uint8)PCM_LOCK_MODE_X, UINT64_C(14), UINT64_C(14), (uint8)PCM_STATE_S, 0)); - UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts( - (uint8)PCM_LOCK_MODE_S, UINT64_C(14), UINT64_C(14), (uint8)PCM_STATE_S, - PCM_OWN_FLAG_GRANT_PENDING)); - UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts( - (uint8)PCM_LOCK_MODE_S, UINT64_C(14), UINT64_C(14), (uint8)PCM_STATE_S, - PCM_OWN_FLAG_REVOKING)); - UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts( - (uint8)PCM_LOCK_MODE_N, UINT64_C(14), UINT64_C(14), (uint8)PCM_STATE_X, 0)); + UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_X, UINT64_C(13), + UINT64_C(14), (uint8)PCM_STATE_X, 0)); + UT_ASSERT(cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_X, UINT64_C(14), + UINT64_C(14), (uint8)PCM_STATE_X, 0)); + UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_X, UINT64_C(14), + UINT64_C(14), (uint8)PCM_STATE_S, 0)); + UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_S, UINT64_C(14), + UINT64_C(14), (uint8)PCM_STATE_S, + PCM_OWN_FLAG_GRANT_PENDING)); + UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_S, UINT64_C(14), + UINT64_C(14), (uint8)PCM_STATE_S, + PCM_OWN_FLAG_REVOKING)); + UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_N, UINT64_C(14), + UINT64_C(14), (uint8)PCM_STATE_X, 0)); } UT_TEST(test_revoke_commit_is_exact_and_classifies_live_races) @@ -930,14 +929,11 @@ UT_TEST(test_pending_x_denied_retry_leaves_master_invalidate_gap) { static const char *const retry_contract[] = { "cluster_pcm_own_abort_grant_reservation", - "cluster_bufmgr_pcm_pending_x_retry_delay_ms(wait_index)", - "WaitLatch", + "cluster_bufmgr_pcm_pending_x_retry_delay_ms(wait_index)", "WaitLatch", "cluster_bufmgr_pcm_begin_grant_reservation_wait" }; static const char *const delay_contract[] - = { "cluster_gcs_block_starvation_backoff_ms", - "cluster_lmon_main_loop_interval", - "retry_delay_ms <<= shift", - "retry_delay_ms * 2" }; + = { "cluster_gcs_block_starvation_backoff_ms", "cluster_lmon_main_loop_interval", + "retry_delay_ms <<= shift", "retry_delay_ms * 2" }; char *source = read_bufmgr_source(); /* A queue INVALIDATE that met mirror-N + GRANT_PENDING returns BUSY and @@ -946,8 +942,8 @@ UT_TEST(test_pending_x_denied_retry_leaves_master_invalidate_gap) * absent for strictly longer than that master retry delay. Otherwise the * two exponential schedules phase-lock forever: each reader request * republishes GRANT_PENDING just as every INVALIDATE retry arrives. */ - assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_retry_denied_rearm(", - "\nstatic ", retry_contract, lengthof(retry_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_retry_denied_rearm(", "\nstatic ", + retry_contract, lengthof(retry_contract)); assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_pending_x_retry_delay_ms(", "\nstatic ", delay_contract, lengthof(delay_contract)); free(source); @@ -1172,8 +1168,8 @@ UT_TEST(test_queue_contract_exposes_opaque_retained_revoke_api) typedef ClusterPcmOwnResult (*BeginRevokeFn)(BufferDesc *, const ClusterPcmOwnSnapshot *, ClusterPcmOwnSnapshot *); typedef ClusterPcmOwnResult (*PrepareNSourceFn)(BufferDesc *, const ClusterPcmOwnSnapshot *, - ClusterPcmOwnSnapshot *, char *, XLogRecPtr *, - uint64 *); + ClusterPcmOwnSnapshot *, char *, XLogRecPtr *, + uint64 *); typedef ClusterPcmOwnResult (*PrepareSSourceFn)( BufferDesc *, const ClusterPcmOwnSnapshot *, SCN, ClusterPcmOwnSnapshot *, char *, XLogRecPtr *, uint64 *, ClusterPcmOwnSourcePrepareRefusal *); @@ -1279,10 +1275,9 @@ UT_TEST(test_queue_s_source_dirty_flush_makes_progress_and_reports_exact_refusal * pin and flush it before opening REVOKING, then retry. Keep I/O and both * content-lock refusals separately diagnosed so a native stall cannot be * collapsed back into an opaque materialize-begin BUSY. */ - assert_ordered_in_function( - source, "\ncluster_bufmgr_pcm_own_prepare_s_source_image(", - "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_abort_s_revoke(", prepare_contract, - lengthof(prepare_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_prepare_s_source_image(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_abort_s_revoke(", + prepare_contract, lengthof(prepare_contract)); free(source); } @@ -1372,13 +1367,12 @@ UT_TEST(test_pcm_tracking_uses_one_tag_gate_for_acquire_direct_init_and_eviction UT_ASSERT_NOT_NULL(victim_end); if (should_track != NULL && should_track_end != NULL) assert_source_range_contains(should_track, should_track_end, - "cluster_pcm_x_buffer_tag_tracked"); + "cluster_pcm_x_buffer_tag_tracked"); if (invalidate_commit != NULL && invalidate_tail != NULL) assert_source_range_contains(invalidate_commit, invalidate_tail, - "cluster_pcm_x_buffer_tag_tracked"); + "cluster_pcm_x_buffer_tag_tracked"); if (victim != NULL && victim_end != NULL) - assert_source_range_contains(victim, victim_end, - "cluster_pcm_x_buffer_tag_tracked"); + assert_source_range_contains(victim, victim_end, "cluster_pcm_x_buffer_tag_tracked"); /* The dedicated VM/FSM initialization wrapper retains provenance. Its * common tag gate decides whether PCM proof is armed: FSM falls through to @@ -1387,13 +1381,14 @@ UT_TEST(test_pcm_tracking_uses_one_tag_gate_for_acquire_direct_init_and_eviction direct_init_end = strstr(source, "\nvoid\nLockBufferForVisibilityMapPageInit("); UT_ASSERT_NOT_NULL(direct_init); UT_ASSERT_NOT_NULL(direct_init_end); - direct_gate = direct_init != NULL ? strstr(direct_init, "cluster_bufmgr_should_pcm_track(buf)") - : NULL; + direct_gate + = direct_init != NULL ? strstr(direct_init, "cluster_bufmgr_should_pcm_track(buf)") : NULL; arm = direct_init != NULL ? strstr(direct_init, "cluster_bufmgr_pcm_arm_direct_init") : NULL; consume = direct_init != NULL ? strstr(direct_init, "cluster_bufmgr_pcm_gate_direct_init") : NULL; content_lock = direct_init != NULL - ? strstr(direct_init, "LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE)") + ? strstr(direct_init, + "LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE)") : NULL; UT_ASSERT_NOT_NULL(direct_gate); UT_ASSERT_NOT_NULL(arm); @@ -2136,26 +2131,29 @@ UT_TEST(test_queue_passive_n_mirror_is_never_gcs_ship_authority) { static const char *const probe_contract[] = { "LockBufHdr", "cluster_bufmgr_pcm_current_image_locked", "UnlockBufHdr" }; - static const char *const copy_contract[] = { "LockBufHdr", - "cluster_bufmgr_pcm_current_image_locked", - "cluster_bufmgr_pin_for_gcs_locked", - "LWLockConditionalAcquire(content_lock, LW_SHARED)", - "cluster_bufmgr_pcm_current_image_locked", - "FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL)", - "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED", - "memcpy(dst, page, BLCKSZ)" }; - static const char *const live_sge_contract[] = { "LockBufHdr", - "cluster_bufmgr_pcm_current_image_locked", - "cluster_bufmgr_pin_for_gcs_locked", - "LWLockConditionalAcquire(content_lock, LW_SHARED)", - "cluster_bufmgr_pcm_current_image_locked", - "*out_page_addr = page" }; - static const char *const smart_contract[] = { "LockBufHdr", - "cluster_bufmgr_pcm_current_image_locked", - "cluster_bufmgr_pin_for_gcs_locked", - "LWLockConditionalAcquire(content_lock, LW_SHARED)", - "cluster_bufmgr_pcm_current_image_locked", - "memcpy(dst, page, BLCKSZ)" }; + static const char *const copy_contract[] + = { "LockBufHdr", + "cluster_bufmgr_pcm_current_image_locked", + "cluster_bufmgr_pin_for_gcs_locked", + "LWLockConditionalAcquire(content_lock, LW_SHARED)", + "cluster_bufmgr_pcm_current_image_locked", + "FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL)", + "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED", + "memcpy(dst, page, BLCKSZ)" }; + static const char *const live_sge_contract[] + = { "LockBufHdr", + "cluster_bufmgr_pcm_current_image_locked", + "cluster_bufmgr_pin_for_gcs_locked", + "LWLockConditionalAcquire(content_lock, LW_SHARED)", + "cluster_bufmgr_pcm_current_image_locked", + "*out_page_addr = page" }; + static const char *const smart_contract[] + = { "LockBufHdr", + "cluster_bufmgr_pcm_current_image_locked", + "cluster_bufmgr_pin_for_gcs_locked", + "LWLockConditionalAcquire(content_lock, LW_SHARED)", + "cluster_bufmgr_pcm_current_image_locked", + "memcpy(dst, page, BLCKSZ)" }; char *source = read_bufmgr_source(); assert_ordered_in_function(source, "\ncluster_bufmgr_probe_block_for_gcs(", @@ -2374,8 +2372,7 @@ UT_TEST(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_aut "if (!writer_retry && !holder_retry)", "pg_usleep(1000L)" }; static const char *const snapshot_failure_contract[] = { "release_result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim)", - "entry->claim_cleanup_complete = true", - "entry->phase = PCM_X_WRITER_LEDGER_DEFERRED", + "entry->claim_cleanup_complete = true", "entry->phase = PCM_X_WRITER_LEDGER_DEFERRED", "cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_CORRUPT, buf" }; static const char *const writer_holder_publish_contract[] @@ -2594,9 +2591,8 @@ UT_TEST(test_own_lifecycle_counters_land_on_exact_begin_and_x_commit) "if (handoff_transitioned)", "cluster_pcm_x_stats_note_own_begin();" }; static const char *const direct_init_begin_contract[] - = { "cluster_pcm_own_reservation_begin_exact", "PCM_OWN_FLAG_GRANT_PENDING", - "UnlockBufHdr", "pending_result == CLUSTER_PCM_OWN_OK", - "cluster_pcm_x_stats_note_own_begin();" }; + = { "cluster_pcm_own_reservation_begin_exact", "PCM_OWN_FLAG_GRANT_PENDING", "UnlockBufHdr", + "pending_result == CLUSTER_PCM_OWN_OK", "cluster_pcm_x_stats_note_own_begin();" }; static const char *const x_commit_contract[] = { "cluster_pcm_own_grant_commit_exact", "UnlockBufHdr", "result == CLUSTER_PCM_OWN_OK && new_pcm_state == (uint8)PCM_STATE_X", @@ -2695,8 +2691,8 @@ UT_TEST(test_pcm_x_retain_flush_error_injection_is_exact_and_pre_write) assert_ordered_in_function(source, "\nFlushBuffer(", "\n/*\n * RelationGetNumberOfBlocksInFork", flush_error_contract, lengthof(flush_error_contract)); assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_finish_revoke_retain(", - "\ncluster_bufmgr_pcm_own_release_retained_image(", catch_contract, - lengthof(catch_contract)); + "\ncluster_bufmgr_pcm_own_release_retained_image(", catch_contract, + lengthof(catch_contract)); finish = strstr(source, "\ncluster_bufmgr_pcm_own_finish_revoke_retain("); UT_ASSERT_NOT_NULL(finish); catch = strstr(finish, "PG_CATCH();"); @@ -2710,19 +2706,15 @@ UT_TEST(test_pcm_x_retain_flush_error_injection_is_exact_and_pre_write) UT_TEST(test_writer_activation_diagnostic_covers_commit_clear_and_unguarded_n_boundaries) { - static const char *const commit_contract[] = { - "cluster_pcm_own_writer_grant_commit_exact(", - "buf->pcm_state = new_pcm_state", - "cluster_pcm_own_snapshot_locked(buf, &activation_diag)", - "UnlockBufHdr(buf, buf_state)", - "cluster_pcm_own_activation_diag_emit(\"writer-commit\"" - }; - static const char *const clear_contract[] = { - "cluster_pcm_own_snapshot_locked(buf, &live)", - "cluster_pcm_own_writer_activation_clear_exact(", - "UnlockBufHdr(buf, buf_state)", - "cluster_pcm_own_activation_diag_emit(\"writer-activation-clear\"" - }; + static const char *const commit_contract[] + = { "cluster_pcm_own_writer_grant_commit_exact(", "buf->pcm_state = new_pcm_state", + "cluster_pcm_own_snapshot_locked(buf, &activation_diag)", + "UnlockBufHdr(buf, buf_state)", + "cluster_pcm_own_activation_diag_emit(\"writer-commit\"" }; + static const char *const clear_contract[] + = { "cluster_pcm_own_snapshot_locked(buf, &live)", + "cluster_pcm_own_writer_activation_clear_exact(", "UnlockBufHdr(buf, buf_state)", + "cluster_pcm_own_activation_diag_emit(\"writer-activation-clear\"" }; char *source = read_bufmgr_source(); UT_ASSERT_EQ(sizeof(ClusterPcmOwnSnapshot), 56); diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index aa7e387fb8..805b44f99c 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -407,7 +407,7 @@ LWLockAcquire(LWLock *lock, LWLockMode mode) drive_terminal_interlock_tag->active_slot_generation = 0; drive_terminal_interlock_tag->queue_state_sequence++; (void)pg_atomic_fetch_and_u32(&drive_terminal_interlock_tag->queued_node_bitmap, - ~node_bit); + ~node_bit); drive_terminal_interlock_ticket->next_index = PCM_X_INVALID_SLOT_INDEX; drive_terminal_interlock_ticket->prev_index = PCM_X_INVALID_SLOT_INDEX; drive_terminal_interlock_ticket->reliable.retry_deadline_ms = 0; @@ -3949,10 +3949,10 @@ UT_TEST(test_local_queue_invalidate_authority_is_exact_and_read_only) PCM_X_QUEUE_STALE); /* An in-flight nonempty set still needs its exact ACK/graph proof. */ blocker = make_blocker(holder_key.identity.node_id, holder_key.identity.procno, - holder_key.identity.request_id); - UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact( - &probe_ref, 1, master_session, &holder_snapshot, &holder, 1, - &blocker, 1, &blocker_snapshot), + holder_key.identity.request_id); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, &holder, 1, + &blocker, 1, &blocker_snapshot), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_local_queue_invalidate_authorize_exact( &probe_ref.identity.tag, probe_ref.identity.cluster_epoch, @@ -6564,34 +6564,33 @@ UT_TEST(test_master_invalidate_busy_backs_off_exact_revoke_ticket) admit_active_probe(814, 0, 24, UINT64_C(8484), requester_session, 1, &admission); arm_blocker_probe(&admission.ref, 2, source_session); blocker_commit = make_blocker_header(&admission.ref, 1, NULL, 0); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 2, - source_session), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact( - &blocker_commit, 2, source_session, NULL, 0, &blocker_snapshot), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_graph_commit_exact( - &blocker_snapshot, NULL, 0, graph_generation), + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 2, source_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 2, - source_session), + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 2, source_session, + NULL, 0, &blocker_snapshot), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&admission.ref, graph_generation, - &transfer), + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_graph_commit_exact(&blocker_snapshot, NULL, 0, + graph_generation), PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, graph_generation, &transfer), + PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(&transfer, 2, source_session, &revoke), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact( - &transfer.identity.tag, transfer.identity.cluster_epoch, &armed), + UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact(&transfer.identity.tag, + transfer.identity.cluster_epoch, &armed), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_drive_bitmap_replace_exact( - &armed, source_bit | busy_bit, source_bit, &waiting), + UT_ASSERT_EQ(cluster_pcm_x_master_drive_bitmap_replace_exact(&armed, source_bit | busy_bit, + source_bit, &waiting), PCM_X_QUEUE_OK); ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; before = *ticket; - UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact( - &waiting, 3, UINT64_C(1000), UINT64_C(100), &backed_off), + UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact(&waiting, 3, UINT64_C(1000), + UINT64_C(100), &backed_off), PCM_X_QUEUE_OK); UT_ASSERT_EQ(backed_off.retry_deadline_ms, UINT64_C(1100)); UT_ASSERT_EQ(backed_off.pending_s_holders_bitmap, source_bit | busy_bit); @@ -6604,26 +6603,26 @@ UT_TEST(test_master_invalidate_busy_backs_off_exact_revoke_ticket) /* The pre-BUSY snapshot is consumed exactly. Replayed BUSY during the * same live window is an idempotent scheduling no-op. */ memset(&replay, 0xA5, sizeof(replay)); - UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact( - &waiting, 3, UINT64_C(1001), UINT64_C(100), &replay), + UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact(&waiting, 3, UINT64_C(1001), + UINT64_C(100), &replay), PCM_X_QUEUE_STALE); UT_ASSERT(memcmp(&replay, &(PcmXMasterDriveSnapshot){ 0 }, sizeof(replay)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact( - &backed_off, 3, UINT64_C(1099), UINT64_C(100), &replay), + UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact(&backed_off, 3, UINT64_C(1099), + UINT64_C(100), &replay), PCM_X_QUEUE_DUPLICATE); UT_ASSERT_EQ(replay.retry_deadline_ms, UINT64_C(1100)); UT_ASSERT_EQ(ticket->reliable.retry_deadline_ms, UINT64_C(1100)); /* The source already supplied the retained image and is ACKed; a BUSY * attributed to it cannot pause invalidation of another holder. */ - UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact( - &backed_off, 2, UINT64_C(1100), UINT64_C(100), &replay), + UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact(&backed_off, 2, UINT64_C(1100), + UINT64_C(100), &replay), PCM_X_QUEUE_INVALID); UT_ASSERT_EQ(ticket->reliable.retry_deadline_ms, UINT64_C(1100)); /* Expiry permits one new bounded window, still on the same identity. */ - UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact( - &backed_off, 3, UINT64_C(1100), UINT64_C(100), &replay), + UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact(&backed_off, 3, UINT64_C(1100), + UINT64_C(100), &replay), PCM_X_QUEUE_OK); UT_ASSERT_EQ(replay.retry_deadline_ms, UINT64_C(1200)); UT_ASSERT(ticket_refs_equal(&replay.ref, &waiting.ref)); @@ -6656,8 +6655,8 @@ UT_TEST(test_master_commit_retry_loses_race_to_terminal_progress_without_fail_cl for (mode = 0; mode < 2; mode++) { init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - admit_active_probe_with_base(816 + mode, 0, 22 + mode, UINT64_C(8383), - requester_session, 1, UINT64_C(5), &admission); + admit_active_probe_with_base(816 + mode, 0, 22 + mode, UINT64_C(8383), requester_session, 1, + UINT64_C(5), &admission); drive_master_ticket_to_prepared(&admission, source_session, UINT64_C(9383), &transfer, &image_id); tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; @@ -6668,8 +6667,8 @@ UT_TEST(test_master_commit_retry_loses_race_to_terminal_progress_without_fail_cl install_ready.image_id = image_id; install_ready.result = PCM_X_QUEUE_OK; install_ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; - UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact( - &install_ready, 0, 0, requester_session, &first_commit), + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, 0, 0, + requester_session, &first_commit), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact( &transfer.identity.tag, transfer.identity.cluster_epoch, &armed), @@ -6684,8 +6683,8 @@ UT_TEST(test_master_commit_retry_loses_race_to_terminal_progress_without_fail_cl drive_terminal_interlock_armed = true; memset(&retry_commit, 0xA5, sizeof(retry_commit)); memset(&retried, 0xA5, sizeof(retried)); - UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_exact( - &armed, UINT64_C(1000), UINT64_C(1100), &retry_commit, &retried), + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_exact(&armed, UINT64_C(1000), UINT64_C(1100), + &retry_commit, &retried), mode == 0 ? PCM_X_QUEUE_NOT_READY : PCM_X_QUEUE_STALE); UT_ASSERT(!drive_terminal_interlock_armed); UT_ASSERT(memcmp(&retry_commit, &(PcmXPhasePayload){ 0 }, sizeof(retry_commit)) == 0); @@ -8096,8 +8095,8 @@ UT_TEST(test_master_debug_iterators_report_live_slots) init_active_pcm_x(UINT64_C(78)); local_identity = make_wait_identity(717, 0, 26, UINT64_C(21002)); bind_local_master(1, local_identity.cluster_epoch, UINT64_C(4501)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&local_identity, 1, UINT64_C(4501), - &local_leader), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&local_identity, 1, UINT64_C(4501), &local_leader), + PCM_X_QUEUE_OK); cursor = 0; UT_ASSERT(cluster_pcm_x_local_tag_debug_next(&cursor, &index, line, sizeof(line))); UT_ASSERT(strstr(line, "blocker_gen=") != NULL); @@ -10260,7 +10259,7 @@ UT_TEST(test_local_tag_only_holder_transfer_persists_until_exact_drain) /* Complete generation 1 normally. The separate gate-contention test below * covers an ACK frame that is delivered but cannot apply before REVOKE. */ UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, - 1, master_session), + 1, master_session), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, 1, master_session), @@ -10272,11 +10271,11 @@ UT_TEST(test_local_tag_only_holder_transfer_persists_until_exact_drain) * empty replay instead of waiting forever for an ACK the master no longer * needs to send. */ UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, - NULL, 0, &holder_snapshot), + NULL, 0, &holder_snapshot), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, - &holder_snapshot, NULL, 0, NULL, 0, - &blocker_snapshot), + &holder_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), PCM_X_QUEUE_OK); UT_ASSERT_EQ(blocker_snapshot.set_generation, 2); /* The exact blocker ACK tombstone is durable evidence, but it cannot @@ -10309,8 +10308,7 @@ UT_TEST(test_local_tag_only_holder_transfer_persists_until_exact_drain) tag_slot = &local_tag_slots(header)[holder_snapshot.tag_slot.slot_index]; state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); pg_atomic_write_u32(&tag_slot->slot.state_flags, - state_flags - | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); + state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); ready_refusal = PCM_X_LOCAL_IMAGE_READY_REFUSAL_NONE; UT_ASSERT_EQ(cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( &ready, 1, master_session, &replay, &ready_refusal), @@ -10325,10 +10323,9 @@ UT_TEST(test_local_tag_only_holder_transfer_persists_until_exact_drain) UT_ASSERT_EQ(ready_refusal, PCM_X_LOCAL_IMAGE_READY_REFUSAL_ACTIVE_HOLDER); tag_slot->active_holder_head_index = PCM_X_INVALID_SLOT_INDEX; ready_refusal = PCM_X_LOCAL_IMAGE_READY_REFUSAL_ACTIVE_HOLDER; - UT_ASSERT_EQ( - cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( - &ready, 1, master_session, &replay, &ready_refusal), - PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( + &ready, 1, master_session, &replay, &ready_refusal), + PCM_X_QUEUE_OK); UT_ASSERT_EQ(ready_refusal, PCM_X_LOCAL_IMAGE_READY_REFUSAL_NONE); UT_ASSERT(memcmp(&ready, &replay, sizeof(ready)) == 0); memset(&replay, 0, sizeof(replay)); @@ -10454,7 +10451,7 @@ UT_TEST(test_local_non_source_blocker_participant_drains_and_retires_exactly) PCM_X_QUEUE_OK); UT_ASSERT_EQ(holder_snapshot.holder_count, 2); blocker = make_blocker(holder_keys[0].identity.node_id, holder_keys[0].identity.procno, - holder_keys[0].identity.request_id); + holder_keys[0].identity.request_id); UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, &holder_snapshot, holder_copies, 2, &blocker, 1, &blocker_snapshot), @@ -10486,9 +10483,9 @@ UT_TEST(test_local_non_source_blocker_participant_drains_and_retires_exactly) &probe_ref, 1, master_session, holder_copies, 2, &holder_snapshot), PCM_X_QUEUE_OK); UT_ASSERT_EQ(holder_snapshot.holder_count, 0); - UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact( - &probe_ref, 1, master_session, &holder_snapshot, NULL, 0, NULL, 0, - &blocker_snapshot), + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), PCM_X_QUEUE_OK); UT_ASSERT_EQ(blocker_snapshot.set_generation, 2); UT_ASSERT_EQ(tag_slot->blocker_snapshot_count, 0); @@ -13141,14 +13138,14 @@ UT_TEST(test_local_blocker_snapshot_replays_exact_holder_set_and_gates_revoke) cluster_pcm_x_local_blocker_snapshot_copy_exact(&blocker_snapshot, &header, &edge, 1), PCM_X_QUEUE_NOT_READY); /* A later nonempty generation never qualifies for the replay exception. */ - UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact( - &probe_ref, 1, master_session, &holder_copy, 1, &holder_snapshot), + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + &holder_copy, 1, &holder_snapshot), PCM_X_QUEUE_OK); blocker = make_blocker(holder_key.identity.node_id, holder_key.identity.procno, - holder_key.identity.request_id); - UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact( - &probe_ref, 1, master_session, &holder_snapshot, &holder_copy, 1, &blocker, 1, - &blocker_snapshot), + holder_key.identity.request_id); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, &holder_copy, 1, + &blocker, 1, &blocker_snapshot), PCM_X_QUEUE_OK); UT_ASSERT_EQ(blocker_snapshot.set_generation, 2); UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), @@ -13178,16 +13175,16 @@ UT_TEST(test_local_first_blocker_ack_busy_then_exact_revoke_consumes_empty_commi UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); bind_local_master(1, holder_key.identity.cluster_epoch, master_session); UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&holder_key.identity.tag, &holder_copy, 1, - &holder_snapshot), + &holder_snapshot), PCM_X_QUEUE_OK); memset(&probe_ref, 0, sizeof(probe_ref)); probe_ref.identity = make_wait_identity(7091, 2, 2, UINT64_C(70312)); probe_ref.handle.ticket_id = UINT64_C(90111); probe_ref.handle.queue_generation = UINT64_C(11); - UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact( - &probe_ref, 1, master_session, &holder_snapshot, &holder_copy, 1, NULL, 0, - &blocker_snapshot), + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, &holder_copy, 1, + NULL, 0, &blocker_snapshot), PCM_X_QUEUE_OK); UT_ASSERT_EQ(blocker_snapshot.set_generation, 1); UT_ASSERT_EQ(blocker_snapshot.blocker_count, 0); @@ -13199,9 +13196,8 @@ UT_TEST(test_local_first_blocker_ack_busy_then_exact_revoke_consumes_empty_commi state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); pg_atomic_write_u32(&tag_slot->slot.state_flags, state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); - UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, - blocker_snapshot.set_generation, 1, - master_session), + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), PCM_X_QUEUE_BUSY); pg_atomic_write_u32(&tag_slot->slot.state_flags, state_flags); @@ -16150,8 +16146,8 @@ UT_TEST(test_requester_wait_preserves_late_revoke_barrier_refusal) /* A type-49 barrier that freezes after initial admission but before the * next WaitLatch must escape as BARRIER_CLOSED. Sleeping here would keep * the source holder registered and deadlock IMAGE_READY forever. */ - UT_ASSERT_EQ(cluster_pcm_x_requester_wait_once_result( - requester_wait_revalidate_result, requester_wait_block, &capture), + UT_ASSERT_EQ(cluster_pcm_x_requester_wait_once_result(requester_wait_revalidate_result, + requester_wait_block, &capture), PCM_X_QUEUE_BARRIER_CLOSED); UT_ASSERT_EQ(capture.revalidate_calls, 1); UT_ASSERT_EQ(capture.wait_calls, 0); @@ -16160,8 +16156,8 @@ UT_TEST(test_requester_wait_preserves_late_revoke_barrier_refusal) /* An exact OK proof still linearizes the count before the physical wait. */ capture.revalidate_ok = true; - UT_ASSERT_EQ(cluster_pcm_x_requester_wait_once_result( - requester_wait_revalidate_result, requester_wait_block, &capture), + UT_ASSERT_EQ(cluster_pcm_x_requester_wait_once_result(requester_wait_revalidate_result, + requester_wait_block, &capture), PCM_X_QUEUE_OK); UT_ASSERT_EQ(capture.revalidate_calls, 2); UT_ASSERT_EQ(capture.wait_calls, 1); diff --git a/src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c b/src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c index 79ca409fc9..b1fa6341f1 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c @@ -168,22 +168,22 @@ UT_TEST(test_fetch_holder_authenticates_requester_master_and_record_generation) env.dest_node_id = 3; env.payload_length = sizeof(request); - UT_ASSERT(cluster_pcm_x_image_fetch_request_exact_diagnosed( - &env, &request, &holder, 3, 2, 0, &refusal)); + UT_ASSERT(cluster_pcm_x_image_fetch_request_exact_diagnosed(&env, &request, &holder, 3, 2, 0, + &refusal)); UT_ASSERT_EQ((int)refusal, (int)PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_NONE); env.source_node_id = 0; - UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact_diagnosed( - &env, &request, &holder, 3, 2, 0, &refusal)); + UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact_diagnosed(&env, &request, &holder, 3, 2, 0, + &refusal)); UT_ASSERT_EQ((int)refusal, (int)PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_ENVELOPE); env.source_node_id = 1; holder.image.image_id++; - UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact_diagnosed( - &env, &request, &holder, 3, 2, 0, &refusal)); + UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact_diagnosed(&env, &request, &holder, 3, 2, 0, + &refusal)); UT_ASSERT_EQ((int)refusal, (int)PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_IMAGE); holder.image = requester.image; holder.master_node = 0; - UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact_diagnosed( - &env, &request, &holder, 3, 2, 0, &refusal)); + UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact_diagnosed(&env, &request, &holder, 3, 2, 0, + &refusal)); UT_ASSERT_EQ((int)refusal, (int)PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_MASTER); } diff --git a/src/test/cluster_unit/test_cluster_sf_dep.c b/src/test/cluster_unit/test_cluster_sf_dep.c index 3d39718038..2a7bf2c0e2 100644 --- a/src/test/cluster_unit/test_cluster_sf_dep.c +++ b/src/test/cluster_unit/test_cluster_sf_dep.c @@ -317,8 +317,8 @@ UT_TEST(test_pcm_x_capability_family_sample_is_record_coherent) UT_TEST(test_pcm_x_source_floor_capability_guard_is_generation_exact) { ClusterSfPeerCap cap; - const uint32 family = PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 - | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1; + const uint32 family + = PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1; memset(&cap, 0, sizeof(cap)); cluster_sf_peer_cap_note(&cap, family, 41); diff --git a/src/test/cluster_unit/test_cluster_visibility_fork.c b/src/test/cluster_unit/test_cluster_visibility_fork.c index 8e70f79054..a39c7f5399 100644 --- a/src/test/cluster_unit/test_cluster_visibility_fork.c +++ b/src/test/cluster_unit/test_cluster_visibility_fork.c @@ -127,22 +127,21 @@ read_source(const char *path) length = ftell(fp); UT_ASSERT(length > 0); UT_ASSERT_EQ(fseek(fp, 0, SEEK_SET), 0); - source = malloc((size_t) length + 1); + source = malloc((size_t)length + 1); UT_ASSERT(source != NULL); - if (source == NULL) - { + if (source == NULL) { fclose(fp); return NULL; } - UT_ASSERT_EQ((long) fread(source, 1, (size_t) length, fp), length); + UT_ASSERT_EQ((long)fread(source, 1, (size_t)length, fp), length); source[length] = '\0'; fclose(fp); return source; } static void -assert_data_active_publish(const char *source, const char *start_marker, - const char *end_marker, const char *uba_name) +assert_data_active_publish(const char *source, const char *start_marker, const char *end_marker, + const char *uba_name) { const char *start = strstr(source, start_marker); const char *end = start != NULL ? strstr(start + strlen(start_marker), end_marker) : NULL; @@ -150,8 +149,8 @@ assert_data_active_publish(const char *source, const char *start_marker, const char *publish; const char *crit_end = start != NULL ? strstr(start, "END_CRIT_SECTION();") : NULL; - snprintf(publish_call, sizeof(publish_call), - "cluster_tt_local_record_data_active(xid, %s);", uba_name); + snprintf(publish_call, sizeof(publish_call), "cluster_tt_local_record_data_active(xid, %s);", + uba_name); publish = start != NULL ? strstr(start, publish_call) : NULL; if (publish != NULL && end != NULL && publish >= end) @@ -166,8 +165,7 @@ assert_data_active_publish(const char *source, const char *start_marker, /* The ACTIVE identity is published only after the tuple + ITL stamp is * WAL-protected, and before the function can release its heap buffer. */ - while (true) - { + while (true) { const char *next = strstr(crit_end + 1, "END_CRIT_SECTION();"); if (next == NULL || next >= publish) @@ -327,20 +325,19 @@ UT_TEST(test_p033_data_dml_publishes_active_identity) char *heap_source = read_source(HEAPAM_SOURCE_PATH); char *tt_source = read_source(TT_LOCAL_SOURCE_PATH); - if (heap_source == NULL || tt_source == NULL) - { + if (heap_source == NULL || tt_source == NULL) { free(heap_source); free(tt_source); return; } assert_data_active_publish(heap_source, "\nheap_insert(Relation", - "\nheap_prepare_insert(Relation", "cluster_itl_uba"); + "\nheap_prepare_insert(Relation", "cluster_itl_uba"); assert_data_active_publish(heap_source, "\nheap_multi_insert(Relation", - "\nsimple_heap_insert(Relation", "cluster_mi_uba"); + "\nsimple_heap_insert(Relation", "cluster_mi_uba"); assert_data_active_publish(heap_source, "\nheap_delete(Relation", - "\nsimple_heap_delete(Relation", "cluster_itl_uba"); + "\nsimple_heap_delete(Relation", "cluster_itl_uba"); assert_data_active_publish(heap_source, "\nheap_update(Relation", - "\nsimple_heap_update(Relation", "cluster_itl_uba"); + "\nsimple_heap_update(Relation", "cluster_itl_uba"); /* A real undo-record UBA may live in a different record segment from the * transaction's canonical TT segment after rollover. The producer must @@ -361,23 +358,22 @@ UT_TEST(test_p033_data_dml_publishes_active_identity) * terminal/FROZEN/stale boundaries are widened by the producer fix. */ UT_TEST(test_p033_active_and_safety_boundary_matrix) { - UT_ASSERT_EQ((int) cluster_vis_evidence_route(CLUSTER_VIS_EVIDENCE_REMOTE, false), - (int) CLUSTER_VIS_ROUTE_REMOTE_VERDICT); - UT_ASSERT_EQ((int) cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_IN_PROGRESS), - (int) CVV_INVISIBLE); - UT_ASSERT_EQ((int) cluster_vis_update_xmax_verdict(CLUSTER_TT_STATUS_IN_PROGRESS, false), - (int) CVV_BEING_MODIFIED); - - UT_ASSERT_EQ((int) cluster_vis_xmin_needs_resolution(HEAP_XMIN_FROZEN), 0); - UT_ASSERT_EQ((int) cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_COMMITTED), - (int) CVV_VISIBLE); - UT_ASSERT_EQ((int) cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_ABORTED), - (int) CVV_INVISIBLE); - UT_ASSERT_EQ((int) cluster_vis_evidence_route( - CLUSTER_VIS_EVIDENCE_STALE_OR_AMBIGUOUS, false), - (int) CLUSTER_VIS_ROUTE_FAILCLOSED_UNKNOWN); - UT_ASSERT_EQ((int) cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_UNKNOWN), - (int) CVV_FAILCLOSED_UNKNOWN); + UT_ASSERT_EQ((int)cluster_vis_evidence_route(CLUSTER_VIS_EVIDENCE_REMOTE, false), + (int)CLUSTER_VIS_ROUTE_REMOTE_VERDICT); + UT_ASSERT_EQ((int)cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_IN_PROGRESS), + (int)CVV_INVISIBLE); + UT_ASSERT_EQ((int)cluster_vis_update_xmax_verdict(CLUSTER_TT_STATUS_IN_PROGRESS, false), + (int)CVV_BEING_MODIFIED); + + UT_ASSERT_EQ((int)cluster_vis_xmin_needs_resolution(HEAP_XMIN_FROZEN), 0); + UT_ASSERT_EQ((int)cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_COMMITTED), + (int)CVV_VISIBLE); + UT_ASSERT_EQ((int)cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_ABORTED), + (int)CVV_INVISIBLE); + UT_ASSERT_EQ((int)cluster_vis_evidence_route(CLUSTER_VIS_EVIDENCE_STALE_OR_AMBIGUOUS, false), + (int)CLUSTER_VIS_ROUTE_FAILCLOSED_UNKNOWN); + UT_ASSERT_EQ((int)cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_UNKNOWN), + (int)CVV_FAILCLOSED_UNKNOWN); }