diff --git a/scripts/ci/run-cppcheck.sh b/scripts/ci/run-cppcheck.sh index 379c5a4efa..e20c5128b3 100755 --- a/scripts/ci/run-cppcheck.sh +++ b/scripts/ci/run-cppcheck.sh @@ -240,6 +240,8 @@ cppcheck \ --xml --xml-version=2 \ -I src/include \ -DUSE_PGRAC_CLUSTER=1 \ + '-DHEAPAM_SOURCE_PATH="src/backend/access/heap/heapam.c"' \ + '-DTT_LOCAL_SOURCE_PATH="src/backend/cluster/cluster_tt_local.c"' \ --quiet \ "${CLUSTER_DIRS[@]}" \ 2> cppcheck.xml diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 9a6ffef55a..20572529c6 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -1719,12 +1719,12 @@ heap_hot_search_buffer(ItemPointer tid, Relation relation, Buffer buffer, if ( #ifdef USE_PGRAC_CLUSTER - /* spec-3.14 D5b: HeapTupleIsSurelyDead has no Buffer and - * bypasses VacuumHorizon; guard the HOT all_dead probe here - * so a remote-evidence chain is never declared all-dead. */ - (cluster_storage_mode_enabled() - && cluster_tuple_has_remote_evidence(buffer, heapTuple->t_data)) - ? true : /* remote evidence -> not surely dead -> *all_dead=false */ + /* P0-28: no cluster-wide prune horizon; even a recycled ITL + * slot cannot turn a shared-storage HOT member surely dead. */ + cluster_vis_prune_must_defer(cluster_storage_mode_enabled() + && !RelationUsesLocalBuffers(relation), + false) + ? true : #endif !HeapTupleIsSurelyDead(heapTuple, vistest)) *all_dead = false; @@ -2323,6 +2323,11 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, { ClusterItlTouchHandle handle; + /* P0-33: publish the exact data-writer binding as IN_PROGRESS only + * after the tuple + ACTIVE ITL stamp is WAL-protected, and before the + * content lock is released. */ + cluster_tt_local_record_data_active(xid, cluster_itl_uba); + handle.rloc = relation->rd_locator; handle.block = ItemPointerGetBlockNumber(&heaptup->t_self); handle.forknum = MAIN_FORKNUM; @@ -2851,6 +2856,10 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, { ClusterItlTouchHandle handle; + /* P0-33: pair this page's fresh ACTIVE data ref with its exact + * IN_PROGRESS overlay/hint before releasing the content lock. */ + cluster_tt_local_record_data_active(xid, cluster_mi_uba); + handle.rloc = relation->rd_locator; handle.block = BufferGetBlockNumber(buffer); handle.forknum = MAIN_FORKNUM; @@ -3438,6 +3447,75 @@ cluster_xwait_has_remote_evidence(Page page, HeapTupleHeader tuple, TransactionI } #endif /* USE_PGRAC_CLUSTER */ +/* + * PGRAC (t/400 L3 item 3): bound for the BARRIER_CLOSED caller-owned unwind + * at the pre-crit VM lock sites. The all-visible clear race is one-shot per + * page (whoever clears first removes the need for every later contender), so + * the retry converges; the bound only guards the no-node-cache configuration + * where the warmed conversion can be revoked before the re-attempt. Once + * exhausted, the plain LockBuffer restores the historical fail-closed ERROR. + */ +#define CLUSTER_HEAP_VM_BARRIER_MAX_RETRIES 16 + +/* + * PGRAC (t/400 L3 item 3): resolve a refused map-page conversion while the + * caller holds NO content lock. Blocking is legal here (the nested guard + * only refuses a wait entered while a frozen-tag content lock is held); + * hold-until-revoked retains the node X after the local release, so the + * caller's re-attempt is normally served by the cached-cover fast path + * without consulting the guard again. + */ +static void +cluster_heap_vm_barrier_warm(Buffer vmbuf) +{ + LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE); + LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK); +} + +/* + * Acquire heap content X without carrying a visibility-map pin across the + * cluster PCM wait. VM/FSM pages cannot use the passive-pin retained-PI + * shape because their callers may read pinned bytes without a content lock. + * Keeping this pin while another node transfers the VM page therefore makes + * the source's exact-drop finish return BUSY forever. + * + * visibilitymap_pin() runs before heap content authority and may do I/O. We + * then release that pin before LockBuffer can wait. Once heap X is held, + * visibilitymap_pin_recent() may only pin the exact still-resident descriptor; + * it cannot do I/O. A retag/drop race releases heap X and repeats the safe + * pre-read sequence. + */ +static void +cluster_heap_lock_with_vm_repin(Relation relation, BlockNumber heap_block, + Buffer heap_buffer, Buffer *vmbuffer) +{ + Buffer recent_vm = InvalidBuffer; + Page heap_page; + + Assert(relation != NULL); + Assert(BufferIsValid(heap_buffer)); + Assert(vmbuffer != NULL); + heap_page = BufferGetPage(heap_buffer); + + for (;;) + { + if (PageIsAllVisible(heap_page) && !BufferIsValid(*vmbuffer)) + visibilitymap_pin(relation, heap_block, vmbuffer); + if (BufferIsValid(*vmbuffer)) + { + recent_vm = *vmbuffer; + ReleaseBuffer(*vmbuffer); + *vmbuffer = InvalidBuffer; + } + + LockBuffer(heap_buffer, BUFFER_LOCK_EXCLUSIVE); + if (!PageIsAllVisible(heap_page) + || visibilitymap_pin_recent(relation, heap_block, recent_vm, vmbuffer)) + return; + LockBuffer(heap_buffer, BUFFER_LOCK_UNLOCK); + } +} + TM_Result heap_delete(Relation relation, ItemPointer tid, @@ -3453,6 +3531,8 @@ heap_delete(Relation relation, ItemPointer tid, Buffer buffer; Buffer vmbuffer = InvalidBuffer; bool vm_locked; /* PGRAC: pre-crit VM content lock held */ + int vm_barrier_retries = 0; /* PGRAC: t/400 L3 item 3 */ + CommandId pgrac_entry_cid = cid; /* PGRAC: restored on barrier requalify */ TransactionId new_xmax; uint16 new_infomask, new_infomask2; @@ -3855,7 +3935,36 @@ heap_delete(Relation relation, ItemPointer tid, vm_locked = false; if (PageIsAllVisible(page) && BufferIsValid(vmbuffer)) { - LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); + if (vm_barrier_retries >= CLUSTER_HEAP_VM_BARRIER_MAX_RETRIES) + LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); + else if (!ClusterLockBufferExclusiveBarrierAware(vmbuffer)) + { + /* + * PGRAC: vm barrier unwind (delete requalify) — this backend + * holds the heap page whose tag sits under a frozen revoke + * barrier, so it may not sleep on the map-page conversion. + * Release our own lock, resolve the conversion unlocked, and + * requalify from scratch: the ITL slot pick is idempotent and an + * unstamped undo record is unreachable garbage, exactly as after + * any pre-crit ERROR. The entry cid is restored (no + * combo-of-combo) and the abandoned replica-identity copy freed. + */ + vm_barrier_retries++; + cid = pgrac_entry_cid; + iscombo = false; + if (old_key_tuple != NULL && old_key_copied) + { + heap_freetuple(old_key_tuple); + old_key_tuple = NULL; + old_key_copied = false; + } + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + cluster_heap_vm_barrier_warm(vmbuffer); + ReleaseBuffer(vmbuffer); + vmbuffer = InvalidBuffer; + cluster_heap_lock_with_vm_repin(relation, block, buffer, &vmbuffer); + goto l1; + } vm_locked = true; } @@ -4012,6 +4121,10 @@ heap_delete(Relation relation, ItemPointer tid, { ClusterItlTouchHandle handle; + /* P0-33: publish the exact ACTIVE data-writer identity while the + * freshly stamped page is still protected by its content lock. */ + cluster_tt_local_record_data_active(xid, cluster_itl_uba); + handle.rloc = relation->rd_locator; handle.block = BufferGetBlockNumber(buffer); handle.forknum = MAIN_FORKNUM; @@ -4147,6 +4260,9 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, vmbuffer_new = InvalidBuffer; bool vm_locked; /* PGRAC: pre-crit VM content lock held */ bool vm_locked_new; + bool old_tuple_temp_locked = false; /* PGRAC: t/400 L3 item 3 */ + int vm_barrier_retries = 0; /* PGRAC: t/400 L3 item 3 */ + CommandId pgrac_entry_cid = cid; /* PGRAC: restored on barrier requalify */ bool need_toast; Size newtupsize, pagefree; @@ -4244,10 +4360,14 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, * in the middle of changing this, so we'll need to recheck after we have * the lock. */ +#ifdef USE_PGRAC_CLUSTER + cluster_heap_lock_with_vm_repin(relation, block, buffer, &vmbuffer); +#else if (PageIsAllVisible(page)) visibilitymap_pin(relation, block, &vmbuffer); LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); +#endif lp = PageGetItemId(page, ItemPointerGetOffsetNumber(otid)); @@ -4675,8 +4795,7 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, if (vmbuffer == InvalidBuffer && PageIsAllVisible(page)) { LockBuffer(buffer, BUFFER_LOCK_UNLOCK); - visibilitymap_pin(relation, block, &vmbuffer); - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + cluster_heap_lock_with_vm_repin(relation, block, buffer, &vmbuffer); goto l2; } @@ -4816,7 +4935,28 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, vm_locked = false; if (PageIsAllVisible(page) && BufferIsValid(vmbuffer)) { - LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); + if (vm_barrier_retries >= CLUSTER_HEAP_VM_BARRIER_MAX_RETRIES) + LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); + else if (!ClusterLockBufferExclusiveBarrierAware(vmbuffer)) + { + /* + * PGRAC: vm barrier unwind (update pre-toast) — the old + * tuple carries no temporary lock yet and nothing + * irreversible has happened in this pass, so a full + * requalify is required and sufficient. Restore the entry + * cid: a pass-1 combo cmax must not feed requalification or + * be recombined into a combo-of-combo. + */ + vm_barrier_retries++; + cid = pgrac_entry_cid; + iscombo = false; + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + cluster_heap_vm_barrier_warm(vmbuffer); + ReleaseBuffer(vmbuffer); + vmbuffer = InvalidBuffer; + cluster_heap_lock_with_vm_repin(relation, block, buffer, &vmbuffer); + goto l2; + } vm_locked = true; } @@ -4870,6 +5010,11 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, END_CRIT_SECTION(); + /* PGRAC (t/400 L3 item 3): the lock-only xmax now shields the old + * tuple from concurrent updaters, so later unlocked windows (the + * page-reacquire loop below) do not require requalification. */ + old_tuple_temp_locked = true; + if (vm_locked) LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK); @@ -4914,6 +5059,16 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, * pins; but if we don't, we have to handle that here. Hence we need * a loop. */ + /* + * PGRAC (t/400 L3 item 3): the VM-barrier unwind re-enters here with + * both content locks released (and a foreign newbuf unpinned) after + * resolving the refused map-page conversion. The lock-only xmax + * stamped above makes requalification unnecessary, exactly as for + * the stock unlocked TOAST window; everything below — page choice, + * serializable-conflict check, HOT decision, replica identity, ITL + * pick and undo emit — is re-derived on the way back down. + */ +l_pgrac_reacquire: for (;;) { if (newtupsize > pagefree) @@ -4981,6 +5136,13 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, * one pin is held. */ + /* + * PGRAC (t/400 L3 item 3 review): pass-local decision — the VM-barrier + * reacquire can flip same-page <-> cross-page between passes, and a + * stale HOT verdict on a cross-page retry would skip index entries. + */ + use_hot_update = false; + summarized_update = false; if (newbuf == buffer) { /* @@ -5028,6 +5190,17 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, * EXCLUSIVE content locks already held; OVERFLOW raises ERROR * before the critical section. */ + /* + * PGRAC (t/400 L3 item 3 review): the VM-barrier unwind can abandon a + * pass and re-enter this block with a DIFFERENT page choice (a cross-page + * pass may retry same-page once the old page regains space). Every pick + * below is pass-local: reset both pairs so a stale cross-page slot index + * can never be stamped onto a page it was not chosen on. + */ + cluster_itl_old_slot = CLUSTER_ITL_SLOT_UNALLOCATED; + cluster_itl_new_slot = CLUSTER_ITL_SLOT_UNALLOCATED; + cluster_itl_old_active = false; + cluster_itl_new_active = false; if (cluster_itl_write_path_enabled(relation) && PageHasItl(BufferGetPage(buffer))) { /* spec-3.4b D5: single xact-local TT binding shared by both stamps (F11). */ @@ -5140,6 +5313,8 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, bool need_new = newbuf != buffer && PageIsAllVisible(BufferGetPage(newbuf)) && BufferIsValid(vmbuffer_new); + bool barrier_aware = vm_barrier_retries < CLUSTER_HEAP_VM_BARRIER_MAX_RETRIES; + Buffer refused_vm = InvalidBuffer; if (need_old && need_new && vmbuffer_new == vmbuffer) need_new = false; /* same map page: one lock covers both */ @@ -5147,23 +5322,120 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, if (need_old && need_new && vmbuffer_new < vmbuffer) { - LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE); - vm_locked_new = true; - LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); - vm_locked = true; + if (!barrier_aware) + { + LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE); + vm_locked_new = true; + LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); + vm_locked = true; + } + else + { + if (ClusterLockBufferExclusiveBarrierAware(vmbuffer_new)) + vm_locked_new = true; + else + refused_vm = vmbuffer_new; + if (refused_vm == InvalidBuffer) + { + if (ClusterLockBufferExclusiveBarrierAware(vmbuffer)) + vm_locked = true; + else + refused_vm = vmbuffer; + } + } } else { if (need_old) { - LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); - vm_locked = true; + if (!barrier_aware) + { + LockBuffer(vmbuffer, BUFFER_LOCK_EXCLUSIVE); + vm_locked = true; + } + else if (ClusterLockBufferExclusiveBarrierAware(vmbuffer)) + vm_locked = true; + else + refused_vm = vmbuffer; } - if (need_new) + if (need_new && refused_vm == InvalidBuffer) { - LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE); - vm_locked_new = true; + if (!barrier_aware) + { + LockBuffer(vmbuffer_new, BUFFER_LOCK_EXCLUSIVE); + vm_locked_new = true; + } + else if (ClusterLockBufferExclusiveBarrierAware(vmbuffer_new)) + vm_locked_new = true; + else + refused_vm = vmbuffer_new; + } + } + + if (refused_vm != InvalidBuffer) + { + /* + * PGRAC: BARRIER_CLOSED caller-owned unwind. This backend holds + * heap content lock(s) whose tag sits under a frozen revoke + * barrier; sleeping on the map-page conversion here could + * deadlock across locks, and bufmgr must never release the + * outer lock itself. Release our OWN locks, resolve the + * conversion while holding no content lock, then re-enter a + * proven point. + */ + vm_barrier_retries++; + if (vm_locked_new) + { + LockBuffer(vmbuffer_new, BUFFER_LOCK_UNLOCK); + vm_locked_new = false; + } + if (vm_locked) + { + LockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK); + vm_locked = false; + } + if (old_key_tuple != NULL && old_key_copied) + { + heap_freetuple(old_key_tuple); + old_key_tuple = NULL; + old_key_copied = false; } + if (!old_tuple_temp_locked) + { + /* + * PGRAC: vm barrier unwind (update requalify) — no + * temporary lock protects the old tuple yet (same-page + * non-TOAST path, so newbuf == buffer), hence a concurrent + * updater may win the race while we are unlocked: requalify + * from scratch. ITL pick and undo emit of this pass are + * abandoned safely (idempotent pick, unreachable record). + */ + Assert(newbuf == buffer); + cid = pgrac_entry_cid; + iscombo = false; + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + cluster_heap_vm_barrier_warm(refused_vm); + ReleaseBuffer(vmbuffer); + vmbuffer = InvalidBuffer; + cluster_heap_lock_with_vm_repin(relation, block, buffer, &vmbuffer); + goto l2; + } + + /* + * PGRAC: vm barrier unwind (update reacquire) — the lock-only + * xmax shields the old tuple, so re-entering the page-reacquire + * loop (stock unlocked-TOAST idiom) is sufficient. A distinct + * newbuf is unlocked AND unpinned: the loop re-picks the target + * page through RelationGetBufferForTuple. + */ + if (newbuf != buffer) + { + LockBuffer(newbuf, BUFFER_LOCK_UNLOCK); + ReleaseBuffer(newbuf); + } + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + cluster_heap_vm_barrier_warm(refused_vm); + goto l_pgrac_reacquire; } } @@ -5300,7 +5572,29 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, if (vm_locked_new) LockBuffer(vmbuffer_new, BUFFER_LOCK_UNLOCK); + /* VM pages require a zero-pin exact DROP at remote source handoff. Do not + * carry these pins into the heap writer's queue release: that release may + * wait for DRAIN after a VM successor has already frozen this node as its + * source, creating a VM-drop -> pin -> heap-DRAIN cycle. Every VM byte use + * is complete once the VM content locks above are released. */ + if (BufferIsValid(vmbuffer_new)) + { + ReleaseBuffer(vmbuffer_new); + vmbuffer_new = InvalidBuffer; + } + if (BufferIsValid(vmbuffer)) + { + ReleaseBuffer(vmbuffer); + vmbuffer = InvalidBuffer; + } + #ifdef USE_PGRAC_CLUSTER + /* P0-33: old and new versions share one xact-local TT binding. Publish + * that exact binding once, after both possible ACTIVE stamps are WAL- + * protected and before either heap content lock is released. */ + if (cluster_itl_old_active || cluster_itl_new_active) + cluster_tt_local_record_data_active(xid, cluster_itl_uba); + /* * PGRAC (spec-3.4a D4): register touched ITL handle(s). Outside * CRIT so palloc is safe. Same-page UPDATE registers one handle @@ -5348,10 +5642,6 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, if (newbuf != buffer) ReleaseBuffer(newbuf); ReleaseBuffer(buffer); - if (BufferIsValid(vmbuffer_new)) - ReleaseBuffer(vmbuffer_new); - if (BufferIsValid(vmbuffer)) - ReleaseBuffer(vmbuffer); /* * Release the lmgr tuple lock, if we had it. diff --git a/src/backend/access/heap/heapam_visibility.c b/src/backend/access/heap/heapam_visibility.c index db14f79967..4520189f19 100644 --- a/src/backend/access/heap/heapam_visibility.c +++ b/src/backend/access/heap/heapam_visibility.c @@ -828,37 +828,42 @@ cluster_satisfies_update_fork(HeapTuple htup, Buffer buffer, TM_Result *res) * striping-off/below-floor collision with a remote inserter's xid; the * pre-round-6 early return handed that collision to the PG-native cmin * path -> "attempted to update invisible tuple". Only the no-remote- - * evidence route treats the raw match as our own insert. + * evidence route treats the raw match as our own insert. VACUUM freeze is + * stronger: it proves xmin committed even after that historical ITL slot is + * recycled, so only the exact FROZEN bit pair skips xmin resolution. */ - cluster_visibility_resolve_tuple(buffer, tuple, raw_xmin, CLUSTER_VIS_XMIN, &r); - switch (cluster_vis_evidence_route(r.evidence, TransactionIdIsCurrentTransactionId(raw_xmin))) { - case CLUSTER_VIS_ROUTE_FAILCLOSED_UNKNOWN: - ereport(ERROR, (errcode(ERRCODE_CLUSTER_TT_STATUS_UNKNOWN), - errmsg("cluster TT slot recycled for xmin %u", raw_xmin), - errhint("ITL slot no longer maps to this xid; retry."))); - break; - case CLUSTER_VIS_ROUTE_NATIVE_SELF: - /* Our own insert: PG-native handles cmin / self-modified correctly. */ - return false; - case CLUSTER_VIS_ROUTE_REMOTE_VERDICT: - switch (cluster_vis_update_xmin_verdict(r.status)) { - case CVV_INVISIBLE: - *res = TM_Invisible; - return true; - case CVV_FAILCLOSED_UNKNOWN: + if (cluster_vis_xmin_needs_resolution(tuple->t_infomask)) { + cluster_visibility_resolve_tuple(buffer, tuple, raw_xmin, CLUSTER_VIS_XMIN, &r); + switch ( + cluster_vis_evidence_route(r.evidence, TransactionIdIsCurrentTransactionId(raw_xmin))) { + case CLUSTER_VIS_ROUTE_FAILCLOSED_UNKNOWN: ereport(ERROR, (errcode(ERRCODE_CLUSTER_TT_STATUS_UNKNOWN), - errmsg("cluster TT status unknown for xmin %u", raw_xmin), - errhint("Remote commit_scn not yet propagated; retry or abort."))); + errmsg("cluster TT slot recycled for xmin %u", raw_xmin), + errhint("ITL slot no longer maps to this xid; retry."))); break; - case CVV_VISIBLE: - xmin_remote_visible = true; + case CLUSTER_VIS_ROUTE_NATIVE_SELF: + /* Our own insert: PG-native handles cmin / self-modified correctly. */ + return false; + case CLUSTER_VIS_ROUTE_REMOTE_VERDICT: + switch (cluster_vis_update_xmin_verdict(r.status)) { + case CVV_INVISIBLE: + *res = TM_Invisible; + return true; + case CVV_FAILCLOSED_UNKNOWN: + ereport(ERROR, (errcode(ERRCODE_CLUSTER_TT_STATUS_UNKNOWN), + errmsg("cluster TT status unknown for xmin %u", raw_xmin), + errhint("Remote commit_scn not yet propagated; retry or abort."))); + break; + case CVV_VISIBLE: + xmin_remote_visible = true; + break; + default: + break; + } break; - default: + case CLUSTER_VIS_ROUTE_NATIVE: break; } - break; - case CLUSTER_VIS_ROUTE_NATIVE: - break; } /* xmin NONE/LOCAL: leave xmin to PG unless we must intercept the xmax. */ @@ -2465,17 +2470,15 @@ HeapTupleSatisfiesVacuumHorizon(HeapTuple htup, Buffer buffer, TransactionId *de #ifdef USE_PGRAC_CLUSTER /* - * spec-3.14 D5 (hole #2): a tuple with REMOTE writer evidence must never - * be judged DEAD by this node's local horizon -- overlapping xid spaces - * mean local oldest_xmin can pass a still-live remote xid (false-dead). - * Return HEAPTUPLE_LIVE: the RECENTLY_DEAD path re-tests dead_after - * against the LOCAL oldest_xmin / vistest (heap_prune_satisfies_vacuum) - * and would re-introduce the false-dead, so LIVE (unconditional keep) is - * the only safe verdict. Real reclamation waits for the cross-node - * vacuum-coordination spec (spec-3.14 §10). spec wrote RECENTLY_DEAD; - * LIVE is the implementation-time correction (dead_after re-test hazard). + * P0-28 / spec-3.14 §10: a peer may retain this exact TID across a + * PCM-X wait. ITL evidence is not a durable prune-horizon token: its slot + * may be reused before that peer reacquires the page, after which native + * xmin/CLOG checks can misclassify and recycle the target line pointer. + * Keep every shared-storage tuple until a real cluster horizon exists. */ - if (cluster_storage_mode_enabled() && cluster_tuple_has_remote_evidence(buffer, tuple)) { + if (cluster_vis_prune_must_defer(cluster_storage_mode_enabled() + && BufferIsValid(buffer) && !BufferIsLocal(buffer), + false)) { cluster_vis_bump_prune_remote_keep_count(); return HEAPTUPLE_LIVE; } diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c index bc2bc1741f..f3e4a027cc 100644 --- a/src/backend/access/heap/hio.c +++ b/src/backend/access/heap/hio.c @@ -66,6 +66,30 @@ cluster_hio_lease_target_block(Relation relation) } #endif +/* P0-20: acquire an ordered pair without ever sleeping on the second PCM-X + * conversion while the first content lock is held. BARRIER_CLOSED means the + * first page was frozen before this nested edge became visible. Release our + * first lock, resolve the second conversion alone, then retry in the original + * block order; pins remain owned by the caller throughout. */ +static void +cluster_hio_lock_buffer_pair(Buffer first, Buffer second) +{ + Assert(BufferIsValid(first)); + Assert(BufferIsValid(second)); + Assert(first != second); + + for (;;) + { + LockBuffer(first, BUFFER_LOCK_EXCLUSIVE); + if (ClusterLockBufferExclusiveBarrierAware(second)) + return; + + LockBuffer(first, BUFFER_LOCK_UNLOCK); + LockBuffer(second, BUFFER_LOCK_EXCLUSIVE); + LockBuffer(second, BUFFER_LOCK_UNLOCK); + } +} + /* * RelationPutHeapTuple - place tuple at specified page * @@ -233,10 +257,12 @@ GetVisibilityMapPins(Relation relation, Buffer buffer1, Buffer buffer2, if (need_to_pin_buffer2) visibilitymap_pin(relation, block2, vmbuffer2); - /* Relock buffers. */ - LockBuffer(buffer1, BUFFER_LOCK_EXCLUSIVE); + /* Relock buffers. A cross-page relock keeps the same block order, + * but must unwind a frozen-barrier refusal from the second page. */ if (buffer2 != InvalidBuffer && buffer2 != buffer1) - LockBuffer(buffer2, BUFFER_LOCK_EXCLUSIVE); + cluster_hio_lock_buffer_pair(buffer1, buffer2); + else + LockBuffer(buffer1, BUFFER_LOCK_EXCLUSIVE); /* * If there are two buffers involved and we pinned just one of them, @@ -716,8 +742,7 @@ RelationGetBufferForTuple(Relation relation, Size len, buffer = ReadBuffer(relation, targetBlock); if (PageIsAllVisible(BufferGetPage(buffer))) visibilitymap_pin(relation, targetBlock, vmbuffer); - LockBuffer(otherBuffer, BUFFER_LOCK_EXCLUSIVE); - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + cluster_hio_lock_buffer_pair(otherBuffer, buffer); } else { @@ -725,8 +750,7 @@ RelationGetBufferForTuple(Relation relation, Size len, buffer = ReadBuffer(relation, targetBlock); if (PageIsAllVisible(BufferGetPage(buffer))) visibilitymap_pin(relation, targetBlock, vmbuffer); - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); - LockBuffer(otherBuffer, BUFFER_LOCK_EXCLUSIVE); + cluster_hio_lock_buffer_pair(buffer, otherBuffer); } /* @@ -900,8 +924,9 @@ RelationGetBufferForTuple(Relation relation, Size len, { /* released lock on target buffer above */ if (otherBuffer != InvalidBuffer) - LockBuffer(otherBuffer, BUFFER_LOCK_EXCLUSIVE); - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + cluster_hio_lock_buffer_pair(otherBuffer, buffer); + else + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); recheckVmPins = true; } else if (otherBuffer != InvalidBuffer) @@ -926,8 +951,7 @@ RelationGetBufferForTuple(Relation relation, Size len, { unlockedTargetBuffer = true; LockBuffer(buffer, BUFFER_LOCK_UNLOCK); - LockBuffer(otherBuffer, BUFFER_LOCK_EXCLUSIVE); - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + cluster_hio_lock_buffer_pair(otherBuffer, buffer); } recheckVmPins = true; } diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index f98a30191e..e1a97c0ee6 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -255,6 +255,32 @@ visibilitymap_pin(Relation rel, BlockNumber heapBlk, Buffer *vmbuf) *vmbuf = vm_readbuf(rel, mapBlock, true); } +/* + * Repin the exact visibility-map block in a recently observed descriptor. + * + * Unlike visibilitymap_pin(), this never performs a mapping lookup, storage + * read, or relation extension. A heap mutation can therefore release its VM + * pin before a potentially blocking cluster PCM acquire, take heap content + * authority, and use this helper without introducing I/O below that lock. A + * false result means the descriptor was invalidated or retagged; the caller + * must release heap content authority and retry through visibilitymap_pin(). + */ +bool +visibilitymap_pin_recent(Relation rel, BlockNumber heapBlk, Buffer recent_buffer, + Buffer *vmbuf) +{ + BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk); + + if (rel == NULL || vmbuf == NULL || BufferIsValid(*vmbuf) + || !BufferIsValid(recent_buffer)) + return false; + if (!ReadRecentBuffer(RelationGetSmgr(rel)->smgr_rlocator.locator, + VISIBILITYMAP_FORKNUM, mapBlock, recent_buffer)) + return false; + *vmbuf = recent_buffer; + return true; +} + /* * visibilitymap_pin_ok - do we already have the correct page pinned? * diff --git a/src/backend/cluster/cluster_cr_server.c b/src/backend/cluster/cluster_cr_server.c index 94656cc045..3007bde432 100644 --- a/src/backend/cluster/cluster_cr_server.c +++ b/src/backend/cluster/cluster_cr_server.c @@ -1049,7 +1049,7 @@ cr_serve_slot(ClusterLmsCrSlot *slot) CLUSTER_INJECTION_POINT("cluster-lms-cr-construct"); if (!cluster_injection_should_skip("cluster-lms-cr-construct") && cluster_crossnode_cr_data_plane - && cluster_bufmgr_copy_block_for_gcs(slot->tag, &page_lsn, cur_copy.data)) { + && cluster_bufmgr_copy_block_for_gcs(slot->tag, &page_lsn, cur_copy.data, NULL)) { PG_TRY(); { cluster_cr_construct_page_for_server(cur_copy.data, slot->read_scn, slot->tag, diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 85134ca511..fc28d67d81 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -1451,6 +1451,8 @@ dump_lms(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_lms_obs_get_outbound_not_admitted(-1))); emit_row(rsinfo, "lms", "lms_outbound_requeue_drop_count", fmt_int64((int64)cluster_lms_obs_get_outbound_requeue_drop(-1))); + emit_row(rsinfo, "lms", "lms_outbound_cap_guard_drop_count", + fmt_int64((int64)cluster_lms_obs_get_outbound_cap_guard_drop(-1))); { int w, hb; @@ -1474,6 +1476,9 @@ dump_lms(ReturnSetInfo *rsinfo) snprintf(wkey, sizeof(wkey), "lms_outbound_requeue_drop_count_w%d", w); emit_row(rsinfo, "lms", wkey, fmt_int64((int64)cluster_lms_obs_get_outbound_requeue_drop(w))); + snprintf(wkey, sizeof(wkey), "lms_outbound_cap_guard_drop_count_w%d", w); + emit_row(rsinfo, "lms", wkey, + fmt_int64((int64)cluster_lms_obs_get_outbound_cap_guard_drop(w))); } /* Inline-serve duration histogram: aggregate 16 rows + per live @@ -1995,15 +2000,60 @@ dump_pcm(ReturnSetInfo *rsinfo) emit_row(rsinfo, "pcm", "wm_prov_insert_fail_count", fmt_int64((int64)cluster_pcm_get_wm_prov_insert_fail_count())); - /* PCM-X queue core: 30 stable keys in the existing pcm category. */ + /* PCM-X queue core: 30 stable integer keys in the existing pcm category, + * plus one text-valued fail-closed provenance key (excluded from the + * integer-only snapshot tooling in the TAP suite). */ { PcmXStatsSnapshot stats = { 0 }; PcmXRuntimeSnapshot runtime = cluster_pcm_x_runtime_snapshot(); + char fail_closed_site[PCM_X_FAIL_CLOSED_SITE_LEN]; (void)cluster_pcm_x_stats_snapshot(&stats); emit_row(rsinfo, "pcm", "pcm_x_runtime_state", fmt_int32((int32)runtime.state)); emit_row(rsinfo, "pcm", "pcm_x_runtime_generation", fmt_int64((int64)runtime.gate_generation)); + if (!cluster_pcm_x_runtime_fail_closed_site(fail_closed_site, sizeof(fail_closed_site))) + strlcpy(fail_closed_site, "(none)", sizeof(fail_closed_site)); + emit_row(rsinfo, "pcm", "pcm_x_runtime_fail_closed_site", fail_closed_site); + + /* Text-valued per-slot diagnostic rows; present only while master tag + * or ticket slots are occupied, so the quiescent key count above is + * unchanged. */ + { + Size cursor = 0; + Size index = 0; + char line[768]; + char key[64]; + + while (cluster_pcm_x_master_tag_debug_next(&cursor, &index, line, sizeof(line))) { + snprintf(key, sizeof(key), "pcm_x_tag_%zu", index); + emit_row(rsinfo, "pcm", key, line); + } + cursor = 0; + while (cluster_pcm_x_master_ticket_debug_next(&cursor, &index, line, sizeof(line))) { + snprintf(key, sizeof(key), "pcm_x_ticket_%zu", index); + emit_row(rsinfo, "pcm", key, line); + } + cursor = 0; + while (cluster_pcm_x_local_tag_debug_next(&cursor, &index, line, sizeof(line))) { + snprintf(key, sizeof(key), "pcm_x_ltag_%zu", index); + emit_row(rsinfo, "pcm", key, line); + } + { + uint32 note_op = 0; + uint32 note_result = 0; + uint32 note_count = 0; + uint64 note_ticket = 0; + + if (cluster_pcm_x_terminal_note_read(¬e_op, ¬e_result, ¬e_ticket, + ¬e_count)) { + snprintf(line, sizeof(line), + "op=%u result=%u ticket=" UINT64_FORMAT " count=%u", note_op, + note_result, note_ticket, note_count); + emit_row(rsinfo, "pcm", "pcm_x_terminal_last_note", line); + } + } + } emit_row(rsinfo, "pcm", "pcm_x_queue_enqueue_count", fmt_int64((int64)stats.enqueue_count)); emit_row(rsinfo, "pcm", "pcm_x_queue_admit_count", fmt_int64((int64)stats.admit_count)); emit_row(rsinfo, "pcm", "pcm_x_queue_confirm_count", fmt_int64((int64)stats.confirm_count)); @@ -2043,6 +2093,8 @@ dump_pcm(ReturnSetInfo *rsinfo) emit_row(rsinfo, "pcm", "pcm_x_own_busy_count", fmt_int64((int64)stats.own_busy_count)); emit_row(rsinfo, "pcm", "pcm_x_own_corrupt_count", fmt_int64((int64)stats.own_corrupt_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_barrier_unwind_count", + fmt_int64((int64)stats.barrier_unwind_count)); } } @@ -2354,6 +2406,8 @@ dump_gcs(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_gcs_get_pi_watermark_advance_count())); emit_row(rsinfo, "gcs", "pi_watermark_retire_count", fmt_int64((int64)cluster_gcs_get_pi_watermark_retire_count())); + emit_row(rsinfo, "gcs", "pi_durable_note_apply_count", + fmt_int64((int64)cluster_gcs_get_pi_durable_note_apply_count())); emit_row(rsinfo, "gcs", "lost_write_detected_count", fmt_int64((int64)cluster_gcs_get_lost_write_detected_count())); emit_row(rsinfo, "gcs", "lost_write_avoid_count", diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index cb267c4aa2..2f86529350 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -260,10 +260,11 @@ typedef struct ClusterGcsBlockShared { pg_atomic_uint64 block_x_granted_from_holder_count; /* sender install X_GRANTED_FROM_HOLDER */ pg_atomic_uint64 starvation_denied_pending_x_count; /* N→S short-circuit by pending_x */ /* PGRAC: spec-2.37 D12 — 4 NEW counters for PI watermark + lost-write detection. */ - pg_atomic_uint64 pi_watermark_advance_count; /* X→N/S downgrade caller advance ticks */ - pg_atomic_uint64 pi_watermark_retire_count; /* tag lifecycle + durable-confirm retire */ - pg_atomic_uint64 lost_write_detected_count; /* master direct OR holder forward detect */ - pg_atomic_uint64 lost_write_avoid_count; /* durable-confirm retire avoided false-pos */ + pg_atomic_uint64 pi_watermark_advance_count; /* X→N/S downgrade caller advance ticks */ + pg_atomic_uint64 pi_watermark_retire_count; /* tag lifecycle + durable-confirm retire */ + pg_atomic_uint64 pi_durable_note_apply_count; /* target accepted status-3 DATA/CONTROL note */ + pg_atomic_uint64 lost_write_detected_count; /* master direct OR holder forward detect */ + pg_atomic_uint64 lost_write_avoid_count; /* durable-confirm retire avoided false-pos */ /* PGRAC: spec-2.41 D7 — SCN lost-write detector + redo-coverage observability. * Pure counters (no behavior change): the verdict still maps STALE+ANOMALY to * DENIED_LOST_WRITE and bumps lost_write_detected_count; these break that down @@ -447,7 +448,8 @@ static bool gcs_block_get_ship_image(BufferTag tag, int32 dest_node, bool allow_ const char **out_block_payload, uint32 *out_block_lkey, ClusterICSgeReleaseCallback *out_release_cb, void **out_release_arg, ClusterSfDepVec *out_sf_dep_vec, - bool *out_sf_dep_valid); + bool *out_sf_dep_valid, + ClusterBufmgrGcsCopyRefusal *out_copy_refusal); static void gcs_block_release_ship_image(ClusterICSgeReleaseCallback release_cb, void *release_arg); static uint32 gcs_block_compute_checksum(const char *block_data); static uint32 gcs_block_compute_invalidate_checksum(const GcsBlockInvalidatePayload *inv); @@ -585,6 +587,7 @@ cluster_gcs_block_shmem_init(void) /* PGRAC: spec-2.37 D12 — 4 NEW counters init. */ pg_atomic_init_u64(&ClusterGcsBlock->pi_watermark_advance_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->pi_watermark_retire_count, 0); + pg_atomic_init_u64(&ClusterGcsBlock->pi_durable_note_apply_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->lost_write_detected_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->lost_write_avoid_count, 0); /* PGRAC: spec-2.41 D7 — 4 NEW SCN detector + redo-coverage counters init. */ @@ -858,8 +861,24 @@ static void gcs_block_release_slot(ClusterGcsBlockOutstandingSlot *slot) { ClusterGcsBlockBackendBlock *blk = gcs_block_my_block(); + BufferTag released_tag; + uint64 released_request_id = 0; + uint64 released_epoch = 0; + int released_slot_index = -1; + int released_direct_state = (int)GCS_BLOCK_DIRECT_UNARMED; + bool released_reply_received = false; + bool released_live_direct = false; LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); + if (slot->in_use && slot->direct_target_prepared) { + released_tag = slot->tag; + released_request_id = slot->request_id; + released_epoch = slot->request_epoch; + released_slot_index = (int)(slot - &blk->slots[0]); + released_direct_state = (int)slot->direct_state; + released_reply_received = slot->reply_received; + released_live_direct = true; + } slot->in_use = false; slot->reply_received = false; slot->request_id = 0; @@ -881,6 +900,14 @@ gcs_block_release_slot(ClusterGcsBlockOutstandingSlot *slot) slot->direct_target_prepared = false; slot->direct_abort_reason = GCS_BLOCK_DIRECT_ABORT_NONE; LWLockRelease(&blk->lock.lock); + if (released_live_direct) + elog(LOG, + "cluster GCS block slot released with live direct target observation: " + "backend=%d slot=%d request_id=%llu epoch=%llu rel=%u fork=%d blk=%u " + "reply_received=%d direct_state=%d direct_prepared=1", + (int)MyBackendId - 1, released_slot_index, (unsigned long long)released_request_id, + (unsigned long long)released_epoch, released_tag.relNumber, (int)released_tag.forkNum, + released_tag.blockNum, released_reply_received ? 1 : 0, released_direct_state); } static uint32 @@ -949,11 +976,23 @@ gcs_block_direct_prepare_attempt(ClusterGcsBlockOutstandingSlot *slot, BufferDes int backend_idx = MyBackendId - 1; int slot_idx; int32 holder_node; + bool slot_eligible; if (slot == NULL || buf == NULL) return false; if (transition_id != PCM_TRANS_N_TO_S) return false; + /* A current-master INVALIDATE can now deliver a local exact denial before + * this attempt is sent. Close both sides of the target-preparation window: + * do not begin after that denial, and recheck after the bufmgr prepare in + * case it landed while the target was being pinned/prepared. */ + LWLockAcquire(&blk->lock.lock, LW_SHARED); + slot_eligible = slot->in_use && !slot->reply_received + && slot->direct_state == GCS_BLOCK_DIRECT_UNARMED + && !slot->direct_target_prepared; + LWLockRelease(&blk->lock.lock); + if (!slot_eligible) + return false; holder_node = cluster_pcm_master_holder_node_by_tag(tag); if (!GcsBlockDirectCanArmExpectedPeer(holder_node, expected_peer)) return false; @@ -964,6 +1003,12 @@ gcs_block_direct_prepare_attempt(ClusterGcsBlockOutstandingSlot *slot, BufferDes slot_idx = gcs_block_slot_index(blk, slot); LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); + if (!slot->in_use || slot->reply_received || slot->direct_state != GCS_BLOCK_DIRECT_UNARMED + || slot->direct_target_prepared) { + LWLockRelease(&blk->lock.lock); + gcs_block_direct_finish_target(buf, true, false, InvalidXLogRecPtr); + return false; + } slot->direct_generation = cluster_ic_rdma_direct_land_next_generation(slot->direct_generation); slot->direct_state = GCS_BLOCK_DIRECT_ARMING; slot->direct_expected_peer = expected_peer; @@ -1026,6 +1071,55 @@ cluster_gcs_block_compute_checksum(const char *block_data) return gcs_block_compute_checksum(block_data); } +/* + * A requester-side status=6 only says that some producer refused the image; + * the historical aggregate counter is also bumped by requesters and therefore + * cannot identify the producer branch. Emit one reason-tagged record at the + * producer with the unchanged wire identity and ONE entry-lock-coherent PCM + * authority snapshot. Do not reconstruct authority with separate state / + * holder / bitmap queries here: that would turn diagnostics into another + * observer race under the hot-block handoff this record is meant to explain. + */ +static void +gcs_block_log_master_not_holder_producer(const char *reason, BufferTag tag, uint64 request_id, + uint64 request_epoch, int32 requester_node, + uint8 transition_id) +{ + PcmAuthoritySnapshot authority; + bool authority_found; + + authority_found = cluster_pcm_lock_authority_snapshot(tag, &authority); + ereport( + LOG, + (errmsg_internal("cluster_gcs_block: master-not-holder reply produced"), + errdetail_internal( + "reason=%s producer=%d requester=%d request_id=" UINT64_FORMAT + " request_epoch=" UINT64_FORMAT + " transition=%u tag spc=%u db=%u relNumber=%u fork=%d block=%u " + "authority_found=%d state=%u x_holder=%d s_holders=0x%08x " + "master_holder_node=%u master_holder_procno=%u master_holder_epoch=" UINT64_FORMAT + " master_holder_request_id=" UINT64_FORMAT + " pending_x_requester=%d pending_x_since_lsn=" UINT64_FORMAT + " transition_count=" UINT64_FORMAT, + reason != NULL ? reason : "unknown", cluster_node_id, requester_node, request_id, + request_epoch, (unsigned int)transition_id, tag.spcOid, tag.dbOid, + (unsigned int)BufTagGetRelNumber(&tag), (int)tag.forkNum, (unsigned int)tag.blockNum, + (int)authority_found, (unsigned int)authority.state, authority.x_holder_node, + (unsigned int)authority.s_holders_bitmap, authority.master_holder.node_id, + authority.master_holder.procno, authority.master_holder.cluster_epoch, + authority.master_holder.request_id, authority.pending_x_requester_node, + authority.pending_x_since_lsn, authority.transition_count))); +} + +#define GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, reason) \ + gcs_block_log_master_not_holder_producer((reason), (req)->tag, (req)->request_id, \ + (req)->epoch, (req)->sender_node, \ + (req)->transition_id) +#define GCS_BLOCK_LOG_MASTER_NOT_HOLDER_FORWARD(fwd, reason) \ + gcs_block_log_master_not_holder_producer((reason), (fwd)->tag, (fwd)->request_id, \ + (fwd)->epoch, (fwd)->original_requester_node, \ + (fwd)->transition_id) + static void gcs_block_release_ship_image(ClusterICSgeReleaseCallback release_cb, void *release_arg) { @@ -1116,6 +1210,21 @@ gcs_block_send_direct_reply_sge(int32 dest_node, const GcsBlockReplyHeader *hdr, return rc; } +ClusterICSendResult +cluster_gcs_block_send_direct_zero_reply(int32 dest_node, const GcsBlockReplyHeader *header) +{ + ClusterICSendResult rc; + + if (header == NULL || dest_node < 0 || dest_node >= CLUSTER_MAX_NODES + || dest_node == cluster_node_id + || header->status != (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X) + return CLUSTER_IC_SEND_HARD_ERROR; + rc = gcs_block_send_direct_reply_sge(dest_node, header, NULL, 0, NULL, NULL); + if (rc == CLUSTER_IC_SEND_DONE && ClusterGcsBlock != NULL) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_reply_count, 1); + return rc; +} + static bool gcs_block_try_send_direct_reply(int32 dest_node, bool direct_armed, GcsBlockReplyHeader *hdr, const char *block_payload, uint32 block_lkey, @@ -1154,7 +1263,7 @@ gcs_block_get_ship_image(BufferTag tag, int32 dest_node, bool allow_live_sge, XLogRecPtr *out_page_lsn, char *copy_buf, const char **out_block_payload, uint32 *out_block_lkey, ClusterICSgeReleaseCallback *out_release_cb, void **out_release_arg, ClusterSfDepVec *out_sf_dep_vec, - bool *out_sf_dep_valid) + bool *out_sf_dep_valid, ClusterBufmgrGcsCopyRefusal *out_copy_refusal) { void *scratch = NULL; uint32 scratch_lkey = 0; @@ -1171,6 +1280,8 @@ gcs_block_get_ship_image(BufferTag tag, int32 dest_node, bool allow_live_sge, *out_release_arg = NULL; if (out_sf_dep_valid != NULL) *out_sf_dep_valid = false; + if (out_copy_refusal != NULL) + *out_copy_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE; if (out_sf_dep_vec != NULL) cluster_sf_dep_vec_reset(out_sf_dep_vec); @@ -1213,8 +1324,11 @@ gcs_block_get_ship_image(BufferTag tag, int32 dest_node, bool allow_live_sge, copied = cluster_bufmgr_copy_block_for_gcs_smart_fusion( tag, out_page_lsn, (char *)scratch, out_sf_dep_vec); else - copied = cluster_bufmgr_copy_block_for_gcs(tag, out_page_lsn, (char *)scratch); + copied = cluster_bufmgr_copy_block_for_gcs(tag, out_page_lsn, (char *)scratch, + out_copy_refusal); if (!copied) { + if (smart_fusion_reply && out_copy_refusal != NULL) + *out_copy_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_SMART_FUSION_UNCLASSIFIED; if (release_cb != NULL) release_cb(release_arg); return false; @@ -1237,12 +1351,15 @@ gcs_block_get_ship_image(BufferTag tag, int32 dest_node, bool allow_live_sge, if (smart_fusion_reply) { if (!cluster_bufmgr_copy_block_for_gcs_smart_fusion(tag, out_page_lsn, copy_buf, - out_sf_dep_vec)) + out_sf_dep_vec)) { + if (out_copy_refusal != NULL) + *out_copy_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_SMART_FUSION_UNCLASSIFIED; return false; + } if (out_sf_dep_valid != NULL) *out_sf_dep_valid = true; gcs_block_note_scratch_copy(); - } else if (!cluster_bufmgr_copy_block_for_gcs(tag, out_page_lsn, copy_buf)) + } else if (!cluster_bufmgr_copy_block_for_gcs(tag, out_page_lsn, copy_buf, out_copy_refusal)) return false; else gcs_block_note_scratch_copy(); @@ -1335,6 +1452,10 @@ gcs_block_install_block(BufferDesc *buf, const char *block_data, XLogRecPtr page memcpy(page, block_data, GCS_BLOCK_DATA_SIZE); gcs_block_note_install_copy(); PageSetLSN(page, page_lsn); + /* The shipped image just proved these bytes current: a kept-pinned + * retained PI mirror regains CURRENT inside the same content-EXCLUSIVE + * hold, or the grant finish would refuse the frozen PI shape. */ + cluster_bufmgr_pcm_own_republish_grant_pending_image(buf); LWLockRelease(content_lock); } @@ -1529,6 +1650,10 @@ cluster_gcs_block_fallback_verify_refresh(BufferDesc *buf, BufferTag tag, SCN ex verdict = gcs_block_lost_write_verdict(expected_scn, page_scn); if (verdict == GCS_LOST_WRITE_PASS) { pg_atomic_fetch_add_u64(&ClusterGcsBlock->fallback_scn_verify_pass_count, 1); + /* SCN proof: the local bytes (possibly a kept-pinned retained PI + * mirror) are at least the master watermark — republish CURRENT so + * the grant finish can commit over them. */ + cluster_bufmgr_pcm_own_republish_grant_pending_image(buf); return; } @@ -1545,8 +1670,12 @@ cluster_gcs_block_fallback_verify_refresh(BufferDesc *buf, BufferTag tag, SCN ex page_scn = InvalidScn; verdict = gcs_block_lost_write_verdict(expected_scn, page_scn); - if (verdict == GCS_LOST_WRITE_PASS) + if (verdict == GCS_LOST_WRITE_PASS) { + /* Refreshed from shared storage and proven: same republish as + * the direct PASS proof above. */ + cluster_bufmgr_pcm_own_republish_grant_pending_image(buf); return; + } } /* Refresh refused (dirty local copy) or the storage page is itself @@ -1881,7 +2010,8 @@ cluster_gcs_block_redo_lsn_covered(int dead_origin, XLogRecPtr required_lsn) bool cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition transition_id, - int master_node, bool clean_eligible) + int master_node, bool clean_eligible, + bool *out_retry_denied) { ClusterGcsBlockOutstandingSlot *slot; uint64 request_id = 0; @@ -1891,6 +2021,7 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans bool granted_storage_fallback = false; bool read_image = false; /* spec-5.2 D2: one-shot read image, non-durable */ bool terminal_denied = false; + bool retry_denied = false; bool retransmit_warning_emitted = false; bool suppress_direct_land = false; uint8 final_status = GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE; @@ -1912,6 +2043,11 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans /* PGRAC: spec-7.2 D6 — ship-latency histogram start stamp. */ TimestampTz ship_started_at; + Assert(out_retry_denied != NULL); + if (out_retry_denied == NULL) + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cluster_gcs_send_block_request_and_wait: NULL retry result"))); + *out_retry_denied = false; if (buf == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cluster_gcs_send_block_request_and_wait: NULL BufferDesc"))); @@ -2473,34 +2609,13 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans continue; } - /* PGRAC: spec-2.36 D6 (HC117) — reader starvation guard transient - * denial. N→S request was rejected because an X writer's broadcast - * is in flight at the master; reader exponential-backoffs per - * cluster.gcs_block_starvation_max_retries and - * cluster.gcs_block_starvation_backoff_ms. Budget exhaustion - * surfaces as 53R92 ERRCODE_CLUSTER_GCS_BLOCK_STARVATION_EXHAUSTED. */ + /* Queue arbitration owns this retry boundary. Consume the exact + * denial, release this request slot, and return to bufmgr so it can + * abort the matching GRANT_PENDING token before any backoff. The + * next acquire mints both a fresh token and a fresh request_id; a + * fixed starvation budget must never surface a client ERROR. */ if (final_status == GCS_BLOCK_REPLY_DENIED_PENDING_X) { - int starvation_attempt = retry_attempt; - int starvation_max = cluster_gcs_block_starvation_max_retries; - long backoff_ms; - - if (starvation_attempt >= starvation_max) { - terminal_denied = true; - ereport(ERROR, (errcode(ERRCODE_CLUSTER_GCS_BLOCK_STARVATION_EXHAUSTED), - errmsg("cluster_gcs_block: reader starvation retry budget " - "exhausted (HC117)"))); - break; - } - backoff_ms = (long)cluster_gcs_block_starvation_backoff_ms - * (1L << (starvation_attempt < 16 ? starvation_attempt : 16)); - if (backoff_ms > 25000) - backoff_ms = 25000; - (void)WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, backoff_ms, - WAIT_EVENT_GCS_BLOCK_STARVATION_RETRY); - ResetLatch(MyLatch); - current_master = cluster_gcs_lookup_master(tag); - if (retry_attempt < max_retries) - continue; + retry_denied = true; break; } @@ -2665,9 +2780,10 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans * histogram (GRANTED / STORAGE_FALLBACK / READ_IMAGE all delivered a * usable page; the terminal-denied tail below ereports and loses the * sample, mirroring the xp scopes). */ - if (granted || granted_storage_fallback || read_image) { + if (granted || granted_storage_fallback || read_image) gcs_block_ship_hist_record(ship_started_at); + if (granted || granted_storage_fallback || read_image || retry_denied) { /* * PGRAC: GCS-race round-2 RC-F — completion proof. The terminal * reply was verified and consumed, so no retransmit of this @@ -2715,6 +2831,10 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans return true; if (read_image) return false; + if (retry_denied) { + *out_retry_denied = true; + return false; + } if (terminal_denied) { switch ((GcsBlockReplyStatus)final_status) { @@ -3193,18 +3313,61 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle volatile PcmXQueueResult result = PCM_X_QUEUE_NOT_READY; int max_retries; int retry_attempt; + bool diagnostic; if (buf == NULL || leader == NULL || reservation_base == NULL || request_runtime == NULL || MyBackendId <= 0) return PCM_X_QUEUE_INVALID; + diagnostic = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); + memset(&progress_before, 0, sizeof(progress_before)); queue_result = cluster_pcm_x_local_progress_exact(leader, &progress_before); cluster_pcm_x_stats_note_queue_result(queue_result); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch requester boundary: entry"), + errdetail("progress_result=%d local=%d backend=%d source=%u master=%d " + "member_state=%u pending_opcode=%u last_response=%u request_id=%llu " + "ticket=%llu grant_generation=%llu image_id=%llu", + (int)queue_result, cluster_node_id, (int)MyBackendId, + progress_before.image.source_node, progress_before.master_node, + (unsigned)progress_before.member_state, + (unsigned)progress_before.pending_opcode, + (unsigned)progress_before.last_response_opcode, + (unsigned long long)progress_before.identity.request_id, + (unsigned long long)progress_before.ref.handle.ticket_id, + (unsigned long long)progress_before.ref.grant_generation, + (unsigned long long)progress_before.image.image_id))); if (queue_result != PCM_X_QUEUE_OK) return queue_result; if (!BufferTagsEqual(&buf->tag, &reservation_base->tag) || !BufferTagsEqual(&leader->identity.tag, &reservation_base->tag) - || reservation_base->generation != leader->identity.base_own_generation) + || reservation_base->generation + != (progress_before.grant_base_own_generation != 0 + ? progress_before.grant_base_own_generation + : leader->identity.base_own_generation)) { + if (diagnostic) + ereport( + LOG, + (errmsg_internal("PCM-X image fetch requester boundary: pre-slot reject"), + errdetail("branch=identity-base buf_tag_exact=%s leader_tag_exact=%s " + "base_generation=%llu progress_identity_base=%llu " + "progress_grant_base=%llu effective_grant_base=%llu " + "base_reservation_token=%llu base_writer_activation_token=%llu " + "base_flags=0x%x base_state=%u", + BufferTagsEqual(&buf->tag, &reservation_base->tag) ? "true" : "false", + BufferTagsEqual(&leader->identity.tag, &reservation_base->tag) ? "true" + : "false", + (unsigned long long)reservation_base->generation, + (unsigned long long)progress_before.identity.base_own_generation, + (unsigned long long)progress_before.grant_base_own_generation, + (unsigned long long)(progress_before.grant_base_own_generation != 0 + ? progress_before.grant_base_own_generation + : leader->identity.base_own_generation), + (unsigned long long)reservation_base->reservation_token, + (unsigned long long)reservation_base->writer_activation_token, + reservation_base->flags, (unsigned)reservation_base->pcm_state))); return PCM_X_QUEUE_STALE; + } if (!cluster_pcm_x_image_fetch_build_request(&progress_before, cluster_node_id, (int32)MyBackendId, &request)) return PCM_X_QUEUE_NOT_READY; @@ -3215,8 +3378,31 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle if (queue_result != PCM_X_QUEUE_OK) return queue_result; if (!cluster_pcm_x_image_fetch_reservation_exact(&live_own, reservation_base, - reservation_token)) - return gcs_block_pcm_x_fetch_reservation_mismatch(&live_own); + reservation_token)) { + queue_result = gcs_block_pcm_x_fetch_reservation_mismatch(&live_own); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch requester boundary: pre-slot reject"), + errdetail( + "branch=reservation-live result=%d live_tag_exact=%s " + "base_generation=%llu live_generation=%llu " + "base_reservation_token=%llu reservation_token=%llu " + "live_reservation_token=%llu base_writer_activation_token=%llu " + "live_writer_activation_token=%llu base_flags=0x%x live_flags=0x%x " + "base_state=%u live_state=%u", + (int)queue_result, + BufferTagsEqual(&live_own.tag, &reservation_base->tag) ? "true" : "false", + (unsigned long long)reservation_base->generation, + (unsigned long long)live_own.generation, + (unsigned long long)reservation_base->reservation_token, + (unsigned long long)reservation_token, + (unsigned long long)live_own.reservation_token, + (unsigned long long)reservation_base->writer_activation_token, + (unsigned long long)live_own.writer_activation_token, + reservation_base->flags, live_own.flags, + (unsigned)reservation_base->pcm_state, (unsigned)live_own.pcm_state))); + return queue_result; + } cluster_gcs_block_dedup_register_backend_exit_hook(); slot = gcs_block_try_reserve_exact_slot(request.tag, request.transition_id, @@ -3238,6 +3424,7 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle TimestampTz deadline; bool fence_lost = false; bool got_reply = false; + bool enqueue_result; bool slot_stale = false; if (retry_attempt > 0) { @@ -3312,9 +3499,19 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_request_count, 1); else pg_atomic_fetch_add_u64(&ClusterGcsBlock->retransmit_send_count, 1); - if (!cluster_grd_outbound_enqueue_backend_msg(PGRAC_IC_MSG_GCS_BLOCK_REQUEST, - progress_now.image.source_node, &request, - sizeof(request))) { + enqueue_result = cluster_grd_outbound_enqueue_backend_msg( + PGRAC_IC_MSG_GCS_BLOCK_REQUEST, progress_now.image.source_node, &request, + sizeof(request)); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch requester boundary: send"), + errdetail("attempt=%d enqueue_result=%s source=%u request_id=%llu " + "backend=%d transition=%u", + retry_attempt, enqueue_result ? "true" : "false", + progress_now.image.source_node, + (unsigned long long)request.request_id, + request.requester_backend_id, (unsigned)request.transition_id))); + if (!enqueue_result) { ConditionVariableCancelSleep(); if (retry_attempt < max_retries) continue; @@ -3355,6 +3552,15 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle } } ConditionVariableCancelSleep(); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch requester boundary: receive"), + errdetail("attempt=%d got_reply=%s slot_stale=%s fence_lost=%s " + "status=%d request_id=%llu", + retry_attempt, got_reply ? "true" : "false", + slot_stale ? "true" : "false", fence_lost ? "true" : "false", + got_reply ? (int)slot->reply_header.status : -1, + (unsigned long long)request.request_id))); if (fence_lost) { result = PCM_X_QUEUE_NOT_READY; break; @@ -3455,6 +3661,12 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle PG_END_TRY(); gcs_block_release_slot(slot); + if (diagnostic) + ereport(LOG, (errmsg_internal("PCM-X image fetch requester boundary: complete"), + errdetail("result=%d requester=%d backend=%d source=%u request_id=%llu", + (int)result, cluster_node_id, (int)MyBackendId, + progress_before.image.source_node, + (unsigned long long)request.request_id))); return (PcmXQueueResult)result; } @@ -3483,111 +3695,165 @@ cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle * be obtained — never a silent stale read (Rule 8.A). */ bool -cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, int32 holder_node) +cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, const PcmAuthoritySnapshot *expected, + bool *out_retry_denied) { ClusterGcsBlockOutstandingSlot *slot; uint64 request_id = 0; BufferTag tag; + int max_retries; + int retry_attempt; + int attempts = 0; + int last_status = -1; GcsBlockForwardPayload fwd; bool got_reply = false; bool installed = false; bool durable_s = false; /* spec-6.12a ㉕ — holder downgraded, we registered */ + int32 holder_node; - if (buf == NULL) - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("cluster_gcs_local_master_read_image_and_wait: NULL BufferDesc"))); + Assert(out_retry_denied != NULL); + if (buf == NULL || expected == NULL || out_retry_denied == NULL) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cluster_gcs_local_master_read_image_and_wait: invalid exact input"))); + *out_retry_denied = false; + holder_node = expected->x_holder_node; + if (expected->state != PCM_STATE_X || holder_node < 0 || holder_node >= 32 + || holder_node == cluster_node_id || expected->s_holders_bitmap != 0 + || expected->master_holder.node_id != (uint32)holder_node) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cluster_gcs_block: invalid remote-X authority for exact read image"))); tag = buf->tag; + /* P0-21 residual: the holder is an exact authority identity, not a route + * hint. A queue winner may have displaced a caller's optimistic sample + * before this helper starts; let bufmgr abort/rearm GRANT_PENDING and + * drive the new authority rather than sending to the stale node. */ + if (!cluster_pcm_lock_authority_matches(tag, expected)) { + *out_retry_denied = true; + return false; + } cluster_gcs_block_dedup_register_backend_exit_hook(); /* expected_master == self: the holder's reply carries forwarding_master = * self, which the HC108 authorized chain validates against this slot. */ slot = gcs_block_reserve_slot(tag, (uint8)PCM_TRANS_N_TO_S, cluster_node_id, &request_id); + max_retries = cluster_gcs_block_retransmit_max_retries >= 0 + ? cluster_gcs_block_retransmit_max_retries + : 4; PG_TRY(); { ClusterGcsBlockBackendBlock *blk = gcs_block_my_block(); TimestampTz deadline; - LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); - slot->reply_received = false; - memset(&slot->reply_header, 0, sizeof(slot->reply_header)); - memset(slot->reply_block_data, 0, sizeof(slot->reply_block_data)); - slot->reply_sf_dep_valid = false; - slot->reply_sf_flags = 0; - cluster_sf_dep_vec_reset(&slot->reply_sf_dep_vec); - slot->request_epoch = cluster_epoch_get_current(); - slot->expected_master_node = cluster_node_id; - slot->stale = false; - LWLockRelease(&blk->lock.lock); + for (retry_attempt = 0; retry_attempt <= max_retries; retry_attempt++) { + attempts = retry_attempt + 1; + got_reply = false; + if (retry_attempt > 0) { + long backoff_ms = gcs_block_backoff_ms_for_retry(retry_attempt); - memset(&fwd, 0, sizeof(fwd)); - fwd.request_id = request_id; - fwd.epoch = cluster_epoch_get_current(); - fwd.tag = tag; - fwd.original_requester_node = cluster_node_id; /* reply returns to us */ - fwd.requester_backend_id = (int32)MyBackendId; - fwd.master_node = cluster_node_id; - fwd.transition_id = (uint8)PCM_TRANS_N_TO_S; - GcsBlockForwardPayloadSetExpectedPiWatermarkScn( - &fwd, cluster_pcm_lock_pi_watermark_scn_query(tag)); - GcsBlockForwardPayloadSetReadImage(&fwd, true); - /* PGRAC: spec-6.12a ㉕ — ask the remote X holder to TRY the quiescent + pg_atomic_fetch_add_u64(&ClusterGcsBlock->retransmit_attempt_count, 1); + (void)WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, backoff_ms, + WAIT_EVENT_GCS_BLOCK_RETRANSMIT_WAIT); + ResetLatch(MyLatch); + /* The bounded retry belongs only to the exact remote-X authority + * sampled by the caller. Do not spend another request identity on + * a holder displaced while we backed off. */ + if (!cluster_pcm_lock_authority_matches(tag, expected)) { + *out_retry_denied = true; + break; + } + if (!gcs_block_pcm_x_next_request_id(&request_id)) + ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("cluster_gcs_block: read-image request id exhausted"))); + } + + LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); + slot->request_id = request_id; + slot->reply_received = false; + memset(&slot->reply_header, 0, sizeof(slot->reply_header)); + memset(slot->reply_block_data, 0, sizeof(slot->reply_block_data)); + slot->reply_sf_dep_valid = false; + slot->reply_sf_flags = 0; + cluster_sf_dep_vec_reset(&slot->reply_sf_dep_vec); + slot->request_epoch = cluster_epoch_get_current(); + slot->expected_master_node = cluster_node_id; + slot->stale = false; + LWLockRelease(&blk->lock.lock); + + memset(&fwd, 0, sizeof(fwd)); + fwd.request_id = request_id; + fwd.epoch = cluster_epoch_get_current(); + fwd.tag = tag; + fwd.original_requester_node = cluster_node_id; /* reply returns to us */ + fwd.requester_backend_id = (int32)MyBackendId; + fwd.master_node = cluster_node_id; + fwd.transition_id = (uint8)PCM_TRANS_N_TO_S; + GcsBlockForwardPayloadSetExpectedPiWatermarkScn( + &fwd, cluster_pcm_lock_pi_watermark_scn_query(tag)); + GcsBlockForwardPayloadSetReadImage(&fwd, true); + /* PGRAC: spec-6.12a ㉕ — ask the remote X holder to TRY the quiescent * X->S downgrade so this read (and every later one) becomes a durable * cached S. We ARE the master here, so on the holder's durable reply * the registration is a local transition apply — no ACK wire. */ - if (cluster_read_scache) - GcsBlockForwardPayloadSetDowngradeRequest(&fwd, true); - - pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_forward_sent_count, 1); - if (!cluster_grd_outbound_enqueue_backend_msg(PGRAC_IC_MSG_GCS_BLOCK_FORWARD, - (uint32)holder_node, &fwd, sizeof(fwd))) - ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), - errmsg("cluster_gcs_block: failed to enqueue read-image FORWARD " - "to X holder %d", - holder_node))); + if (cluster_read_scache) + GcsBlockForwardPayloadSetDowngradeRequest(&fwd, true); - deadline = GetCurrentTimestamp() - + ((TimestampTz)cluster_gcs_reply_timeout_ms) * (TimestampTz)1000; + if (retry_attempt > 0) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->retransmit_send_count, 1); - ConditionVariablePrepareToSleep(&slot->reply_cv); - for (;;) { - TimestampTz now; - long timeout_ms; - bool have_reply; + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_forward_sent_count, 1); + if (!cluster_grd_outbound_enqueue_backend_msg(PGRAC_IC_MSG_GCS_BLOCK_FORWARD, + (uint32)holder_node, &fwd, sizeof(fwd))) + ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("cluster_gcs_block: failed to enqueue read-image FORWARD " + "to X holder %d", + holder_node))); - LWLockAcquire(&blk->lock.lock, LW_SHARED); - have_reply = slot->in_use && slot->reply_received; - LWLockRelease(&blk->lock.lock); - if (have_reply) { - got_reply = true; - break; - } - now = GetCurrentTimestamp(); - if (now >= deadline) - break; - timeout_ms = (long)((deadline - now) / 1000); - if (timeout_ms <= 0) - timeout_ms = 1; - (void)ConditionVariableTimedSleep(&slot->reply_cv, timeout_ms, - WAIT_EVENT_GCS_BLOCK_SHIP_WAIT); - } - ConditionVariableCancelSleep(); + deadline = GetCurrentTimestamp() + + ((TimestampTz)cluster_gcs_reply_timeout_ms) * (TimestampTz)1000; - if (got_reply - && (slot->reply_header.status == (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER - || slot->reply_header.status - == (uint8)GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE)) { - uint32 expected = slot->reply_header.checksum; - uint32 got = gcs_block_compute_checksum(slot->reply_block_data); + ConditionVariablePrepareToSleep(&slot->reply_cv); + for (;;) { + TimestampTz now; + long timeout_ms; + bool have_reply; - if (expected == got) { - gcs_block_install_reply_block(buf, slot->reply_block_data, - (XLogRecPtr)slot->reply_header.page_lsn, slot); - /* spec-5.14 D2 class 2: this node (local master) consumed the + LWLockAcquire(&blk->lock.lock, LW_SHARED); + have_reply = slot->in_use && slot->reply_received; + LWLockRelease(&blk->lock.lock); + if (have_reply) { + got_reply = true; + break; + } + now = GetCurrentTimestamp(); + if (now >= deadline) + break; + timeout_ms = (long)((deadline - now) / 1000); + if (timeout_ms <= 0) + timeout_ms = 1; + (void)ConditionVariableTimedSleep(&slot->reply_cv, timeout_ms, + WAIT_EVENT_GCS_BLOCK_SHIP_WAIT); + } + ConditionVariableCancelSleep(); + last_status = got_reply ? (int)slot->reply_header.status : -1; + + if (got_reply + && (slot->reply_header.status == (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER + || slot->reply_header.status + == (uint8)GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE)) { + uint32 expected = slot->reply_header.checksum; + uint32 got = gcs_block_compute_checksum(slot->reply_block_data); + + if (expected == got) { + gcs_block_install_reply_block(buf, slot->reply_block_data, + (XLogRecPtr)slot->reply_header.page_lsn, slot); + /* spec-5.14 D2 class 2: this node (local master) consumed the * remote holder's volatile image. */ - gcs_block_stamp_touched(holder_node, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); - installed = true; - /* + gcs_block_stamp_touched(holder_node, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + installed = true; + /* * PGRAC: spec-6.12a ㉕ — the holder downgraded X->S and shipped * a DURABLE S grant. We are the master: register ourselves as * an S holder with a LOCAL transition apply (no ACK wire). The @@ -3598,17 +3864,45 @@ cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, int32 holder_node) * pcm_state stays N (Rule 8.A: never a durable copy the * master entry does not track). */ - if (slot->reply_header.status - == (uint8)GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE) { - if (cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_S, - cluster_node_id)) - durable_s = true; - else - cluster_lever_a_note_remote_ack_degraded(); + if (slot->reply_header.status + == (uint8)GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE) { + if (cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_S, + cluster_node_id)) + durable_s = true; + else + cluster_lever_a_note_remote_ack_degraded(); + } + } else { + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_checksum_fail_count, 1); } - } else { - pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_checksum_fail_count, 1); } + + if (installed || durable_s) + break; + /* P0-21 completion: holder-side conditional BufferContent refusal is + * HC105 transient. Retry without ever blocking the DATA worker. A new + * request id on the next iteration prevents a delayed old denial from + * winning the re-armed slot. */ + if (got_reply + && slot->reply_header.status == (uint8)GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER + && retry_attempt < max_retries) { + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_master_not_holder_count, 1); + /* A refusal is transient only while the complete holder identity + * remains authoritative. Drift exits through the outer + * GRANT_PENDING abort/rearm path; this helper never guesses the + * successor holder. */ + if (!cluster_pcm_lock_authority_matches(tag, expected)) { + *out_retry_denied = true; + break; + } + continue; + } + /* Preserve the stable-authority terminal fail-close below, but do + * not surface it for an old identity displaced during a timeout, + * checksum failure, or other non-success terminal. */ + if (!cluster_pcm_lock_authority_matches(tag, expected)) + *out_retry_denied = true; + break; } } PG_CATCH(); @@ -3620,6 +3914,8 @@ cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, int32 holder_node) gcs_block_release_slot(slot); + if (*out_retry_denied) + return false; if (durable_s) return true; /* spec-6.12a ㉕ — durable S; caller mirrors pcm_state = S */ if (installed) @@ -3632,6 +3928,7 @@ cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, int32 holder_node) "for tag spc=%u db=%u relNumber=%u block=%u", holder_node, tag.spcOid, tag.dbOid, (unsigned int)BufTagGetRelNumber(&tag), (unsigned int)tag.blockNum), + errdetail("attempts=%d last_status=%d", attempts, last_status), errhint("The X holder did not ship a current image in time; retry, or " "inspect dump_gcs.cf_xheld_read_ship_count."))); return true; /* unreachable */ @@ -4373,8 +4670,8 @@ cluster_gcs_block_undo_multi_verdict_fetch_and_wait(int32 origin_node, MultiXact * of cluster_gcs_local_master_read_image_and_wait. */ bool -cluster_gcs_local_master_x_transfer_and_wait(BufferDesc *buf, int32 holder_node, - bool clean_eligible) +cluster_gcs_local_master_x_transfer_and_wait(BufferDesc *buf, const PcmAuthoritySnapshot *expected, + bool clean_eligible, bool *out_retry_denied) { ClusterGcsBlockOutstandingSlot *slot; uint64 request_id = 0; @@ -4386,15 +4683,32 @@ cluster_gcs_local_master_x_transfer_and_wait(BufferDesc *buf, int32 holder_node, uint8 reply_status = (uint8)GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE; /* spec-5.2a D3 */ XLogRecPtr installed_page_lsn = InvalidXLogRecPtr; SCN installed_page_scn = InvalidScn; + int32 holder_node; + PcmXTransferCommitResult commit_result; - if (buf == NULL) - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("cluster_gcs_local_master_x_transfer_and_wait: NULL BufferDesc"))); + Assert(out_retry_denied != NULL); + if (buf == NULL || expected == NULL || out_retry_denied == NULL) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cluster_gcs_local_master_x_transfer_and_wait: invalid exact input"))); + *out_retry_denied = false; + holder_node = expected->x_holder_node; + if (expected->state != PCM_STATE_X || holder_node < 0 || holder_node >= 32 + || holder_node == cluster_node_id || expected->s_holders_bitmap != 0 + || expected->master_holder.node_id != (uint32)holder_node) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cluster_gcs_block: invalid remote-X authority for exact transfer"))); tag = buf->tag; + /* P0-26 first barrier: do not emit a request for an authority token + * already displaced by another queue winner/session. */ + if (!cluster_pcm_lock_authority_matches(tag, expected)) { + *out_retry_denied = true; + return false; + } cluster_gcs_block_dedup_register_backend_exit_hook(); slot = gcs_block_reserve_slot(tag, (uint8)PCM_TRANS_N_TO_X, cluster_node_id, &request_id); - PG_TRY(); { ClusterGcsBlockBackendBlock *blk = gcs_block_my_block(); @@ -4561,17 +4875,31 @@ cluster_gcs_local_master_x_transfer_and_wait(BufferDesc *buf, int32 holder_node, gcs_block_release_slot(slot); if (installed) { - /* The holder shipped its current image and released its X; record self - * as the new authoritative X holder (durable). No master round-trip: - * THIS node is the master. spec-2.41 D2 — also advance the detector's - * SCN watermark from the installed image's pd_block_scn (local-page - * source; bytes are slot->reply_block_data). */ - cluster_pcm_lock_master_take_x_after_transfer(tag, installed_page_lsn, installed_page_scn, - holder_node, request_id, fwd.epoch); + /* P0-26 final barrier: the slot/request id isolates late wire + * replies; this exact authority compare isolates a valid old reply + * from a newer queue handoff, holder generation, or session. */ + commit_result = cluster_pcm_lock_master_take_x_after_transfer( + tag, expected, installed_page_lsn, installed_page_scn, holder_node, + (uint32)MyProc->pgprocno, request_id, fwd.epoch); + if (commit_result == PCM_X_TRANSFER_COMMIT_STALE + || commit_result == PCM_X_TRANSFER_COMMIT_NOT_FOUND) { + *out_retry_denied = true; + return false; + } + if (commit_result != PCM_X_TRANSFER_COMMIT_OK) + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cluster_gcs_block: invalid exact X-transfer commit state"))); pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_x_granted_from_holder_count, 1); if (clean_eligible) pg_atomic_fetch_add_u64(&ClusterGcsBlock->clean_page_xfer_count, 1); - return true; /* durable X grant — bufmgr mirrors buf->pcm_state = X */ + return true; + } + /* A denial/timeout from a holder whose authority was concurrently + * replaced is not a client terminal. Return to bufmgr so the exact + * GRANT_PENDING token and request id are both replaced. */ + if (!cluster_pcm_lock_authority_matches(tag, expected)) { + *out_retry_denied = true; + return false; } if (read_image) @@ -4859,6 +5187,26 @@ gcs_block_forward_send_admitted(int32 holder_node, const GcsBlockForwardPayload return rc == CLUSTER_IC_SEND_DONE || rc == CLUSTER_IC_SEND_WOULD_BLOCK; } +/* The generic IC sender intentionally treats self-destination as a no-op. + * GCS block replies are completion signals, so a same-node denial must enter + * the normal registered handler just like a wire reply. */ +static ClusterICSendResult +gcs_block_send_envelope_or_loopback(uint8 msg_type, int32 dest_node, const void *payload, + uint32 payload_len) +{ + ClusterICEnvelope envelope; + + if (dest_node != cluster_node_id) + return cluster_ic_send_envelope(msg_type, dest_node, payload, payload_len); + if (payload == NULL + || !cluster_ic_envelope_build(&envelope, msg_type, (uint32)cluster_node_id, + (uint32)cluster_node_id, payload, payload_len)) + return CLUSTER_IC_SEND_HARD_ERROR; + return cluster_ic_dispatch_envelope(&envelope, payload, cluster_node_id) + ? CLUSTER_IC_SEND_DONE + : CLUSTER_IC_SEND_HARD_ERROR; +} + static void gcs_block_send_reply(int32 dest_node, const GcsBlockRequestPayload *req, GcsBlockReplyStatus status, XLogRecPtr page_lsn, const char *block_data) @@ -4879,7 +5227,9 @@ gcs_block_send_reply(int32 dest_node, const GcsBlockRequestPayload *req, GcsBloc if (status == GCS_BLOCK_REPLY_GRANTED && block_data != NULL) hdr.checksum = gcs_block_compute_checksum(block_data); - if (GcsBlockRequestPayloadIsDirectLandArmed(req)) { + if (GcsBlockRequestPayloadIsDirectLandArmed(req) && dest_node != cluster_node_id) { + if (!GcsBlockReplyStatusIsDirectLandSendable(status)) + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "direct-land-nonsendable"); if (status != GCS_BLOCK_REPLY_GRANTED || block_data == NULL) { char zero_page[GCS_BLOCK_DATA_SIZE]; @@ -4914,7 +5264,8 @@ gcs_block_send_reply(int32 dest_node, const GcsBlockRequestPayload *req, GcsBloc wire_hdr = (GcsBlockReplyHeader *)buf; *wire_hdr = hdr; wire_hdr->checksum = gcs_block_compute_checksum(buf + sizeof(GcsBlockReplyHeader)); - rc = cluster_ic_send_envelope(PGRAC_IC_MSG_GCS_BLOCK_REPLY, dest_node, buf, total); + rc = gcs_block_send_envelope_or_loopback(PGRAC_IC_MSG_GCS_BLOCK_REPLY, dest_node, buf, + total); pfree(buf); } @@ -4938,6 +5289,7 @@ gcs_block_deny_direct_armed_forward_request(const GcsBlockRequestPayload *req) * consume/abort the direct receive first and let the requester retry * without direct-land. */ + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "direct-land-forward-rearm"); gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER, InvalidXLogRecPtr, NULL); return true; @@ -4956,7 +5308,7 @@ gcs_block_deny_direct_armed_forward_request(const GcsBlockRequestPayload *req) * then issues a new request with a fresh 4-tuple key (different cluster_ * epoch field) which will MISS_REGISTERED and produce a fresh reply. */ -static void +static bool gcs_block_resend_cached_reply(int32 dest_node, const GcsBlockDedupEntry *entry) { uint32 header_len; @@ -4965,6 +5317,10 @@ gcs_block_resend_cached_reply(int32 dest_node, const GcsBlockDedupEntry *entry) GcsBlockReplyHeader *hdr; char *block_data; bool has_block_payload; + ClusterICSendResult rc; + + if (entry == NULL) + return false; header_len = entry->has_sf_dep && entry->sf_dep_count > 0 ? (uint32)sizeof(GcsBlockReplyHeaderV2) @@ -5001,7 +5357,7 @@ gcs_block_resend_cached_reply(int32 dest_node, const GcsBlockDedupEntry *entry) != GCS_BLOCK_REPLY_NO_FORWARDING_MASTER) { Assert(false); /* classification bug — lookup must route FORWARDED */ pfree(buf); - return; + return false; } block_data = buf + header_len; has_block_payload = entry->status == GCS_BLOCK_REPLY_GRANTED @@ -5009,11 +5365,23 @@ gcs_block_resend_cached_reply(int32 dest_node, const GcsBlockDedupEntry *entry) if (has_block_payload) memcpy(block_data, entry->block_data, GCS_BLOCK_DATA_SIZE); /* else: block_data already zeroed by palloc0 */ - - cluster_gcs_block_note_send_outcome( - GCS_BLOCK_SEND_FAMILY_REPLY, - cluster_ic_send_envelope(PGRAC_IC_MSG_GCS_BLOCK_REPLY, dest_node, buf, total)); + hdr->checksum = gcs_block_compute_checksum(block_data); + + if ((entry->request_flags & GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND) != 0 + && dest_node != cluster_node_id) { + if (!GcsBlockReplyStatusIsDirectLandSendable((GcsBlockReplyStatus)hdr->status)) + gcs_block_log_master_not_holder_producer( + "cached-direct-land-nonsendable", entry->tag, entry->key.request_id, + entry->key.cluster_epoch, (int32)entry->key.origin_node_id, entry->transition_id); + (void)gcs_block_try_send_direct_reply(dest_node, true, hdr, + has_block_payload ? block_data : NULL, 0, NULL, NULL); + pfree(buf); + return true; + } + rc = gcs_block_send_envelope_or_loopback(PGRAC_IC_MSG_GCS_BLOCK_REPLY, dest_node, buf, total); + cluster_gcs_block_note_send_outcome(GCS_BLOCK_SEND_FAMILY_REPLY, rc); pfree(buf); + return rc == CLUSTER_IC_SEND_DONE || rc == CLUSTER_IC_SEND_WOULD_BLOCK; } @@ -5040,20 +5408,36 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe GcsBlockPcmXImageResult image_result; PcmXLocalHolderProgress holder_after; PcmXLocalHolderProgress holder_before; + PcmXImageFetchRequestRefusal request_refusal; PcmXQueueResult progress_result; uint64 master_session; uint64 requester_session; uint64 current_epoch; + int32 decoded_backend = -1; int32 encoded_master; + int32 decoded_requester = -1; int32 tag_master; + bool diagnostic; + bool master_authenticated = false; + bool master_binding_valid = false; + bool request_exact = false; + bool reply_result; if (req == NULL || !cluster_pcm_x_image_id_decode(req->request_id, &encoded_master, NULL)) return false; /* From here onward the reserved namespace is always consumed here. */ if (env == NULL || worker_id < 0 || worker_id >= cluster_lms_workers) return true; + diagnostic = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); current_epoch = cluster_epoch_get_current(); tag_master = cluster_gcs_lookup_master(req->tag); + if (diagnostic) + ereport(LOG, (errmsg_internal("PCM-X image fetch source boundary: ingress"), + errdetail("worker=%d source=%u dest=%u requester=%d backend=%d " + "request_id=%llu epoch=%llu encoded_master=%d tag_master=%d", + worker_id, env->source_node_id, env->dest_node_id, req->sender_node, + req->requester_backend_id, (unsigned long long)req->request_id, + (unsigned long long)req->epoch, encoded_master, tag_master))); if (encoded_master != tag_master || req->epoch != current_epoch || !gcs_block_pcm_x_source_capable(req->sender_node) || !cluster_qvotec_in_quorum() || !cluster_membership_is_member(cluster_node_id) @@ -5064,14 +5448,59 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe || !gcs_block_pcm_x_revalidate_peer_binding(req->sender_node, current_epoch, requester_session)) return true; + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch source boundary: validate"), + errdetail("stage=auth requester_session=%llu current_epoch=%llu tag_master=%d", + (unsigned long long)requester_session, (unsigned long long)current_epoch, + tag_master))); progress_result = cluster_pcm_x_local_holder_progress_exact(&req->tag, &holder_before); - if (progress_result != PCM_X_QUEUE_OK - || !cluster_pcm_x_image_fetch_request_exact(env, req, &holder_before, cluster_node_id, - tag_master, current_epoch) - || !gcs_block_pcm_x_authenticated_session(tag_master, current_epoch, &master_session) - || master_session != holder_before.master_session_incarnation - || !gcs_block_pcm_x_revalidate_peer_binding(tag_master, current_epoch, master_session)) + if (progress_result == PCM_X_QUEUE_OK) + request_exact = cluster_pcm_x_image_fetch_request_exact_diagnosed( + env, req, &holder_before, cluster_node_id, tag_master, current_epoch, &request_refusal); + else + request_refusal = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_ARGUMENT; + (void)cluster_gcs_requester_id_decode(holder_before.ref.identity.request_id, &decoded_requester, + &decoded_backend, NULL); + if (request_exact) + master_authenticated + = gcs_block_pcm_x_authenticated_session(tag_master, current_epoch, &master_session); + if (master_authenticated) + master_binding_valid + = master_session == holder_before.master_session_incarnation + && gcs_block_pcm_x_revalidate_peer_binding(tag_master, current_epoch, master_session); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X image fetch source boundary: validate"), + errdetail("stage=binding progress_result=%d request_exact=%s refusal=%d " + "master_authenticated=%s master_binding_valid=%s master_session=%llu " + "holder_master_session=%llu holder_master=%d holder_requester=%d " + "holder_pending=%u holder_phase=%u holder_flags=%u " + "payload_len=%u expected_len=%zu request_backend=%d decoded_backend=%d " + "decoded_requester=%d transition=%u holder_epoch=%llu wait_seq=%llu " + "ticket=%llu queue_generation=%llu grant_generation=%llu " + "image_source=%u holder_request_id=%llu holder_image_id=%llu", + (int)progress_result, request_exact ? "true" : "false", + (int)request_refusal, master_authenticated ? "true" : "false", + master_binding_valid ? "true" : "false", + (unsigned long long)(master_authenticated ? master_session : 0), + (unsigned long long)holder_before.master_session_incarnation, + holder_before.master_node, holder_before.ref.identity.node_id, + (unsigned)holder_before.pending_opcode, (unsigned)holder_before.phase, + (unsigned)holder_before.flags, (unsigned)env->payload_length, + sizeof(*req), req->requester_backend_id, decoded_backend, + decoded_requester, (unsigned)req->transition_id, + (unsigned long long)holder_before.ref.identity.cluster_epoch, + (unsigned long long)holder_before.ref.identity.wait_seq, + (unsigned long long)holder_before.ref.handle.ticket_id, + (unsigned long long)holder_before.ref.handle.queue_generation, + (unsigned long long)holder_before.ref.grant_generation, + holder_before.image.source_node, + (unsigned long long)holder_before.ref.identity.request_id, + (unsigned long long)holder_before.image.image_id))); + if (progress_result != PCM_X_QUEUE_OK || !request_exact || !master_authenticated + || !master_binding_valid) return true; memset(&key, 0, sizeof(key)); @@ -5083,11 +5512,22 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe binding.identity.ref = holder_before.ref; binding.identity.image = holder_before.image; binding.master_session = holder_before.master_session_incarnation; + binding.required_page_scn = holder_before.required_page_scn; memset(&cached, 0, sizeof(cached)); image_result = cluster_gcs_block_dedup_pcm_x_lookup(worker_id, &key, &req->tag, &binding, &cached); + if (diagnostic) + ereport(LOG, (errmsg_internal("PCM-X image fetch source boundary: lookup"), + errdetail("result=%d worker=%d requester=%u backend=%d request_id=%llu " + "ticket=%llu grant_generation=%llu image_id=%llu", + (int)image_result, worker_id, key.origin_node_id, + key.requester_backend_id, (unsigned long long)key.request_id, + (unsigned long long)binding.identity.ref.handle.ticket_id, + (unsigned long long)binding.identity.ref.grant_generation, + (unsigned long long)binding.identity.image.image_id))); if (image_result == GCS_BLOCK_PCM_X_IMAGE_NOT_READY) { /* Materialization/publication is still progressing on this same shard. */ + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "pcm-x-image-not-ready"); gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER, InvalidXLogRecPtr, NULL); return true; @@ -5108,7 +5548,13 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe tag_master, current_epoch)) return true; - gcs_block_resend_cached_reply(req->sender_node, &cached); + reply_result = gcs_block_resend_cached_reply(req->sender_node, &cached); + if (diagnostic) + ereport(LOG, (errmsg_internal("PCM-X image fetch source boundary: reply"), + errdetail("stage_result=%s requester=%d backend=%d request_id=%llu status=%u", + reply_result ? "true" : "false", req->sender_node, + req->requester_backend_id, (unsigned long long)req->request_id, + (unsigned)cached.status))); if (ClusterGcsBlock != NULL) { pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_reply_count, 1); pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_ship_bytes_total, GCS_BLOCK_DATA_SIZE); @@ -5139,7 +5585,7 @@ gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRe * dedup install should happen. */ static bool -gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, +gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, bool preprepared_image, GcsBlockReplyStatus *out_status, XLogRecPtr *out_page_lsn, const char **out_block_payload, uint32 *out_block_lkey, ClusterICSgeReleaseCallback *out_release_cb, void **out_release_arg, @@ -5147,11 +5593,18 @@ gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, { uint64 current_epoch; PcmLockMode state; + PcmGcsTransitionApplyResult apply_result; + ClusterBufmgrGcsCopyRefusal copy_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE; bool found; *out_status = GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE; - *out_page_lsn = InvalidXLogRecPtr; - *out_block_payload = NULL; + if (!preprepared_image) { + *out_page_lsn = InvalidXLogRecPtr; + *out_block_payload = NULL; + } else { + Assert(*out_page_lsn != InvalidXLogRecPtr); + Assert(*out_block_payload == block_buf); + } if (out_block_lkey != NULL) *out_block_lkey = 0; if (out_release_cb != NULL) @@ -5191,7 +5644,7 @@ gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, * - buffer present: D4 helper handles HC82 + HC89 then reply GRANTED */ state = cluster_pcm_lock_query(req->tag); - found = cluster_bufmgr_probe_block_for_gcs(req->tag); + found = preprepared_image || cluster_bufmgr_probe_block_for_gcs(req->tag); if (!found && state == PCM_LOCK_MODE_N) { SCN fallback_watermark_scn; @@ -5215,9 +5668,11 @@ gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, * image newer than storage) and must never be overwritten. */ fallback_watermark_scn = cluster_pcm_lock_pi_watermark_scn_query(req->tag); - if (!cluster_pcm_lock_apply_gcs_transition(req->tag, (PcmLockTransition)req->transition_id, - req->sender_node)) { - *out_status = GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE; + apply_result = cluster_pcm_lock_apply_gcs_transition_result( + req->tag, (PcmLockTransition)req->transition_id, req->sender_node); + if (apply_result != PCM_GCS_TRANSITION_APPLIED) { + *out_status + = GcsBlockApplyRefusalStatus(apply_result, (PcmLockTransition)req->transition_id); return true; } pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_storage_fallback_count, 1); @@ -5248,27 +5703,37 @@ gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, if (!found) { pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_master_not_holder_count, 1); + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "produce-no-resident-authority"); *out_status = GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; return true; } /* * D4 bufmgr helper performs HC82 XLogFlush(page_lsn) + content_lock dance - * + HC89 single-retry revalidation. Returns false if revalidation cannot - * stabilize after one retry → DENIED_MASTER_NOT_HOLDER fail-closed. + * + HC89 single-retry revalidation. Conditional BufferContent refusal is + * retried by the owning backend with a fresh reservation/request identity; + * structural and HC89 exhaustion remain DENIED_MASTER_NOT_HOLDER. */ - if (!gcs_block_get_ship_image(req->tag, req->sender_node, true, out_page_lsn, block_buf, - out_block_payload, out_block_lkey, out_release_cb, - out_release_arg, out_sf_dep_vec, out_sf_dep_valid)) { - *out_status = GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; + if (!preprepared_image + && !gcs_block_get_ship_image(req->tag, req->sender_node, true, out_page_lsn, block_buf, + out_block_payload, out_block_lkey, out_release_cb, + out_release_arg, out_sf_dep_vec, out_sf_dep_valid, + ©_refusal)) { + char producer_reason[96]; + + snprintf(producer_reason, sizeof(producer_reason), "produce-copy-refused:%s", + cluster_bufmgr_gcs_copy_refusal_name(copy_refusal)); + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, producer_reason); + *out_status = GcsBlockMasterDirectCopyRefusalStatus(copy_refusal); return true; } if (out_sf_dep_valid == NULL || !*out_sf_dep_valid) pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_wal_flush_before_ship_count, 1); /* HC77: master-side is the single transition-apply owner. */ - if (!cluster_pcm_lock_apply_gcs_transition(req->tag, (PcmLockTransition)req->transition_id, - req->sender_node)) { + apply_result = cluster_pcm_lock_apply_gcs_transition_result( + req->tag, (PcmLockTransition)req->transition_id, req->sender_node); + if (apply_result != PCM_GCS_TRANSITION_APPLIED) { gcs_block_release_ship_image(out_release_cb != NULL ? *out_release_cb : NULL, out_release_arg != NULL ? *out_release_arg : NULL); *out_block_payload = NULL; @@ -5276,7 +5741,8 @@ gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, *out_release_cb = NULL; if (out_release_arg != NULL) *out_release_arg = NULL; - *out_status = GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE; + *out_status + = GcsBlockApplyRefusalStatus(apply_result, (PcmLockTransition)req->transition_id); return true; } @@ -5284,6 +5750,17 @@ gcs_block_produce_reply(const GcsBlockRequestPayload *req, char *block_buf, return true; } +static bool +gcs_block_queue_pending_x_authoritative(BufferTag tag) +{ + PcmAuthoritySnapshot authority; + + if (!cluster_pcm_lock_authority_snapshot(tag, &authority)) + return false; + return authority.pending_x_requester_node >= 0 + && (authority.pending_x_since_lsn & PCM_PENDING_X_QUEUE_KIND) != 0; +} + /* * cluster_gcs_handle_block_request_envelope — master-side dispatcher. * @@ -5319,6 +5796,11 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo void *block_payload_release_arg = NULL; ClusterSfDepVec sf_dep_vec; bool sf_dep_valid = false; + bool scache_image_prepared = false; + ClusterBufmgrGcsDowngradeOutcome local_downgrade_outcome + = CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + bool queue_pending_x_before = false; + uint8 request_flags = 0; (void)env; cluster_sf_dep_vec_reset(&sf_dep_vec); @@ -5419,11 +5901,57 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo key.request_id = req->request_id; key.cluster_epoch = req->epoch; memset(&cached_entry, 0, sizeof(cached_entry)); + if (req->transition_id == PCM_TRANS_N_TO_S) + queue_pending_x_before = gcs_block_queue_pending_x_authoritative(req->tag); dr = cluster_gcs_block_dedup_lookup_or_register( dedup_worker_id, &key, req->tag, req->transition_id, GcsBlockRequestPayloadGetLifetimeHintMs(req), cluster_sf_peer_supports_gcs_done(req->sender_node), &cached_entry); + if (dr != GCS_BLOCK_DEDUP_VALIDATION_FAIL && dr != GCS_BLOCK_DEDUP_FULL) { + if (GcsBlockRequestPayloadIsDirectLandArmed(req)) + request_flags |= GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND; + if (!cluster_gcs_block_dedup_set_request_flags_exact(dedup_worker_id, &key, &req->tag, + req->transition_id, request_flags)) { + /* The tuple was removed/replaced or its immutable request properties + * changed between lookup and pinning. Neither case may inherit the + * earlier entry's grant rights. */ + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, + InvalidXLogRecPtr, NULL); + return; + } + } + + /* Check both sides of registration. A queue claim that linearized before + * lookup routes this request directly to a cached exact denial; a claim + * that races after lookup is caught either here or by the claim-side scan. + * Queue ownership has no same-node exemption. */ + if (req->transition_id == PCM_TRANS_N_TO_S + && (queue_pending_x_before || gcs_block_queue_pending_x_authoritative(req->tag))) { + GcsBlockPendingXDenyResult deny_result; + + if (dr == GCS_BLOCK_DEDUP_VALIDATION_FAIL) { + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, + InvalidXLogRecPtr, NULL); + return; + } + if (dr == GCS_BLOCK_DEDUP_FULL) { + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_DEDUP_FULL, + InvalidXLogRecPtr, NULL); + return; + } + deny_result = cluster_gcs_block_dedup_pending_x_deny_exact( + dedup_worker_id, &key, &req->tag, req->transition_id, &cached_entry); + if (deny_result != GCS_BLOCK_PENDING_X_DENY_NEW + && deny_result != GCS_BLOCK_PENDING_X_DENY_REPLAY) { + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, + InvalidXLogRecPtr, NULL); + return; + } + pg_atomic_fetch_add_u64(&ClusterGcsBlock->starvation_denied_pending_x_count, 1); + (void)gcs_block_resend_cached_reply(req->sender_node, &cached_entry); + return; + } switch (dr) { case GCS_BLOCK_DEDUP_CACHED_REPLY: gcs_block_resend_cached_reply(req->sender_node, &cached_entry); @@ -5530,30 +6058,36 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo int32 pending_x; /* spec-2.36 D16 inject — force DENIED_PENDING_X for TAP coverage of - * reader starvation backoff + 53R92 budget exhaustion. */ + * exact abort + fresh-identity reader backoff. */ CLUSTER_INJECTION_POINT("cluster-gcs-block-starvation-force-denied"); if (cluster_injection_should_skip("cluster-gcs-block-starvation-force-denied")) { + GcsBlockPendingXDenyResult deny_result; + pg_atomic_fetch_add_u64(&ClusterGcsBlock->starvation_denied_pending_x_count, 1); - /* A retry-driven denial must not leave the IN_FLIGHT dedup entry - * behind: the requester's backoff retry reuses the same - * (request_id, epoch) key, so a leftover entry swallows the retry - * as IN_FLIGHT_DUPLICATE until the TTL sweep — each swallowed - * round burns a full cluster.gcs_reply_timeout_ms and the - * convergence the deny promises never re-evaluates. */ - cluster_gcs_block_dedup_remove(dedup_worker_id, &key); - gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_PENDING_X, - InvalidXLogRecPtr, NULL); + deny_result = cluster_gcs_block_dedup_pending_x_deny_exact( + dedup_worker_id, &key, &req->tag, req->transition_id, &cached_entry); + if (deny_result == GCS_BLOCK_PENDING_X_DENY_NEW + || deny_result == GCS_BLOCK_PENDING_X_DENY_REPLAY) + (void)gcs_block_resend_cached_reply(req->sender_node, &cached_entry); + else + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, + InvalidXLogRecPtr, NULL); return; } pending_x = cluster_pcm_lock_query_pending_x_requester(req->tag); if (pending_x >= 0 && pending_x != req->sender_node) { + GcsBlockPendingXDenyResult deny_result; + pg_atomic_fetch_add_u64(&ClusterGcsBlock->starvation_denied_pending_x_count, 1); - /* Same dedup-entry release as the inject branch above: the deny - * contract is "back off and retry, the retry re-evaluates". */ - cluster_gcs_block_dedup_remove(dedup_worker_id, &key); - gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_PENDING_X, - InvalidXLogRecPtr, NULL); + deny_result = cluster_gcs_block_dedup_pending_x_deny_exact( + dedup_worker_id, &key, &req->tag, req->transition_id, &cached_entry); + if (deny_result == GCS_BLOCK_PENDING_X_DENY_NEW + || deny_result == GCS_BLOCK_PENDING_X_DENY_REPLAY) + (void)gcs_block_resend_cached_reply(req->sender_node, &cached_entry); + else + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_VALIDATOR_REJECT, + InvalidXLogRecPtr, NULL); return; } } @@ -5618,7 +6152,7 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo if (gcs_block_get_ship_image(req->tag, req->sender_node, false, &page_lsn, block_buf, &block_payload, &block_payload_lkey, &block_payload_release_cb, &block_payload_release_arg, - &sf_dep_vec, &sf_dep_valid)) { + &sf_dep_vec, &sf_dep_valid, NULL)) { /* * PGRAC: spec-5.2 §3.5 D11 — active-ITL hard boundary. Even as master+holder, * if WE still have an uncommitted ITL slot on this block @@ -5782,6 +6316,7 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo == GCS_CLEAN_XFER_THIRD_PARTY_DENY) { pg_atomic_fetch_add_u64(&ClusterGcsBlock->clean_page_xfer_third_party_denied_count, 1); pg_atomic_fetch_add_u64(&ClusterGcsBlock->clean_page_xfer_fail_closed_count, 1); + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "clean-third-party-master"); gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER, InvalidXLogRecPtr, NULL); return; @@ -5883,6 +6418,7 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo * precedent above). */ cluster_gcs_block_dedup_remove(dedup_worker_id, &key); + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "live-x-other-holder"); gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER, InvalidXLogRecPtr, NULL); return; @@ -5898,10 +6434,12 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo cluster_gcs_block_dedup_remove(dedup_worker_id, &key); gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_PENDING_X, InvalidXLogRecPtr, NULL); - } else + } else { + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "pending-x-reserve-failed"); gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER, InvalidXLogRecPtr, NULL); + } return; } @@ -5991,12 +6529,14 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo &fwd_hdr, NULL); return; } + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "x-forward-send-failed"); (void)cluster_pcm_lock_clear_pending_x_if(req->tag, req->sender_node); status = GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; page_lsn = InvalidXLogRecPtr; block_payload = NULL; goto build_and_send_reply; } + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "x-state-holder-unroutable"); (void)cluster_pcm_lock_clear_pending_x_if(req->tag, req->sender_node); status = GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; page_lsn = InvalidXLogRecPtr; @@ -6049,7 +6589,7 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo && gcs_block_get_ship_image( req->tag, req->sender_node, false, &page_lsn, block_buf, &block_payload, &block_payload_lkey, &block_payload_release_cb, &block_payload_release_arg, - &sf_dep_vec, &sf_dep_valid)) + &sf_dep_vec, &sf_dep_valid, NULL)) xvs_b2_captured = true; if (!xvs_b2_captured) { @@ -6307,11 +6847,18 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo * state raced / flush unavailable) keeps today's one-shot ship. */ if (cluster_read_scache) { - bool downgraded = cluster_bufmgr_downgrade_x_to_s_for_gcs(req->tag); - - cluster_lever_a_note_downgrade(downgraded); - if (downgraded) + local_downgrade_outcome = cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image( + req->tag, &page_lsn, block_buf, NULL); + cluster_lever_a_note_downgrade(local_downgrade_outcome + == CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED); + if (local_downgrade_outcome + == CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY) + return; + if (local_downgrade_outcome == CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED) { + scache_image_prepared = true; + block_payload = block_buf; goto scache_downgraded_fall_through; + } } cluster_xp_begin(&xp_ship, CLXP_R_READIMAGE_SHIP); @@ -6319,7 +6866,7 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo if (gcs_block_get_ship_image(req->tag, req->sender_node, true, &page_lsn, block_buf, &block_payload, &block_payload_lkey, &block_payload_release_cb, &block_payload_release_arg, - &sf_dep_vec, &sf_dep_valid)) { + &sf_dep_vec, &sf_dep_valid, NULL)) { status = GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER; if (ClusterGcsBlock != NULL) pg_atomic_fetch_add_u64(&ClusterGcsBlock->cf_xheld_read_ship_count, 1); @@ -6473,8 +7020,8 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo * master state is now S with a resident clean buffer, so produce_reply * serves a durable GRANTED (image + requester N->S quick re-grant). */ scache_downgraded_fall_through: - (void)gcs_block_produce_reply(req, block_buf, &status, &page_lsn, &block_payload, - &block_payload_lkey, &block_payload_release_cb, + (void)gcs_block_produce_reply(req, block_buf, scache_image_prepared, &status, &page_lsn, + &block_payload, &block_payload_lkey, &block_payload_release_cb, &block_payload_release_arg, &sf_dep_vec, &sf_dep_valid); if (req->transition_id == PCM_TRANS_N_TO_X || req->transition_id == PCM_TRANS_S_TO_X_UPGRADE) (void)cluster_pcm_lock_clear_pending_x_if(req->tag, req->sender_node); @@ -6793,6 +7340,8 @@ build_and_send_reply: { = has_block_payload && block_payload_release_cb == gcs_block_release_live_sge; if (GcsBlockRequestPayloadIsDirectLandArmed(req)) { + if (!GcsBlockReplyStatusIsDirectLandSendable((GcsBlockReplyStatus)hdr->status)) + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_REQUEST(req, "direct-land-nonsendable"); (void)gcs_block_try_send_direct_reply( req->sender_node, true, hdr, has_block_payload ? block_payload : NULL, has_block_payload ? block_payload_lkey : 0, block_payload_release_cb, @@ -6942,12 +7491,29 @@ gcs_block_direct_fail_slot(ClusterGcsBlockBackendBlock *blk, ClusterGcsBlockOuts const GcsBlockReplyHeader *hdr) { BufferDesc *target_buf; + BufferTag target_tag; + ClusterPcmOwnSnapshot own; + ClusterPcmOwnResult own_result = CLUSTER_PCM_OWN_INVALID; + uint64 request_id; + uint64 request_epoch; + int backend_idx; + int slot_idx; + int direct_state; + int buffer_id = -1; bool prepared; + bool reply_received; Assert(blk != NULL); Assert(slot != NULL); target_buf = slot->direct_target_buf; + target_tag = slot->tag; + request_id = slot->request_id; + request_epoch = slot->request_epoch; + backend_idx = (int)(blk - &gcs_block_backend_blocks[0]); + slot_idx = (int)(slot - &blk->slots[0]); + direct_state = (int)slot->direct_state; + reply_received = slot->reply_received; prepared = slot->direct_target_prepared; slot->direct_state = GCS_BLOCK_DIRECT_ABORTED; slot->direct_abort_reason = reason; @@ -6973,6 +7539,22 @@ gcs_block_direct_fail_slot(ClusterGcsBlockBackendBlock *blk, ClusterGcsBlockOuts ConditionVariableSignal(&slot->reply_cv); LWLockRelease(&blk->lock.lock); gcs_block_direct_finish_target(target_buf, prepared, false, InvalidXLogRecPtr); + if (prepared || authoritative_denial) { + memset(&own, 0, sizeof(own)); + own_result = cluster_bufmgr_pcm_own_snapshot_by_tag(&target_tag, &buffer_id, &own); + elog(LOG, + "cluster GCS block direct abort observation: backend=%d slot=%d request_id=%llu " + "epoch=%llu rel=%u fork=%d blk=%u reason=%d authoritative_denial=%d " + "reply_received_before=%d direct_state_before=%d prepared=%d target_cleanup_done=%d " + "own_result=%d buffer=%d own_state=%u own_generation=%llu own_token=%llu " + "own_flags=0x%x", + backend_idx, slot_idx, (unsigned long long)request_id, + (unsigned long long)request_epoch, target_tag.relNumber, (int)target_tag.forkNum, + target_tag.blockNum, (int)reason, authoritative_denial ? 1 : 0, reply_received ? 1 : 0, + direct_state, prepared ? 1 : 0, prepared ? 1 : 0, (int)own_result, buffer_id, + own.pcm_state, (unsigned long long)own.generation, + (unsigned long long)own.reservation_token, own.flags); + } } void @@ -7406,6 +7988,7 @@ gcs_block_forward_reply_immediate_deny(const GcsBlockForwardPayload *fwd) char *deny_buf = (char *)palloc0(deny_total); GcsBlockReplyHeader *deny_hdr = (GcsBlockReplyHeader *)deny_buf; + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_FORWARD(fwd, "holder-immediate-deny"); deny_hdr->request_id = fwd->request_id; deny_hdr->epoch = cluster_epoch_get_current(); deny_hdr->sender_node = cluster_node_id; @@ -7432,6 +8015,10 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo void *block_payload_release_arg = NULL; XLogRecPtr page_lsn = InvalidXLogRecPtr; bool holder_ship_ok; + ClusterBufmgrGcsCopyRefusal copy_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE; + bool holder_evicted_injected = false; + ClusterBufmgrGcsDowngradeOutcome remote_downgrade_outcome + = CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; bool remote_downgraded = false; /* spec-6.12a ㉕ — holder accepted the * master's downgrade request */ ClusterSfDepVec sf_dep_vec; @@ -7577,6 +8164,13 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo return; } + /* Decide the holder-eviction fault before any irreversible downgrade. + * A forced DENIED must never first publish X->S and then discard the only + * image prepared for the requester. */ + CLUSTER_INJECTION_POINT("cluster-gcs-block-evict-holder-before-ship"); + holder_evicted_injected + = cluster_injection_should_skip("cluster-gcs-block-evict-holder-before-ship"); + /* * PGRAC: spec-6.12a ㉕ — remote-holder downgrade. The master asked us * (the X holder) to TRY the quiescent X->S self-downgrade before @@ -7593,23 +8187,31 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo && GcsBlockForwardPayloadIsDowngradeRequest(fwd) && fwd->transition_id == PCM_TRANS_N_TO_S) { CLUSTER_INJECTION_POINT("cluster-gcs-block-remote-downgrade"); - if (!cluster_injection_should_skip("cluster-gcs-block-remote-downgrade")) - remote_downgraded - = cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(fwd->tag, fwd->master_node); + if (!holder_evicted_injected + && !cluster_injection_should_skip("cluster-gcs-block-remote-downgrade")) + remote_downgrade_outcome = cluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image( + fwd->tag, fwd->master_node, &page_lsn, block_buf, ©_refusal); + remote_downgraded = remote_downgrade_outcome == CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED; cluster_lever_a_note_remote_downgrade(remote_downgraded); } + if (remote_downgrade_outcome == CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY) + return; - /* spec-2.35 D15 — fault injection. SKIP simulates evict race: - * holder pretends buffer is not cached so we exercise the HC105 - * DENIED_MASTER_NOT_HOLDER + sender retransmit budget path. */ - CLUSTER_INJECTION_POINT("cluster-gcs-block-evict-holder-before-ship"); - if (cluster_injection_should_skip("cluster-gcs-block-evict-holder-before-ship")) + /* spec-2.35 D15 — SKIP simulates the evict race. */ + if (holder_evicted_injected) { + copy_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INJECTED_EVICT; holder_ship_ok = false; - else + } else if (remote_downgraded) { + /* The exact downgrade prepared these bytes before notify+commit while + * holding the same content EXCLUSIVE lock. A second conditional copy + * would reopen the grant->ship race P0-32 closes. */ + holder_ship_ok = remote_downgraded; + block_payload = block_buf; + } else holder_ship_ok = gcs_block_get_ship_image( fwd->tag, fwd->original_requester_node, true, &page_lsn, block_buf, &block_payload, &block_payload_lkey, &block_payload_release_cb, &block_payload_release_arg, &sf_dep_vec, - &sf_dep_valid); + &sf_dep_valid, ©_refusal); /* Build reply (header + 8KB block or zero pad) and direct-ship to * the original requester. HC109 stores fwd->master_node in the @@ -7858,11 +8460,14 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo if (dres == CLUSTER_BUFMGR_GCS_DROP_PINNED || dres == CLUSTER_BUFMGR_GCS_DROP_STALE) { - if (dres == CLUSTER_BUFMGR_GCS_DROP_STALE) + if (dres == CLUSTER_BUFMGR_GCS_DROP_STALE) { pg_atomic_fetch_add_u64(&ClusterGcsBlock->xfer_stale_deny_count, 1); - else + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_FORWARD(fwd, "holder-drop-stale"); + } else { pg_atomic_fetch_add_u64(&ClusterGcsBlock->drop_pinned_deny_count, 1); + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_FORWARD(fwd, "holder-drop-pinned"); + } /* undo the from-holder ship count taken with the * grant status above — this reply is a deny now */ pg_atomic_fetch_sub_u64(&ClusterGcsBlock->block_from_holder_ship_count, @@ -7893,9 +8498,19 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo } } else { /* HC105 evict race */ + const char *copy_refusal_name = cluster_bufmgr_gcs_copy_refusal_name(copy_refusal); + hdr->checksum = gcs_block_compute_checksum(buf + header_len); hdr->status = (uint8)GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_forward_holder_evicted_count, 1); + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_FORWARD(fwd, "holder-copy-refused"); + ereport(LOG, (errmsg("cluster_gcs_block: holder ship image refused"), + errdetail("reason=%s request_id=" UINT64_FORMAT + " requester=%d master=%d tag spc=%u db=%u relNumber=%u block=%u", + copy_refusal_name, fwd->request_id, fwd->original_requester_node, + fwd->master_node, fwd->tag.spcOid, fwd->tag.dbOid, + (unsigned int)BufTagGetRelNumber(&fwd->tag), + (unsigned int)fwd->tag.blockNum))); } send_sf_dep = sf_peer_v2 && sf_dep_valid && block_payload != NULL @@ -7922,6 +8537,8 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo } if (GcsBlockForwardPayloadIsDirectLandArmed(fwd)) { + if (!GcsBlockReplyStatusIsDirectLandSendable((GcsBlockReplyStatus)hdr->status)) + GCS_BLOCK_LOG_MASTER_NOT_HOLDER_FORWARD(fwd, "holder-direct-land-nonsendable"); (void)gcs_block_try_send_direct_reply(fwd->original_requester_node, true, hdr, holder_ship_ok ? block_payload : NULL, holder_ship_ok ? block_payload_lkey : 0, @@ -8075,6 +8692,80 @@ typedef struct GcsBlockPcmXRequesterCleanupContext { static GcsBlockPcmXRequesterCleanupContext gcs_block_pcm_x_requester_cleanup_context; static bool gcs_block_pcm_x_requester_exit_hook_registered = false; +/* Source line of this backend's most recent non-OK acquire_writer_impl exit. + * Diagnostic only: consumed by the bufmgr failure report so a client-visible + * writer error names the exact escape arm instead of just the result code. */ +static int gcs_block_pcm_x_requester_fail_line = 0; + +/* Own-tuple evidence captured at the most recent non-OK remote-reservation + * preflight. Diagnostic only: printed by the fail-closed LOG so a STALE + * verdict names which preflight arm fired (base-generation drift vs a + * non-N pcm_state vs a live lifecycle flag) without a debugger. */ +typedef struct GcsBlockPcmXPreflightEvidence { + bool valid; + bool rebase_wire_active; + uint8 pcm_state; + uint32 own_flags; + uint64 live_generation; + uint64 base_own_generation; + uint64 reservation_token; + uint64 writer_activation_token; +} GcsBlockPcmXPreflightEvidence; + +static GcsBlockPcmXPreflightEvidence gcs_block_pcm_x_requester_preflight_evidence; + +/* A' rebase: publication is legal only under an ACTIVE runtime whose bound + * formation had full PCM_X_REBASE_V1 coverage at activation. */ +static bool +gcs_block_pcm_x_rebase_wire_active(const PcmXRuntimeSnapshot *runtime) +{ + return runtime != NULL && runtime->state == PCM_X_RUNTIME_ACTIVE && runtime->rebase_wire_active; +} + +/* The V2 (112-byte) frame exists on the wire only to carry a nonzero rebase; + * every other INSTALL_READY stays the V1 104-byte exact frame, so a formation + * without full V2 coverage never emits a length an old master would refuse + * (a nonzero rebase is impossible there: publication is gated on coverage). */ +static uint16 +gcs_block_pcm_x_install_ready_wire_len(const PcmXInstallReadyPayload *install_ready) +{ + return install_ready->rebased_own_generation != 0 ? (uint16)sizeof(*install_ready) + : (uint16)PCM_X_INSTALL_READY_V1_LEN; +} + +/* Drift is rebase-eligible only as the exact clean-N shape: same tag, idle + * lifecycle flags, pcm_state N and a strictly newer live generation. Any + * other preflight STALE (tag churn, non-N state, live flags) keeps the + * fail-closed verdict, as does a formation without full V2 coverage. */ +static bool +gcs_block_pcm_x_reservation_rebase_eligible(const ClusterPcmOwnSnapshot *live, + const PcmXWaitIdentity *identity, + const PcmXRuntimeSnapshot *runtime) +{ + return gcs_block_pcm_x_rebase_wire_active(runtime) && live != NULL && identity != NULL + && BufferTagsEqual(&live->tag, &identity->tag) && live->pcm_state == (uint8)PCM_STATE_N + && live->flags == 0 && live->generation != UINT64_MAX + && live->generation > identity->base_own_generation; +} + +#define GCS_BLOCK_PCM_X_REQUESTER_DONE() \ + do { \ + gcs_block_pcm_x_requester_fail_line = __LINE__; \ + goto requester_done; \ + } while (0) + +#define GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED() \ + do { \ + gcs_block_pcm_x_requester_fail_line = __LINE__; \ + goto requester_fail_closed; \ + } while (0) + +int +cluster_gcs_pcm_x_requester_last_fail_line(void) +{ + return gcs_block_pcm_x_requester_fail_line; +} + static bool gcs_block_pcm_x_revalidate_peer_binding(int32 node_id, uint64 epoch, uint64 session); static void @@ -8095,20 +8786,71 @@ gcs_block_pcm_x_requester_wait(uint32 *wait_index) CHECK_FOR_INTERRUPTS(); } +typedef struct GcsBlockPcmXRequesterWaitContext { + const PcmXRuntimeSnapshot *request_runtime; + int32 master_node; + uint64 cluster_epoch; + uint64 master_session; + long timeout_ms; +} GcsBlockPcmXRequesterWaitContext; + +static PcmXQueueResult +gcs_block_pcm_x_requester_pre_sleep_revalidate(void *callback_arg) +{ + GcsBlockPcmXRequesterWaitContext *context = (GcsBlockPcmXRequesterWaitContext *)callback_arg; + + if (context == NULL + || !gcs_block_pcm_x_requester_authority_exact(context->request_runtime, + context->master_node, context->cluster_epoch, + context->master_session)) + return PCM_X_QUEUE_NOT_READY; + return cluster_pcm_x_nested_wait_guard_before_block(); +} + +static void +gcs_block_pcm_x_requester_wait_latch(void *callback_arg) +{ + GcsBlockPcmXRequesterWaitContext *context = (GcsBlockPcmXRequesterWaitContext *)callback_arg; + + (void)WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, context->timeout_ms, + WAIT_EVENT_PCM_BLOCK_CONVERT_WAIT); +} + /* Retry only inside the exact formation that admitted this requester. Both * sides of the sleep are checked: a runtime fence or peer-session change that * lands while the latch is asleep must be observed before the next protocol * read or mutation. */ -static bool +static PcmXQueueResult gcs_block_pcm_x_requester_wait_exact(uint32 *wait_index, const PcmXRuntimeSnapshot *request_runtime, int32 master_node, uint64 cluster_epoch, uint64 master_session) { + GcsBlockPcmXRequesterWaitContext context; + PcmXQueueResult result; + uint32 current; + + if (wait_index == NULL) + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cluster PCM-X requester wait has no backoff state"))); if (!gcs_block_pcm_x_requester_authority_exact(request_runtime, master_node, cluster_epoch, master_session)) - return false; - gcs_block_pcm_x_requester_wait(wait_index); - return gcs_block_pcm_x_requester_authority_exact(request_runtime, master_node, cluster_epoch, - master_session); + return PCM_X_QUEUE_NOT_READY; + current = *wait_index; + memset(&context, 0, sizeof(context)); + context.request_runtime = request_runtime; + context.master_node = master_node; + context.cluster_epoch = cluster_epoch; + context.master_session = master_session; + context.timeout_ms = cluster_pcm_x_holder_retry_delay_ms(current); + CHECK_FOR_INTERRUPTS(); + result + = cluster_pcm_x_requester_wait_once_result(gcs_block_pcm_x_requester_pre_sleep_revalidate, + gcs_block_pcm_x_requester_wait_latch, &context); + if (result != PCM_X_QUEUE_OK) + return result; + ResetLatch(MyLatch); + *wait_index = cluster_gcs_pcm_x_requester_wait_index_advance(current); + CHECK_FOR_INTERRUPTS(); + return gcs_block_pcm_x_requester_pre_sleep_revalidate(&context); } static void @@ -8418,6 +9160,9 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim || MyProc->pgprocno < 0 || MyBackendId <= 0 || cluster_node_id < 0 || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) return PCM_X_QUEUE_INVALID; + /* Preflight evidence must belong to this acquire, never a previous one. */ + memset(&gcs_block_pcm_x_requester_preflight_evidence, 0, + sizeof(gcs_block_pcm_x_requester_preflight_evidence)); memset(&handle, 0, sizeof(handle)); handle.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; handle.membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; @@ -8456,7 +9201,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (wait_seq == 0) { cluster_pcm_x_runtime_fail_closed(); result = PCM_X_QUEUE_COUNTER_EXHAUSTED; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } wait_published = true; gcs_block_pcm_x_requester_cleanup_context.wait_published = true; @@ -8472,13 +9217,15 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (result == PCM_X_QUEUE_OK) break; if (result == PCM_X_QUEUE_CORRUPT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); if (!cluster_gcs_pcm_x_nested_guard_retryable(result)) - goto requester_done; - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); + result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } @@ -8490,33 +9237,35 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim result = gcs_block_pcm_x_fetch_own_result(own_result); if (result != PCM_X_QUEUE_OK) { if (result == PCM_X_QUEUE_CORRUPT) - goto requester_fail_closed; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } if (!BufferTagsEqual(&initial_own.tag, &buf->tag) || initial_own.generation == UINT64_MAX) { result = PCM_X_QUEUE_STALE; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } live_result = cluster_pcm_own_classify_live_flags(initial_own.flags, initial_own.reservation_token); result = gcs_block_pcm_x_fetch_own_result(live_result); if (result == PCM_X_QUEUE_CORRUPT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); if (initial_own.pcm_state == (uint8)PCM_STATE_READ_IMAGE || result == PCM_X_QUEUE_BUSY) { - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + result = gcs_block_pcm_x_requester_wait_exact( + &wait_index, &request_runtime, master_node, cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } continue; } if (result != PCM_X_QUEUE_OK) - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); if (initial_own.pcm_state != (uint8)PCM_STATE_N && initial_own.pcm_state != (uint8)PCM_STATE_S && initial_own.pcm_state != (uint8)PCM_STATE_X) { result = PCM_X_QUEUE_STALE; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } break; } @@ -8540,11 +9289,13 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action(GCS_BLOCK_PCM_X_RETRY_SITE_JOIN, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } gcs_block_pcm_x_requester_cleanup_context.handle = handle; @@ -8565,11 +9316,13 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_LEADER_REKEY, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + result = gcs_block_pcm_x_requester_wait_exact( + &wait_index, &request_runtime, master_node, cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } for (;;) { @@ -8584,11 +9337,13 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_CLAIM, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + result = gcs_block_pcm_x_requester_wait_exact( + &wait_index, &request_runtime, master_node, cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } for (;;) { @@ -8603,12 +9358,14 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_CUTOFF, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); gcs_block_pcm_x_requester_cleanup_context.cutoff_started = false; - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + result = gcs_block_pcm_x_requester_wait_exact( + &wait_index, &request_runtime, master_node, cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } } else if (handle.role == PCM_X_LOCAL_ROLE_FOLLOWER) { @@ -8622,7 +9379,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim = cluster_lmd_graph_remove_edge_by_waiter_exact_result(&waiter_vertex); if (remove_result == CLUSTER_LMD_GRAPH_REMOVE_STALE) { cluster_lmd_pcm_convert_wfg_note_exact_remove_stale(); - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } if (remove_result == CLUSTER_LMD_GRAPH_REMOVE_REMOVED) cluster_lmd_pcm_convert_wfg_note_remove(); @@ -8630,7 +9387,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim follower_graph_generation); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); follower_graph_generation = 0; gcs_block_pcm_x_requester_cleanup_context.wfg_live = false; gcs_block_pcm_x_requester_cleanup_context.wfg_generation = 0; @@ -8650,7 +9407,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (result == PCM_X_QUEUE_OK) { fail_site = "follower-refresh-compare"; if (!cluster_gcs_pcm_x_role_refresh_exact(&handle, &fresh_handle)) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); handle = fresh_handle; gcs_block_pcm_x_requester_cleanup_context.handle = handle; goto requester_role_dispatch; @@ -8658,20 +9415,21 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_ROLE_REFRESH, result); if (retry_action == GCS_BLOCK_PCM_X_RETRY_WAIT) { - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, - master_node, cluster_epoch, - master_session)) { - result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + result = gcs_block_pcm_x_requester_wait_exact( + &wait_index, &request_runtime, master_node, cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } continue; } - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_CLAIM, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); fail_site = "follower-wfg-snapshot"; result = cluster_pcm_x_local_follower_wfg_snapshot_exact(&handle, &follower_snapshot); @@ -8684,7 +9442,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim &waiter_vertex, &blocker_vertex, 1, handle.identity.request_id); if (follower_graph_generation == 0) { cluster_lmd_pcm_convert_wfg_note_replace_fail(); - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } cluster_lmd_pcm_convert_wfg_note_replace(); gcs_block_pcm_x_requester_cleanup_context.wfg_live = true; @@ -8700,7 +9458,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim = cluster_lmd_graph_remove_edge_by_waiter_exact_result(&waiter_vertex); if (remove_result == CLUSTER_LMD_GRAPH_REMOVE_STALE) { cluster_lmd_pcm_convert_wfg_note_exact_remove_stale(); - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } if (remove_result == CLUSTER_LMD_GRAPH_REMOVE_REMOVED) cluster_lmd_pcm_convert_wfg_note_remove(); @@ -8708,7 +9466,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim gcs_block_pcm_x_requester_cleanup_context.wfg_live = false; gcs_block_pcm_x_requester_cleanup_context.wfg_generation = 0; if (retry_action != GCS_BLOCK_PCM_X_RETRY_RESNAPSHOT_WFG) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } } else { retry_action = cluster_gcs_pcm_x_requester_retry_action( @@ -8725,7 +9483,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim if (result == PCM_X_QUEUE_OK) { fail_site = "follower-refresh-compare"; if (!cluster_gcs_pcm_x_role_refresh_exact(&handle, &fresh_handle)) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); handle = fresh_handle; gcs_block_pcm_x_requester_cleanup_context.handle = handle; goto requester_role_dispatch; @@ -8734,19 +9492,21 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim GCS_BLOCK_PCM_X_RETRY_SITE_ROLE_REFRESH, result); } if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + result = gcs_block_pcm_x_requester_wait_exact( + &wait_index, &request_runtime, master_node, cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } result = PCM_X_QUEUE_OK; - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } else { result = PCM_X_QUEUE_CORRUPT; - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } /* Node leader: every iteration reconstructs the exact durable outbound @@ -8762,12 +9522,12 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_PROGRESS, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); goto requester_wait; } if (progress.role != PCM_X_LOCAL_ROLE_NODE_LEADER || progress.master_node != master_node || progress.master_session_incarnation != master_session) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); if (progress.member_state == PCM_XL_NODE_LEADER) { if ((progress.pending_opcode == 0 && progress.last_response_opcode == 0) @@ -8786,7 +9546,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, arm_result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT && retry_action != GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } } else if (progress.pending_opcode == PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM || (progress.pending_opcode == 0 @@ -8805,7 +9565,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, arm_result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT && retry_action != GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } } } else if (progress.member_state == PCM_XL_REMOTE_WAIT) { @@ -8818,7 +9578,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim own_result = cluster_bufmgr_pcm_own_snapshot(buf, &reservation_base); result = gcs_block_pcm_x_fetch_own_result(own_result); if (result != PCM_X_QUEUE_OK) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); self_source = progress.image.source_node == (uint32)cluster_node_id; if (self_source) result = cluster_gcs_pcm_x_adopt_self_image(buf, &handle, &reservation_base, @@ -8827,15 +9587,65 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim result = cluster_gcs_pcm_x_remote_reservation_preflight(&reservation_base, &progress.identity); cluster_pcm_x_stats_note_queue_result(result); - if (result != PCM_X_QUEUE_OK) - goto requester_fail_closed; + if (result == PCM_X_QUEUE_STALE + && gcs_block_pcm_x_reservation_rebase_eligible( + &reservation_base, &progress.identity, &request_runtime)) { + /* An interleaved revoke consumed the enqueue-time + * base while this request was queued. Publish the + * live clean-N generation as the effective grant + * base (one-shot; replays are DUPLICATE) and carry + * it on the writer claim for the bufmgr cross + * check. The immutable identity never changes. */ + result = cluster_pcm_x_local_grant_rebase_publish_exact( + &handle, reservation_base.generation); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + claim_out->grant_base_own_generation = reservation_base.generation; + result = PCM_X_QUEUE_OK; + } + } + if (result != PCM_X_QUEUE_OK) { + GcsBlockPcmXPreflightEvidence *evidence + = &gcs_block_pcm_x_requester_preflight_evidence; + + evidence->valid = true; + evidence->rebase_wire_active + = gcs_block_pcm_x_rebase_wire_active(&request_runtime); + evidence->pcm_state = reservation_base.pcm_state; + evidence->own_flags = reservation_base.flags; + evidence->live_generation = reservation_base.generation; + evidence->base_own_generation = progress.identity.base_own_generation; + evidence->reservation_token = reservation_base.reservation_token; + evidence->writer_activation_token + = reservation_base.writer_activation_token; + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, result); + if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + goto requester_wait; + } own_result = cluster_bufmgr_pcm_own_begin_x_reservation( buf, &reservation_base, &reservation_token); result = gcs_block_pcm_x_fetch_own_result(own_result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { + GcsBlockPcmXPreflightEvidence *evidence + = &gcs_block_pcm_x_requester_preflight_evidence; + + evidence->valid = true; + evidence->rebase_wire_active + = gcs_block_pcm_x_rebase_wire_active(&request_runtime); + evidence->pcm_state = reservation_base.pcm_state; + evidence->own_flags = reservation_base.flags; + evidence->live_generation = reservation_base.generation; + evidence->base_own_generation = progress.identity.base_own_generation; + evidence->reservation_token = reservation_base.reservation_token; + evidence->writer_activation_token + = reservation_base.writer_activation_token; + } } cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); reservation_started = true; gcs_block_pcm_x_requester_cleanup_context.reservation_started = true; } @@ -8852,7 +9662,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_IMAGE_FETCH, result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } } if (image_installed) { @@ -8861,15 +9671,15 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim cluster_pcm_x_stats_note_queue_result(arm_result); result = arm_result; if (arm_result == PCM_X_QUEUE_OK || arm_result == PCM_X_QUEUE_DUPLICATE) - staged = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_INSTALL_READY, - master_node, &install_ready, - sizeof(install_ready)); + staged = cluster_gcs_pcm_x_stage_frame( + PGRAC_IC_MSG_PCM_X_INSTALL_READY, master_node, &install_ready, + gcs_block_pcm_x_install_ready_wire_len(&install_ready)); else { retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, arm_result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT && retry_action != GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } } } @@ -8883,15 +9693,15 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim cluster_pcm_x_stats_note_queue_result(arm_result); result = arm_result; if (arm_result == PCM_X_QUEUE_OK || arm_result == PCM_X_QUEUE_DUPLICATE) - staged = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_INSTALL_READY, - master_node, &install_ready, - sizeof(install_ready)); + staged = cluster_gcs_pcm_x_stage_frame( + PGRAC_IC_MSG_PCM_X_INSTALL_READY, master_node, &install_ready, + gcs_block_pcm_x_install_ready_wire_len(&install_ready)); else { retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, arm_result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT && retry_action != GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } } else if (progress.pending_opcode == 0 && progress.last_response_opcode == PGRAC_IC_MSG_PCM_X_COMMIT_X) { @@ -8910,7 +9720,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim } cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); ownership_committed = true; gcs_block_pcm_x_requester_cleanup_context.ownership_committed = true; } @@ -8922,7 +9732,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim staged = cluster_gcs_pcm_x_stage_frame( PGRAC_IC_MSG_PCM_X_FINAL_ACK, master_node, &final_ack, sizeof(final_ack)); else - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } else if (progress.pending_opcode == PGRAC_IC_MSG_PCM_X_FINAL_ACK) { PcmXFinalAckPayload final_ack; PcmXLocalReliableToken token; @@ -8938,7 +9748,7 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim retry_action = cluster_gcs_pcm_x_requester_retry_action( GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_REPLAY_ARM, arm_result); if (retry_action != GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS) - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } } } else if (progress.member_state == PCM_XL_GRANTED) { @@ -8959,18 +9769,20 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim result = PCM_X_QUEUE_OK; break; } else - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); } else - goto requester_fail_closed; + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); requester_wait: - if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, - cluster_epoch, master_session)) { - result = PCM_X_QUEUE_NOT_READY; - goto requester_done; + result = gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + GCS_BLOCK_PCM_X_REQUESTER_FAIL_CLOSED(); + GCS_BLOCK_PCM_X_REQUESTER_DONE(); } } - goto requester_done; + GCS_BLOCK_PCM_X_REQUESTER_DONE(); requester_fail_closed: /* Cleanup may cancel a purely local membership. PREPARE/COMMIT evidence @@ -8984,6 +9796,22 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim (unsigned int)progress.pending_opcode, (unsigned int)progress.last_response_opcode, (unsigned long long)request_id, (unsigned long long)wait_seq))); + if (gcs_block_pcm_x_requester_preflight_evidence.valid) + ereport( + LOG, + (errmsg("PCM-X requester reservation-preflight evidence"), + errdetail( + "live_gen=%llu base_gen=%llu reservation_token=%llu " + "writer_activation_token=%llu pcm_state=%u own_flags=%u rebase_active=%d", + (unsigned long long)gcs_block_pcm_x_requester_preflight_evidence.live_generation, + (unsigned long long) + gcs_block_pcm_x_requester_preflight_evidence.base_own_generation, + (unsigned long long)gcs_block_pcm_x_requester_preflight_evidence.reservation_token, + (unsigned long long) + gcs_block_pcm_x_requester_preflight_evidence.writer_activation_token, + (unsigned int)gcs_block_pcm_x_requester_preflight_evidence.pcm_state, + (unsigned int)gcs_block_pcm_x_requester_preflight_evidence.own_flags, + gcs_block_pcm_x_requester_preflight_evidence.rebase_wire_active ? 1 : 0))); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) result = PCM_X_QUEUE_CORRUPT; @@ -9085,19 +9913,25 @@ gcs_block_pcm_x_source_capable(int32 node_id) */ static bool gcs_block_pcm_x_collect_formation(PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_LIMIT], - uint64 *epoch_out, uint64 *self_session_out) + uint64 *epoch_out, uint64 *self_session_out, bool *rebase_all_out) { ClusterMembershipState membership_after[CLUSTER_MAX_NODES]; ClusterMembershipState membership_before[CLUSTER_MAX_NODES]; + uint32 cap_generation_before[CLUSTER_MAX_NODES]; + bool cap_rebase_before[CLUSTER_MAX_NODES]; uint64 epoch_after; uint64 epoch_before; uint64 peer_session; int i; + if (rebase_all_out != NULL) + *rebase_all_out = true; if (bindings == NULL || epoch_out == NULL || self_session_out == NULL || cluster_node_id < 0 || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) return false; memset(bindings, 0, sizeof(PcmXPeerBinding) * PCM_X_PROTOCOL_NODE_LIMIT); + memset(cap_generation_before, 0, sizeof(cap_generation_before)); + memset(cap_rebase_before, 0, sizeof(cap_rebase_before)); *epoch_out = 0; *self_session_out = 0; epoch_before = cluster_epoch_get_current(); @@ -9115,8 +9949,19 @@ gcs_block_pcm_x_collect_formation(PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_L continue; if (i >= PCM_X_PROTOCOL_NODE_LIMIT) return false; - if (i != cluster_node_id && !cluster_sf_peer_supports_pcm_x_convert(i)) - return false; + /* review P0-2: the CONVERT requirement, the REBASE coverage bit and + * the capability-record generation are one lock-coherent sample, so + * the after-pass below can prove the peer connection that advertised + * them is the SAME one the session binding names. Rebase coverage + * never refuses the base protocol: a member without the V2 bit only + * pins the whole formation to V1 frames. */ + if (i != cluster_node_id) { + if (!cluster_sf_peer_pcm_x_capability_sample(i, &cap_rebase_before[i], + &cap_generation_before[i])) + return false; + if (rebase_all_out != NULL && !cap_rebase_before[i]) + *rebase_all_out = false; + } auth_result = gcs_block_pcm_x_authenticated_session_result(i, epoch_before, &peer_session, NULL); if (auth_result != PCM_X_SESSION_AUTH_OK) @@ -9132,6 +9977,20 @@ gcs_block_pcm_x_collect_formation(PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_L if (epoch_before != epoch_after || !cluster_qvotec_in_quorum() || memcmp(membership_before, membership_after, sizeof(membership_before)) != 0) return false; + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + bool cap_rebase_after; + uint32 cap_generation_after; + + if (membership_before[i] != CLUSTER_MEMBER_MEMBER || i == cluster_node_id) + continue; + /* review P0-2 after-pass: any reconnect inside this collection moved + * the capability-record generation; refuse the tick rather than bind + * a stale connection's capability word to the fresh session. */ + if (!cluster_sf_peer_pcm_x_capability_sample(i, &cap_rebase_after, &cap_generation_after) + || cap_rebase_after != cap_rebase_before[i] + || cap_generation_after != cap_generation_before[i]) + return false; + } *epoch_out = epoch_before; *self_session_out = bindings[cluster_node_id].peer_session_incarnation; @@ -9143,6 +10002,10 @@ gcs_block_pcm_x_collect_formation(PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_L static void gcs_block_pcm_x_master_drive_retry_tick(void); static void gcs_block_pcm_x_terminal_retry_tick(void); +static void gcs_block_pcm_x_master_retry_observe(const char *stage, PcmXQueueResult result, + int32 peer_node, Size cursor_before, + Size cursor_after, const BufferTag *tag, + uint64 cluster_epoch); static PcmXQueueResult gcs_block_pcm_x_cancel_claimed_probe_exact(const PcmXMasterPendingXReleaseToken *token); @@ -9164,6 +10027,7 @@ cluster_gcs_block_pcm_x_formation_tick(void) uint64 epoch_before; uint64 self_session_after; uint64 self_session_before; + bool rebase_all = false; int i; runtime = cluster_pcm_x_runtime_snapshot(); @@ -9173,14 +10037,24 @@ cluster_gcs_block_pcm_x_formation_tick(void) * runtime. Both complete samples must first agree; any transient read, * including an unrelated membership flip, is a no-op for this tick. */ - if (!gcs_block_pcm_x_collect_formation(bindings_before, &epoch_before, - &self_session_before)) + if (!gcs_block_pcm_x_collect_formation(bindings_before, &epoch_before, &self_session_before, + NULL)) { + gcs_block_pcm_x_master_retry_observe("collect-before", PCM_X_QUEUE_NOT_READY, -1, 0, 0, + NULL, 0); return; - if (!gcs_block_pcm_x_collect_formation(bindings_after, &epoch_after, &self_session_after)) + } + if (!gcs_block_pcm_x_collect_formation(bindings_after, &epoch_after, &self_session_after, + NULL)) { + gcs_block_pcm_x_master_retry_observe("collect-after", PCM_X_QUEUE_NOT_READY, -1, 0, 0, + NULL, 0); return; + } if (!cluster_gcs_pcm_x_formation_samples_stable(true, bindings_before, true, bindings_after) - || epoch_before != epoch_after || self_session_before != self_session_after) + || epoch_before != epoch_after || self_session_before != self_session_after) { + gcs_block_pcm_x_master_retry_observe("stability", PCM_X_QUEUE_NOT_READY, -1, 0, 0, NULL, + epoch_before); return; + } for (i = 0; i < PCM_X_PROTOCOL_NODE_LIMIT; i++) { if (bindings_before[i].peer_session_incarnation == 0) continue; @@ -9188,14 +10062,20 @@ cluster_gcs_block_pcm_x_formation_tick(void) i, bindings_before[i].cluster_epoch, bindings_before[i].peer_session_incarnation); if (result == PCM_X_QUEUE_STALE || result == PCM_X_QUEUE_CORRUPT) goto fail_closed; - if (result != PCM_X_QUEUE_OK) + if (result != PCM_X_QUEUE_OK) { + gcs_block_pcm_x_master_retry_observe("peer-revalidate", result, i, 0, 0, NULL, + epoch_before); return; + } } runtime_after = cluster_pcm_x_runtime_snapshot(); if (runtime_after.state != PCM_X_RUNTIME_ACTIVE || runtime_after.gate_generation != runtime.gate_generation - || runtime_after.master_session_incarnation != runtime.master_session_incarnation) + || runtime_after.master_session_incarnation != runtime.master_session_incarnation) { + gcs_block_pcm_x_master_retry_observe("runtime-resample", PCM_X_QUEUE_NOT_READY, -1, 0, + 0, NULL, epoch_before); return; + } gcs_block_pcm_x_master_drive_retry_tick(); gcs_block_pcm_x_terminal_retry_tick(); return; @@ -9204,13 +10084,18 @@ cluster_gcs_block_pcm_x_formation_tick(void) /* ACTIVATING and any post-activation BLOCKED state are non-pristine. */ if (runtime.gate_generation != 0 || runtime.master_session_incarnation != 0) return; - if (!gcs_block_pcm_x_collect_formation(bindings_before, &epoch_before, &self_session_before)) + if (!gcs_block_pcm_x_collect_formation(bindings_before, &epoch_before, &self_session_before, + &rebase_all)) return; + /* Connection-bound capabilities are stable for the runtime's lifetime (a + * peer restart changes its session incarnation and permanently closes the + * steady-state core), so formation-wide V2 coverage is sampled once. */ + cluster_pcm_x_runtime_set_rebase_wire_active(rebase_all); (void)cluster_pcm_x_runtime_activate_bound(self_session_before, bindings_before); return; fail_closed: - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, PCM_X_RUNTIME_RECOVERY_BLOCKED); + cluster_pcm_x_runtime_fail_closed(); } @@ -9236,6 +10121,29 @@ cluster_gcs_pcm_x_stage_frame(uint8 msg_type, int32 dest_node_id, const void *pa payload_len); } +/* V2 type-49 is valid only on the exact DATA connection whose verified HELLO + * advertised the source-floor family. Route it to the same tag shard as the + * V1 frame, but retain the capability/generation fence in the LMS ring slot + * until immediately before transport admission. */ +static bool +gcs_block_pcm_x_stage_frame_cap_bound(uint8 msg_type, int32 dest_node_id, const void *payload, + uint16 payload_len, uint32 required_capability, + uint32 connection_generation) +{ + int worker; + + if (msg_type < PGRAC_IC_MSG_PCM_X_ENQUEUE || msg_type > PGRAC_IC_MSG_PCM_X_RETIRE_ACK + || dest_node_id < 0 || dest_node_id >= PCM_X_PROTOCOL_NODE_LIMIT || payload == NULL + || payload_len == 0 || required_capability == 0 || cluster_lms_workers <= 0) + return false; + worker = cluster_gcs_block_payload_shard(msg_type, payload, payload_len, cluster_lms_workers); + if (worker < 0 || worker >= cluster_lms_workers) + return false; + return cluster_lms_outbound_enqueue_cap_bound(worker, msg_type, (uint32)dest_node_id, payload, + payload_len, required_capability, + connection_generation); +} + /* RETIRE deliberately carries a cluster-wide ticket watermark rather than a * BufferTag. The LMON terminal driver must therefore supply the ticket's tag @@ -9307,8 +10215,7 @@ gcs_block_pcm_x_master_drive_fail_closed(PcmXQueueResult result) if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED || result == PCM_X_QUEUE_BAD_STATE || result == PCM_X_QUEUE_INVALID || result == PCM_X_QUEUE_NO_CAPACITY) - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + cluster_pcm_x_runtime_fail_closed(); } @@ -9433,6 +10340,49 @@ gcs_block_pcm_x_ensure_pending_x_claim(const PcmXMasterDriveSnapshot *snapshot) return PCM_X_QUEUE_OK; } +/* Once the queue-kind cookie is visible, every older same-tag legacy reader + * must lose its grant/forward right before PROBE/REVOKE can advance. Drain + * NEW victims one by one (no bounded receiver array), then replay one exact + * unacknowledged denial per drive tick until requester DONE removes it. */ +static PcmXQueueResult +gcs_block_pcm_x_deny_legacy_readers(const PcmXMasterDriveSnapshot *snapshot) +{ + GcsBlockDedupEntry denied; + GcsBlockPendingXDenyResult deny_result; + int worker_id; + + if (snapshot == NULL + || !cluster_pcm_lock_queue_pending_x_exact(snapshot->ref.identity.tag, + snapshot->ref.identity.node_id, + snapshot->ref.handle.ticket_id)) + return PCM_X_QUEUE_BAD_STATE; + worker_id = cluster_lms_shard_for_tag(&snapshot->ref.identity.tag, cluster_lms_workers); + if (worker_id < 0 || worker_id >= cluster_lms_workers) + return PCM_X_QUEUE_INVALID; + + for (;;) { + memset(&denied, 0, sizeof(denied)); + deny_result = cluster_gcs_block_dedup_pending_x_deny_next( + worker_id, &snapshot->ref.identity.tag, &denied); + if (deny_result == GCS_BLOCK_PENDING_X_DENY_NOT_FOUND) + return PCM_X_QUEUE_OK; + if (deny_result != GCS_BLOCK_PENDING_X_DENY_NEW + && deny_result != GCS_BLOCK_PENDING_X_DENY_REPLAY) + return PCM_X_QUEUE_CORRUPT; + if (denied.key.origin_node_id >= PCM_X_PROTOCOL_NODE_LIMIT + || denied.status != GCS_BLOCK_REPLY_DENIED_PENDING_X + || denied.reply_header.status != (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X) + return PCM_X_QUEUE_CORRUPT; + if (!cluster_lms_outbound_enqueue_zero_block_reply( + worker_id, denied.key.origin_node_id, &denied.reply_header, + (denied.request_flags & GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND) != 0 + && (int32)denied.key.origin_node_id != cluster_node_id)) + return PCM_X_QUEUE_BUSY; + if (deny_result == GCS_BLOCK_PENDING_X_DENY_REPLAY) + return PCM_X_QUEUE_OK; + } +} + static PcmXQueueResult gcs_block_pcm_x_stage_queue_invalidations(const PcmXMasterDriveSnapshot *snapshot, @@ -9472,20 +10422,72 @@ gcs_block_pcm_x_stage_queue_invalidations(const PcmXMasterDriveSnapshot *snapsho } +static bool +gcs_block_pcm_x_stage_frame_callback(uint8 msg_type, int32 dest_node_id, const void *payload, + Size payload_len, void *callback_arg) +{ + (void)callback_arg; + return cluster_gcs_pcm_x_stage_frame(msg_type, dest_node_id, payload, payload_len); +} + + +static uint64 +gcs_block_pcm_x_invalidate_busy_retry_delay_ms(const PcmXMasterDriveSnapshot *snapshot) +{ + uint64 delay_ms; + + /* reliable.retry_count belongs to the already-armed REVOKE leg. The + * transfer driver increments it while it is still draining earlier S + * holders, so it can be large before the first INVALIDATE BUSY arrives. + * It is not a BUSY-attempt counter and must not stretch that first retry + * to the 25s saturation ceiling (which phase-locks a denied reader's + * GRANT_PENDING retries against every INVALIDATE). Keep one bounded + * scheduling interval; repeated BUSY replies install fresh exact + * deadlines on the same ticket. */ + (void)snapshot; + delay_ms = (uint64)Max( + Max(cluster_gcs_block_starvation_backoff_ms, cluster_lmon_main_loop_interval), 1); + return delay_ms; +} + + static PcmXQueueResult gcs_block_pcm_x_master_drive_transfer(const PcmXMasterDriveSnapshot *snapshot) { + ClusterGcsPcmXAuthSample auth_sample; + PcmXMasterDriveSnapshot retried; PcmXRevokePayload revoke; + PcmXRevokePayloadV2 revoke_v2; PcmXQueueResult result; + uint64 now_ms; + uint64 retry_delay_ms; uint64 source_session = 0; + uint32 source_connection_generation = 0; uint32 source_bit; uint32 unacked; int32 source; + bool source_floor_capable = false; if (snapshot == NULL || snapshot->ticket_state != PCM_XT_ACTIVE_TRANSFER) return PCM_X_QUEUE_INVALID; + if (snapshot->pending_opcode == PGRAC_IC_MSG_PCM_X_COMMIT_X) { + now_ms = (uint64)(GetCurrentTimestamp() / 1000); + retry_delay_ms = (uint64)Max(cluster_lmon_main_loop_interval, 1); + result = cluster_pcm_x_master_commit_retry_drive( + snapshot, now_ms, retry_delay_ms, gcs_block_pcm_x_stage_frame_callback, NULL, &retried); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + gcs_block_pcm_x_master_retry_observe("commit-retry", result, -1, 0, 0, + &snapshot->ref.identity.tag, + snapshot->ref.identity.cluster_epoch); + return result; + } if (!cluster_gcs_pcm_x_transfer_pre_handoff_phase(snapshot->pending_opcode)) return PCM_X_QUEUE_NOT_READY; + if (snapshot->pending_opcode == PGRAC_IC_MSG_PCM_X_REVOKE && snapshot->retry_deadline_ms != 0) { + now_ms = (uint64)(GetCurrentTimestamp() / 1000); + if (now_ms < snapshot->retry_deadline_ms) + return PCM_X_QUEUE_NOT_READY; + } if (!cluster_pcm_lock_queue_pending_x_exact(snapshot->ref.identity.tag, snapshot->ref.identity.node_id, snapshot->ref.handle.ticket_id)) @@ -9507,8 +10509,13 @@ gcs_block_pcm_x_master_drive_transfer(const PcmXMasterDriveSnapshot *snapshot) if ((snapshot->pending_s_holders_bitmap & source_bit) == 0 || (snapshot->acked_s_holders_bitmap & source_bit) == 0) return PCM_X_QUEUE_CORRUPT; - if (!gcs_block_pcm_x_authenticated_session(source, snapshot->ref.identity.cluster_epoch, - &source_session) + memset(&auth_sample, 0, sizeof(auth_sample)); + if ((source == cluster_node_id + ? !gcs_block_pcm_x_authenticated_session(source, snapshot->ref.identity.cluster_epoch, + &source_session) + : gcs_block_pcm_x_authenticated_session_result( + source, snapshot->ref.identity.cluster_epoch, &source_session, &auth_sample) + != PCM_X_SESSION_AUTH_OK) || !gcs_block_pcm_x_revalidate_peer_binding(source, snapshot->ref.identity.cluster_epoch, source_session)) return PCM_X_QUEUE_NOT_READY; @@ -9519,6 +10526,36 @@ gcs_block_pcm_x_master_drive_transfer(const PcmXMasterDriveSnapshot *snapshot) unacked = snapshot->pending_s_holders_bitmap & ~snapshot->acked_s_holders_bitmap; if (unacked != 0) return gcs_block_pcm_x_stage_queue_invalidations(snapshot, unacked); + if (source == cluster_node_id) { + memset(&revoke_v2, 0, sizeof(revoke_v2)); + revoke_v2.v1 = revoke; + revoke_v2.required_page_scn + = (uint64)cluster_pcm_lock_pi_watermark_scn_query(snapshot->ref.identity.tag); + return cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_REVOKE, source, &revoke_v2, + sizeof(revoke_v2)) + ? PCM_X_QUEUE_OK + : PCM_X_QUEUE_BUSY; + } + /* Resample CONVERT + optional SOURCE_FLOOR under one capability-store + * lock after the authority pass. Any reconnect since authentication is + * retryable: the already-armed reliable leg remains intact. */ + if (!cluster_sf_peer_pcm_x_source_floor_sample(source, &source_floor_capable, + &source_connection_generation) + || source_connection_generation != auth_sample.connection_generation_before + || source_connection_generation != auth_sample.connection_generation_after) + return PCM_X_QUEUE_NOT_READY; + if (source_floor_capable) { + memset(&revoke_v2, 0, sizeof(revoke_v2)); + revoke_v2.v1 = revoke; + revoke_v2.required_page_scn + = (uint64)cluster_pcm_lock_pi_watermark_scn_query(snapshot->ref.identity.tag); + return gcs_block_pcm_x_stage_frame_cap_bound( + PGRAC_IC_MSG_PCM_X_REVOKE, source, &revoke_v2, sizeof(revoke_v2), + PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1, + source_connection_generation) + ? PCM_X_QUEUE_OK + : PCM_X_QUEUE_BUSY; + } return cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_REVOKE, source, &revoke, sizeof(revoke)) ? PCM_X_QUEUE_OK : PCM_X_QUEUE_BUSY; @@ -9690,50 +10727,213 @@ gcs_block_pcm_x_master_drive_cancel_requested(const PcmXMasterDriveSnapshot *sna } +/* Log-once evidence for the periodic master drive: a non-progress result + * that repeats consecutively with the same shape is a stalled drive, and a + * frozen wedge must name its refusing arm. Progress resets the streak. */ +static void +gcs_block_pcm_x_master_drive_note(const char *stage, PcmXQueueResult result, const BufferTag *tag, + uint64 cluster_epoch, const PcmXMasterDriveSnapshot *snapshot) +{ + static const char *last_stage = NULL; + static uint64 last_ticket = 0; + static uint64 last_request = 0; + static uint64 last_epoch = 0; + static BufferTag last_tag; + static bool last_tag_valid = false; + static int last_result = 0; + static bool logged = false; + uint64 ticket_id = snapshot != NULL ? snapshot->ref.handle.ticket_id : 0; + uint64 request_id = snapshot != NULL ? snapshot->ref.identity.request_id : 0; + bool same_tag + = tag == NULL ? !last_tag_valid : last_tag_valid && BufferTagsEqual(tag, &last_tag); + + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + last_stage = NULL; + last_tag_valid = false; + logged = false; + return; + } + if (stage == last_stage && (int)result == last_result && ticket_id == last_ticket + && request_id == last_request && cluster_epoch == last_epoch && same_tag) { + if (!logged) { + logged = true; + if (tag != NULL) + ereport(LOG, + (errmsg("PCM-X master drive is repeating %s", stage), + errdetail( + "result=%d epoch=%llu tag=%u/%u/%u/%d/%u " + "requester=%d procno=%u xid=%u request_id=%llu wait_seq=%llu " + "ticket=%llu queue_generation=%llu grant_generation=%llu " + "image_id=%llu source_generation=%llu source_node=%u", + (int)result, (unsigned long long)cluster_epoch, tag->spcOid, + tag->dbOid, tag->relNumber, (int)tag->forkNum, tag->blockNum, + snapshot != NULL ? snapshot->ref.identity.node_id : -1, + snapshot != NULL ? snapshot->ref.identity.procno : 0, + snapshot != NULL ? snapshot->ref.identity.xid : 0, + (unsigned long long)request_id, + (unsigned long long)(snapshot != NULL ? snapshot->ref.identity.wait_seq + : 0), + (unsigned long long)ticket_id, + (unsigned long long)(snapshot != NULL + ? snapshot->ref.handle.queue_generation + : 0), + (unsigned long long)(snapshot != NULL ? snapshot->ref.grant_generation + : 0), + (unsigned long long)(snapshot != NULL ? snapshot->image.image_id : 0), + (unsigned long long)(snapshot != NULL + ? snapshot->image.source_own_generation + : 0), + snapshot != NULL ? snapshot->image.source_node : 0))); + } + return; + } + last_stage = stage; + last_result = (int)result; + last_ticket = ticket_id; + last_request = request_id; + last_epoch = cluster_epoch; + if (tag != NULL) { + last_tag = *tag; + last_tag_valid = true; + } else + last_tag_valid = false; + logged = false; +} + + +/* One breadcrumb per stable pre-mutation refusal shape. A healthy idle LMON + * may observe NOT_FOUND forever, so the first observation logs and identical + * repeats stay silent until the exit shape changes. */ +static void +gcs_block_pcm_x_master_retry_observe(const char *stage, PcmXQueueResult result, int32 peer_node, + Size cursor_before, Size cursor_after, const BufferTag *tag, + uint64 cluster_epoch) +{ + static const char *last_stage = NULL; + static PcmXQueueResult last_result = PCM_X_QUEUE_OK; + static int32 last_peer_node = -1; + PcmXStatsSnapshot stats; + bool same; + + if (stage == NULL || result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + last_stage = NULL; + return; + } + if (!cluster_pcm_x_stats_snapshot(&stats) || stats.live_tickets == 0) { + last_stage = NULL; + return; + } + same = stage == last_stage && result == last_result && peer_node == last_peer_node; + if (same) + return; + last_stage = stage; + last_result = result; + last_peer_node = peer_node; + if (tag != NULL) + ereport(LOG, (errmsg("PCM-X periodic retry exit (stage %s, result %d, peer %d, " + "cursor %zu->%zu, epoch %llu, rel %u, block %u)", + stage, (int)result, peer_node, cursor_before, cursor_after, + (unsigned long long)cluster_epoch, tag->relNumber, tag->blockNum))); + else + ereport(LOG, (errmsg("PCM-X periodic retry exit (stage %s, result %d, peer %d, " + "cursor %zu->%zu, epoch %llu)", + stage, (int)result, peer_node, cursor_before, cursor_after, + (unsigned long long)cluster_epoch))); +} + + static void gcs_block_pcm_x_master_drive_tag(const BufferTag *tag, uint64 cluster_epoch) { PcmXMasterDriveSnapshot snapshot; PcmXTicketRef active; PcmXQueueResult result; + const char *drive_stage; - if (tag == NULL || cluster_epoch != cluster_epoch_get_current() || cluster_node_id < 0 - || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT - || cluster_gcs_lookup_master(*tag) != cluster_node_id - || cluster_gcs_block_phase_for_tag(*tag) == GCS_BLOCK_RECOVERING - || !cluster_qvotec_in_quorum() || !cluster_membership_is_member(cluster_node_id)) + if (tag == NULL) { + gcs_block_pcm_x_master_retry_observe("drive-precheck-tag", PCM_X_QUEUE_INVALID, -1, 0, 0, + NULL, cluster_epoch); + return; + } + if (cluster_epoch != cluster_epoch_get_current()) { + gcs_block_pcm_x_master_retry_observe("drive-precheck-epoch", PCM_X_QUEUE_STALE, -1, 0, 0, + tag, cluster_epoch); + return; + } + if (cluster_node_id < 0 || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) { + gcs_block_pcm_x_master_retry_observe("drive-precheck-node", PCM_X_QUEUE_INVALID, + cluster_node_id, 0, 0, tag, cluster_epoch); + return; + } + if (cluster_gcs_lookup_master(*tag) != cluster_node_id) { + gcs_block_pcm_x_master_retry_observe("drive-precheck-master", PCM_X_QUEUE_STALE, + cluster_node_id, 0, 0, tag, cluster_epoch); + return; + } + if (cluster_gcs_block_phase_for_tag(*tag) == GCS_BLOCK_RECOVERING) { + gcs_block_pcm_x_master_retry_observe("drive-precheck-recovering", PCM_X_QUEUE_NOT_READY, -1, + 0, 0, tag, cluster_epoch); return; + } + if (!cluster_qvotec_in_quorum()) { + gcs_block_pcm_x_master_retry_observe("drive-precheck-quorum", PCM_X_QUEUE_NOT_READY, -1, 0, + 0, tag, cluster_epoch); + return; + } + if (!cluster_membership_is_member(cluster_node_id)) { + gcs_block_pcm_x_master_retry_observe("drive-precheck-member", PCM_X_QUEUE_NOT_READY, + cluster_node_id, 0, 0, tag, cluster_epoch); + return; + } result = cluster_pcm_x_master_promote_head_exact(tag, cluster_epoch, &active); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_BUSY) { + PcmXMasterDriveSnapshot observed; + PcmXMasterDriveSnapshot *observed_ptr = NULL; + + if (cluster_pcm_x_master_drive_snapshot_exact(tag, cluster_epoch, &observed) + == PCM_X_QUEUE_OK) + observed_ptr = &observed; /* No promotable head means any earlier per-ticket anomaly for this * tag has resolved (cancelled / retired); settle its streaks. */ if (result == PCM_X_QUEUE_NOT_FOUND) cluster_gcs_pcm_x_drive_anomaly_settle(gcs_block_pcm_x_drive_anomaly_table, GCS_BLOCK_PCM_X_DRIVE_ANOMALY_SLOTS, tag); + gcs_block_pcm_x_master_drive_note("promote", result, tag, cluster_epoch, observed_ptr); gcs_block_pcm_x_master_drive_fail_closed(result); return; } result = cluster_pcm_x_master_drive_snapshot_exact(tag, cluster_epoch, &snapshot); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK) { + gcs_block_pcm_x_master_drive_note("snapshot", result, tag, cluster_epoch, NULL); gcs_block_pcm_x_master_drive_fail_closed(result); return; } if (snapshot.ticket_state == PCM_XT_ACTIVE_PROBE && snapshot.flags == (PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED - | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED)) + | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED)) { + drive_stage = "cancel-requested"; result = gcs_block_pcm_x_master_drive_cancel_requested(&snapshot); - else if (snapshot.ticket_state == PCM_XT_ACTIVE_PROBE) { + } else if (snapshot.ticket_state == PCM_XT_ACTIVE_PROBE) { + drive_stage = "probe"; result = gcs_block_pcm_x_ensure_pending_x_claim(&snapshot); + if (result == PCM_X_QUEUE_OK) + result = gcs_block_pcm_x_deny_legacy_readers(&snapshot); if (result == PCM_X_QUEUE_OK) result = gcs_block_pcm_x_master_drive_probe(&snapshot); - } else if (snapshot.ticket_state == PCM_XT_ACTIVE_TRANSFER) - result = gcs_block_pcm_x_master_drive_transfer(&snapshot); - else + } else if (snapshot.ticket_state == PCM_XT_ACTIVE_TRANSFER) { + drive_stage = "transfer"; + result = gcs_block_pcm_x_deny_legacy_readers(&snapshot); + if (result == PCM_X_QUEUE_OK) + result = gcs_block_pcm_x_master_drive_transfer(&snapshot); + } else { + drive_stage = "state"; result = PCM_X_QUEUE_CORRUPT; + } cluster_pcm_x_stats_note_queue_result(result); + gcs_block_pcm_x_master_drive_note(drive_stage, result, tag, cluster_epoch, &snapshot); /* Only definite drive progress settles the tag; indeterminate results * (NOT_READY / BUSY / STALE) must not reset a live streak, or a real * wedge interleaved with transients would never fuse. */ @@ -10201,8 +11401,7 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en commit->nblockers); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK) { - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + cluster_pcm_x_runtime_fail_closed(); goto blocker_commit_done; } for (i = 0; i < commit->nblockers; i++) @@ -10213,8 +11412,7 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en &waiter, blockers, (int)commit->nblockers, waiter.request_id); if (graph_generation == 0) { cluster_lmd_pcm_convert_wfg_note_replace_fail(); - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + cluster_pcm_x_runtime_fail_closed(); goto blocker_commit_done; } cluster_lmd_pcm_convert_wfg_note_replace(); @@ -10225,8 +11423,7 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en /* The ticket set has already been published. Removing by waiter here * could delete a concurrently newer graph generation; retain evidence * and close the runtime for recovery instead. */ - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + cluster_pcm_x_runtime_fail_closed(); goto blocker_commit_done; } @@ -10238,8 +11435,7 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en * this ACK. */ if (!cluster_gcs_pcm_x_blocker_ack_build(&commit->ref, commit->set_generation, &ack)) { Assert(false); - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + cluster_pcm_x_runtime_fail_closed(); goto blocker_commit_done; } if (!cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK, source_node, &ack, @@ -10257,8 +11453,7 @@ cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *en * the ACK above was merely replayed after a legal phase advance. The * classifier keeps BAD_STATE/STALE/NOT_READY/BUSY benign in that replay * case while structural CORRUPT still halts the runtime. */ - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); + cluster_pcm_x_runtime_fail_closed(); goto blocker_commit_done; } gcs_block_pcm_x_master_drive_tag(&commit->ref.identity.tag, commit->ref.identity.cluster_epoch); @@ -10813,6 +12008,13 @@ gcs_block_pcm_x_local_retire_apply_and_wake(const PcmXRetirePayload *request, wake_batch.count = 0; result = cluster_pcm_x_local_retire_up_to_collect_exact( request, authenticated_master_node, authenticated_master_session, &wake_batch); + if ((result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + && !cluster_gcs_block_dedup_pcm_x_retire_up_to( + request->cluster_epoch, authenticated_master_node, authenticated_master_session, + request->retire_through_ticket_id)) { + cluster_pcm_x_runtime_fail_closed(); + result = PCM_X_QUEUE_CORRUPT; + } if (result == PCM_X_QUEUE_OK) { for (i = 0; i < wake_batch.count && i < wake_batch.capacity; i++) (void)gcs_block_pcm_x_wake_requester_exact(&wake_batch.items[i], all_proc_count); @@ -10887,7 +12089,8 @@ gcs_block_pcm_x_image_key(const PcmXTicketRef *ref, uint64 image_id, GcsBlockDed static void gcs_block_pcm_x_reserved_binding(const PcmXRevokePayload *revoke, uint64 source_generation, - uint64 master_session, GcsBlockPcmXImageBinding *binding_out) + uint64 master_session, uint64 required_page_scn, + GcsBlockPcmXImageBinding *binding_out) { memset(binding_out, 0, sizeof(*binding_out)); binding_out->identity.ref = revoke->ref; @@ -10895,6 +12098,7 @@ gcs_block_pcm_x_reserved_binding(const PcmXRevokePayload *revoke, uint64 source_ binding_out->identity.image.source_own_generation = source_generation; binding_out->identity.image.source_node = (uint32)cluster_node_id; binding_out->master_session = master_session; + binding_out->required_page_scn = required_page_scn; } @@ -10912,7 +12116,8 @@ static bool gcs_block_pcm_x_release_image_exact(int worker_id, const GcsBlockPcmXImageWork *work, const GcsBlockPcmXImageBinding *binding) { - return cluster_gcs_block_dedup_pcm_x_release_exact(worker_id, &work->key, &work->tag, binding) + return cluster_gcs_block_dedup_pcm_x_release_exact(worker_id, &work->key, &work->tag, binding, + -1) == GCS_BLOCK_PCM_X_IMAGE_RELEASED; } @@ -10947,26 +12152,49 @@ gcs_block_pcm_x_abort_image_before_finish(int worker_id, const GcsBlockPcmXImage } +static void gcs_block_pcm_x_revoke_refusal_note_exact(const char *site, int own_result, + const ClusterPcmOwnSnapshot *snap, + const PcmXTicketRef *ref, uint64 image_id, + uint64 source_generation); +static void gcs_block_pcm_x_image_ready_arm_refusal_note_work(int result, + const GcsBlockPcmXImageWork *work, + PcmXLocalImageReadyRefusal refusal); + + static void gcs_block_pcm_x_stage_ready_work(int worker_id, const GcsBlockPcmXImageWork *work) { PcmXGrantPayload image_ready; PcmXGrantPayload replay; + PcmXLocalImageReadyRefusal refusal; PcmXQueueResult result; GcsBlockPcmXImageResult mark_result; GcsBlockPcmXImageResult rollback_result; int32 master_node; + bool diagnostic; + bool stage_result; if (!cluster_pcm_x_image_id_decode(work->binding.identity.image.image_id, &master_node, NULL)) { cluster_pcm_x_runtime_fail_closed(); return; } + diagnostic = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); memset(&image_ready, 0, sizeof(image_ready)); image_ready.ref = work->binding.identity.ref; image_ready.image = work->binding.identity.image; - result = cluster_pcm_x_local_holder_image_ready_arm_exact( - &image_ready, master_node, work->binding.master_session, &replay); + result = cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( + &image_ready, master_node, work->binding.master_session, &replay, &refusal); cluster_pcm_x_stats_note_queue_result(result); + if (diagnostic) + ereport(LOG, (errmsg_internal("PCM-X IMAGE_READY stage boundary: arm"), + errdetail("result=%d refusal=%d master=%d request_id=%llu ticket=%llu " + "grant_generation=%llu image_id=%llu", + (int)result, (int)refusal, master_node, + (unsigned long long)image_ready.ref.identity.request_id, + (unsigned long long)image_ready.ref.handle.ticket_id, + (unsigned long long)image_ready.ref.grant_generation, + (unsigned long long)image_ready.image.image_id))); + gcs_block_pcm_x_image_ready_arm_refusal_note_work((int)result, work, refusal); if (result == PCM_X_QUEUE_BUSY || result == PCM_X_QUEUE_NOT_READY) return; if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { @@ -10975,23 +12203,291 @@ gcs_block_pcm_x_stage_ready_work(int worker_id, const GcsBlockPcmXImageWork *wor } mark_result = cluster_gcs_block_dedup_pcm_x_mark_staged_exact(worker_id, &work->key, &work->tag, &work->binding); + if (diagnostic) + ereport(LOG, + (errmsg_internal("PCM-X IMAGE_READY stage boundary: mark"), + errdetail("mark_result=%d worker=%d request_id=%llu ticket=%llu image_id=%llu", + (int)mark_result, worker_id, + (unsigned long long)image_ready.ref.identity.request_id, + (unsigned long long)image_ready.ref.handle.ticket_id, + (unsigned long long)image_ready.image.image_id))); if (mark_result == GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) return; if (mark_result != GCS_BLOCK_PCM_X_IMAGE_STAGED) { cluster_pcm_x_runtime_fail_closed(); return; } - if (cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_IMAGE_READY, master_node, &replay, - sizeof(replay))) + stage_result = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_IMAGE_READY, master_node, + &replay, sizeof(replay)); + if (diagnostic) + ereport(LOG, (errmsg_internal("PCM-X IMAGE_READY stage boundary: DATA ring"), + errdetail("stage_result=%s worker=%d master=%d request_id=%llu ticket=%llu " + "image_id=%llu", + stage_result ? "true" : "false", worker_id, master_node, + (unsigned long long)image_ready.ref.identity.request_id, + (unsigned long long)image_ready.ref.handle.ticket_id, + (unsigned long long)image_ready.image.image_id))); + if (stage_result) return; rollback_result = cluster_gcs_block_dedup_pcm_x_unmark_staged_exact(worker_id, &work->key, &work->tag, &work->binding); + if (diagnostic) + ereport(LOG, (errmsg_internal("PCM-X IMAGE_READY stage boundary: rollback"), + errdetail("rollback_result=%d worker=%d request_id=%llu ticket=%llu " + "image_id=%llu", + (int)rollback_result, worker_id, + (unsigned long long)image_ready.ref.identity.request_id, + (unsigned long long)image_ready.ref.handle.ticket_id, + (unsigned long long)image_ready.image.image_id))); if (rollback_result != GCS_BLOCK_PCM_X_IMAGE_REARMED && rollback_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) cluster_pcm_x_runtime_fail_closed(); } +static void gcs_block_pcm_x_revoke_refusal_note_exact_diagnosed( + const char *site, int own_result, const ClusterPcmOwnSnapshot *snap, const PcmXTicketRef *ref, + uint64 image_id, uint64 source_generation, const ClusterPcmOwnFinishRefusal *finish_refusal); +static void +gcs_block_pcm_x_revoke_refusal_note_work(const char *site, int own_result, + const ClusterPcmOwnSnapshot *snap, + const GcsBlockPcmXImageWork *work) +{ + gcs_block_pcm_x_revoke_refusal_note_exact( + site, own_result, snap, work != NULL ? &work->binding.identity.ref : NULL, + work != NULL ? work->binding.identity.image.image_id : 0, + work != NULL ? work->binding.identity.image.source_own_generation : 0); +} + +static void +gcs_block_pcm_x_revoke_refusal_note_finish_work(int own_result, const ClusterPcmOwnSnapshot *snap, + const GcsBlockPcmXImageWork *work, + const ClusterPcmOwnFinishRefusal *finish_refusal) +{ + const char *site = "materialized-finish-other"; + + if (finish_refusal != NULL) { + if (finish_refusal->reason == CLUSTER_PCM_OWN_FINISH_REFUSAL_VM_FSM_PINNED) + site = "materialized-finish-vm-fsm-pinned"; + else if (finish_refusal->reason == CLUSTER_PCM_OWN_FINISH_REFUSAL_IO_IN_PROGRESS) + site = "materialized-finish-io-in-progress"; + else if (finish_refusal->reason == CLUSTER_PCM_OWN_FINISH_REFUSAL_LIVE_FLAGS) + site = "materialized-finish-live-flags"; + } + gcs_block_pcm_x_revoke_refusal_note_exact_diagnosed( + site, own_result, snap, work != NULL ? &work->binding.identity.ref : NULL, + work != NULL ? work->binding.identity.image.image_id : 0, + work != NULL ? work->binding.identity.image.source_own_generation : 0, finish_refusal); +} + +static void +gcs_block_pcm_x_revoke_refusal_note_source_prepare_work(int own_result, + const ClusterPcmOwnSnapshot *snap, + const GcsBlockPcmXImageWork *work, + ClusterPcmOwnSourcePrepareRefusal refusal) +{ + const char *site = "materialize-begin-s-other"; + + if (refusal == CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_BEGIN_REVOKE) + site = "materialize-begin-s-revoke"; + else if (refusal == CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_CONTENT_LOCK) + site = "materialize-begin-s-content-lock"; + else if (refusal == CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_DIRTY_FLUSHED) + site = "materialize-begin-s-dirty-flushed"; + else if (refusal == CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_DIRTY_RACED) + site = "materialize-begin-s-dirty-raced"; + else if (refusal == CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_IO_IN_PROGRESS) + site = "materialize-begin-s-io-in-progress"; + gcs_block_pcm_x_revoke_refusal_note_work(site, own_result, snap, work); +} + + +static void +gcs_block_pcm_x_image_ready_arm_refusal_note_work(int result, const GcsBlockPcmXImageWork *work, + PcmXLocalImageReadyRefusal refusal) +{ + const char *site = "image-ready-arm-other"; + + if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_NONE) { + gcs_block_pcm_x_revoke_refusal_note_exact(NULL, 0, NULL, NULL, 0, 0); + return; + } + if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_INVALID) + site = "image-ready-arm-invalid"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_RUNTIME) + site = "image-ready-arm-runtime"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_DIRECTORY) + site = "image-ready-arm-directory"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_TAG_SLOT) + site = "image-ready-arm-tag-slot"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_LOCAL_GATE) + site = "image-ready-arm-local-gate"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_RUNTIME_RECHECK) + site = "image-ready-arm-runtime-recheck"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_IDENTITY) + site = "image-ready-arm-identity"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_ACTIVE_HOLDER) + site = "image-ready-arm-active-holder"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_RELIABLE_LEG) + site = "image-ready-arm-reliable-leg"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_BAD_PHASE) + site = "image-ready-arm-bad-phase"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_COUNTER) + site = "image-ready-arm-counter"; + else if (refusal == PCM_X_LOCAL_IMAGE_READY_REFUSAL_GATE_RELEASE) + site = "image-ready-arm-gate-release"; + gcs_block_pcm_x_revoke_refusal_note_work(site, result, NULL, work); +} + +/* FlushBuffer may ERROR after immutable bytes have been staged but before the + * ownership commit. bufmgr first releases its content lock and raw pin. The + * A-record and exact REVOKING token are now recovery evidence across an + * irreversible boundary: validate them, trip the runtime fuse, and never run + * the pre-materialization rollback helper from this catch. The DATA worker + * absorbs the ERROR after preserving it: an aux-process rethrow would turn the + * local I/O failure into a postmaster-wide child crash. */ +static ClusterPcmOwnResult +gcs_block_pcm_x_finish_revoke_retain(int worker_id, const GcsBlockPcmXImageWork *work, + const GcsBlockPcmXImageBinding *binding, BufferDesc *buf, + const ClusterPcmOwnSnapshot *revoking, XLogRecPtr page_lsn, + ClusterPcmOwnSnapshot *retained, + ClusterPcmOwnFinishRefusal *finish_refusal) +{ + MemoryContext error_context = CurrentMemoryContext; + volatile ClusterPcmOwnResult result = CLUSTER_PCM_OWN_INVALID; + + PG_TRY(); + { + result = cluster_bufmgr_pcm_own_finish_revoke_retain(buf, revoking, page_lsn, retained, + finish_refusal); + } + PG_CATCH(); + { + ErrorData *original_error; + GcsBlockPcmXImageResult preserve_result; + + MemoryContextSwitchTo(error_context); + original_error = CopyErrorData(); + FlushErrorState(); + preserve_result = cluster_gcs_block_dedup_pcm_x_preserve_finish_error_exact( + worker_id, &work->key, &work->tag, binding, revoking->reservation_token, + revoking->pcm_state); + if (preserve_result != GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING) + gcs_block_pcm_x_revoke_refusal_note_work("finish-error-evidence", (int)preserve_result, + revoking, work); + ereport( + LOG, + (errmsg_internal("PCM-X finish-error evidence exact"), + errdetail("preserve_result=%d retained=%s worker=%d tag=%u/%u/%u/%d/%u " + "requester=%u backend=%d request_id=%llu ticket=%llu " + "queue_generation=%llu grant_generation=%llu image_id=%llu " + "reservation_token=%llu source_state=%u", + (int)preserve_result, + preserve_result == GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING ? "true" : "false", + worker_id, work->tag.spcOid, work->tag.dbOid, work->tag.relNumber, + (int)work->tag.forkNum, work->tag.blockNum, work->key.origin_node_id, + work->key.requester_backend_id, + (unsigned long long)work->binding.identity.ref.identity.request_id, + (unsigned long long)work->binding.identity.ref.handle.ticket_id, + (unsigned long long)work->binding.identity.ref.handle.queue_generation, + (unsigned long long)work->binding.identity.ref.grant_generation, + (unsigned long long)work->binding.identity.image.image_id, + (unsigned long long)revoking->reservation_token, + (unsigned)revoking->pcm_state))); + cluster_pcm_x_runtime_fail_closed(); + ereport(LOG, (errmsg_internal("cluster PCM-X retained-image finish FlushBuffer failed; " + "preserved immutable evidence and blocked recovery: %s", + original_error->message != NULL ? original_error->message + : "(no message)"))); + FreeErrorData(original_error); + result = CLUSTER_PCM_OWN_CORRUPT; + } + PG_END_TRY(); + return (ClusterPcmOwnResult)result; +} + + +/* Retry only the reversible lock-acquisition part after an immutable A-record + * exists. The by-value work token carries the original generation, revoke + * token, source state, tag, ticket and image identity, so a descriptor ABA or + * a different transfer can never inherit the staged bytes. */ +static void +gcs_block_pcm_x_finish_materialized_work(int worker_id, const GcsBlockPcmXImageWork *work) +{ + ClusterPcmOwnSnapshot current; + ClusterPcmOwnSnapshot retained; + ClusterPcmOwnFinishRefusal finish_refusal; + ClusterPcmOwnResult own_result; + ClusterPcmXRevokeFinishMode finish_mode; + GcsBlockPcmXImageResult image_result; + GcsBlockPcmXImageWork ready_work; + BufferDesc *buf; + bool descriptor_retained; + int buffer_id; + + if (work->reservation_token == 0 + || (work->source_pcm_state != (uint8)PCM_STATE_S + && work->source_pcm_state != (uint8)PCM_STATE_X)) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + own_result = cluster_bufmgr_pcm_own_snapshot_by_tag(&work->tag, &buffer_id, ¤t); + gcs_block_pcm_x_note_own_result(own_result); + if (own_result == CLUSTER_PCM_OWN_NOT_READY) { + gcs_block_pcm_x_revoke_refusal_note_work("materialized-snapshot", (int)own_result, ¤t, + work); + return; + } + if (own_result != CLUSTER_PCM_OWN_OK || !BufferTagsEqual(¤t.tag, &work->tag) + || current.generation != work->binding.identity.image.source_own_generation + || current.reservation_token != work->reservation_token + || current.flags != PCM_OWN_FLAG_REVOKING || current.pcm_state != work->source_pcm_state) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + + buf = GetBufferDescriptor(buffer_id); + own_result = gcs_block_pcm_x_finish_revoke_retain( + worker_id, work, &work->binding, buf, ¤t, + (XLogRecPtr)work->binding.identity.image.page_lsn, &retained, &finish_refusal); + gcs_block_pcm_x_note_own_result(own_result); + if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY) { + gcs_block_pcm_x_revoke_refusal_note_finish_work((int)own_result, ¤t, work, + &finish_refusal); + return; + } + if (own_result != CLUSTER_PCM_OWN_OK) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + + finish_mode = cluster_pcm_x_revoke_finish_mode(¤t.tag, 0); + descriptor_retained = finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_RETAIN; + if (current.generation == UINT64_MAX || retained.generation != current.generation + 1 + || retained.reservation_token != current.reservation_token + || (descriptor_retained ? retained.flags != PCM_OWN_FLAG_REVOKING : retained.flags != 0) + || retained.pcm_state != (uint8)PCM_STATE_N + || (finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_RETAIN + && finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_DROP) + || !BufferTagsEqual(&retained.tag, ¤t.tag)) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + image_result = cluster_gcs_block_dedup_pcm_x_publish_ready_exact(worker_id, &work->key, + &work->tag, &work->binding); + if (image_result != GCS_BLOCK_PCM_X_IMAGE_STORED + && image_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + ready_work = *work; + ready_work.reservation_token = 0; + ready_work.source_pcm_state = 0; + ready_work.entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE; + gcs_block_pcm_x_revoke_refusal_note_exact(NULL, 0, NULL, NULL, 0, 0); + gcs_block_pcm_x_stage_ready_work(worker_id, &ready_work); +} + static void gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImageWork *work) { @@ -10999,12 +12495,16 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage ClusterPcmOwnSnapshot current; ClusterPcmOwnSnapshot retained; ClusterPcmOwnSnapshot revoking; + ClusterPcmOwnFinishRefusal finish_refusal; + ClusterPcmOwnSourcePrepareRefusal source_prepare_refusal + = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_NONE; GcsBlockPcmXImageBinding ready_binding; GcsBlockPcmXImageResult image_result; GcsBlockReplyHeader reply_header; PcmXLocalHolderSnapshot holder_snapshot; PcmXQueueResult holder_result; ClusterPcmOwnResult own_result; + volatile ClusterPcmOwnResult n_prepare_result = CLUSTER_PCM_OWN_INVALID; ClusterPcmXRevokeFinishMode finish_mode; PageHeaderData page_header; BufferDesc *buf; @@ -11013,15 +12513,22 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage bool source_is_n; bool source_is_s; bool source_is_x; + volatile bool copy_ok = false; bool descriptor_retained; bool self_source_handoff; int buffer_id; holder_result = cluster_pcm_x_local_holder_snapshot(&work->tag, NULL, 0, &holder_snapshot); - if (holder_result == PCM_X_QUEUE_NO_CAPACITY && holder_snapshot.holder_count > 0) + if (holder_result == PCM_X_QUEUE_NO_CAPACITY && holder_snapshot.holder_count > 0) { + gcs_block_pcm_x_revoke_refusal_note_work("materialize-holder-capacity", (int)holder_result, + NULL, work); return; - if (holder_result == PCM_X_QUEUE_NOT_READY || holder_result == PCM_X_QUEUE_BUSY) + } + if (holder_result == PCM_X_QUEUE_NOT_READY || holder_result == PCM_X_QUEUE_BUSY) { + gcs_block_pcm_x_revoke_refusal_note_work("materialize-holder-snapshot", (int)holder_result, + NULL, work); return; + } if (holder_result != PCM_X_QUEUE_OK || holder_snapshot.holder_count != 0) { cluster_pcm_x_runtime_fail_closed(); return; @@ -11029,8 +12536,11 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage own_result = cluster_bufmgr_pcm_own_snapshot_by_tag(&work->tag, &buffer_id, ¤t); gcs_block_pcm_x_note_own_result(own_result); - if (own_result == CLUSTER_PCM_OWN_NOT_READY) + if (own_result == CLUSTER_PCM_OWN_NOT_READY) { + gcs_block_pcm_x_revoke_refusal_note_work("materialize-snapshot", (int)own_result, ¤t, + work); return; + } source_is_n = current.pcm_state == (uint8)PCM_STATE_N; source_is_s = current.pcm_state == (uint8)PCM_STATE_S; source_is_x = current.pcm_state == (uint8)PCM_STATE_X; @@ -11044,30 +12554,77 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage return; } buf = GetBufferDescriptor(buffer_id); - if (source_is_n) - own_result = cluster_bufmgr_pcm_own_prepare_n_source_image( - buf, ¤t, &revoking, block_data, &page_lsn, &page_scn); - else if (source_is_s) - own_result = cluster_bufmgr_pcm_own_begin_s_revoke(buf, ¤t, &revoking); - else + if (source_is_n) { + PG_TRY(); + { + n_prepare_result = cluster_bufmgr_pcm_own_prepare_n_source_image( + buf, ¤t, &revoking, block_data, &page_lsn, &page_scn); + } + PG_CATCH(); + { + if (!gcs_block_pcm_x_release_image_exact(worker_id, work, &work->binding)) + cluster_pcm_x_runtime_fail_closed(); + PG_RE_THROW(); + } + PG_END_TRY(); + own_result = (ClusterPcmOwnResult)n_prepare_result; + } else if (source_is_s) { + PG_TRY(); + { + n_prepare_result = cluster_bufmgr_pcm_own_prepare_s_source_image( + buf, ¤t, (SCN)work->binding.required_page_scn, &revoking, block_data, + &page_lsn, &page_scn, &source_prepare_refusal); + } + PG_CATCH(); + { + if (!gcs_block_pcm_x_release_image_exact(worker_id, work, &work->binding)) + cluster_pcm_x_runtime_fail_closed(); + PG_RE_THROW(); + } + PG_END_TRY(); + own_result = (ClusterPcmOwnResult)n_prepare_result; + } else own_result = cluster_bufmgr_pcm_own_begin_x_revoke(buf, ¤t, &revoking); gcs_block_pcm_x_note_own_result(own_result); - if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY) + if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY + || own_result == CLUSTER_PCM_OWN_STALE) { + if (source_is_s) + gcs_block_pcm_x_revoke_refusal_note_source_prepare_work((int)own_result, ¤t, work, + source_prepare_refusal); + else + gcs_block_pcm_x_revoke_refusal_note_work("materialize-begin", (int)own_result, ¤t, + work); return; + } if (own_result != CLUSTER_PCM_OWN_OK) { (void)gcs_block_pcm_x_release_image_exact(worker_id, work, &work->binding); cluster_pcm_x_runtime_fail_closed(); return; } - if (!source_is_n && !cluster_bufmgr_copy_block_for_gcs(work->tag, &page_lsn, block_data)) { + if (source_is_x) { + PG_TRY(); + { + copy_ok = cluster_bufmgr_copy_block_for_gcs(work->tag, &page_lsn, block_data, NULL); + } + PG_CATCH(); + { + gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &work->binding, buf, + &revoking, true); + PG_RE_THROW(); + } + PG_END_TRY(); + } + if (source_is_x && !copy_ok) { + gcs_block_pcm_x_revoke_refusal_note_work("materialize-copy", (int)CLUSTER_PCM_OWN_BUSY, + &revoking, work); gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &work->binding, buf, &revoking, true); return; } memcpy(&page_header, block_data, sizeof(page_header)); if ((uint64)page_lsn != (uint64)PageXLogRecPtrGet(page_header.pd_lsn) - || (source_is_n && page_scn != (uint64)page_header.pd_block_scn)) { + || ((source_is_n || source_is_s) && page_scn != (uint64)page_header.pd_block_scn)) { gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &work->binding, buf, &revoking, true); return; @@ -11089,7 +12646,8 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage reply_header.status = (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER; GcsBlockReplyHeaderSetForwardingMasterNode(&reply_header, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); image_result = cluster_gcs_block_dedup_pcm_x_materialize( - worker_id, &work->key, &work->tag, &ready_binding, &reply_header, block_data); + worker_id, &work->key, &work->tag, &ready_binding, revoking.reservation_token, + revoking.pcm_state, &reply_header, block_data); if (image_result != GCS_BLOCK_PCM_X_IMAGE_STORED && image_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) { gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &work->binding, buf, &revoking, @@ -11115,12 +12673,18 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage return; } } else { - own_result - = cluster_bufmgr_pcm_own_finish_revoke_retain(buf, &revoking, page_lsn, &retained); + own_result = gcs_block_pcm_x_finish_revoke_retain( + worker_id, work, &ready_binding, buf, &revoking, page_lsn, &retained, &finish_refusal); gcs_block_pcm_x_note_own_result(own_result); + if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY) { + gcs_block_pcm_x_revoke_refusal_note_work("materialize-finish", (int)own_result, + &revoking, work); + return; + } if (own_result != CLUSTER_PCM_OWN_OK) { - gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &ready_binding, buf, - &revoking, true); + /* Immutable bytes exist: do not erase the A-record or exact revoke + * evidence on a failed irreversible-boundary check. */ + cluster_pcm_x_runtime_fail_closed(); return; } finish_mode = cluster_pcm_x_revoke_finish_mode(&revoking.tag, 0); @@ -11158,6 +12722,7 @@ gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImage ready_work.binding = ready_binding; ready_work.entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE; + gcs_block_pcm_x_revoke_refusal_note_exact(NULL, 0, NULL, NULL, 0, 0); gcs_block_pcm_x_stage_ready_work(worker_id, &ready_work); } } @@ -11201,7 +12766,8 @@ cluster_gcs_block_pcm_x_image_pump_tick(int worker_id) result = cluster_gcs_block_dedup_pcm_x_next_work(worker_id, &work); if (result == GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND || result == GCS_BLOCK_PCM_X_IMAGE_FULL) return; - if (result != GCS_BLOCK_PCM_X_IMAGE_RESERVED && result != GCS_BLOCK_PCM_X_IMAGE_REPLAY) { + if (result != GCS_BLOCK_PCM_X_IMAGE_RESERVED && result != GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING + && result != GCS_BLOCK_PCM_X_IMAGE_REPLAY) { cluster_pcm_x_runtime_fail_closed(); return; } @@ -11211,6 +12777,8 @@ cluster_gcs_block_pcm_x_image_pump_tick(int worker_id) } if (result == GCS_BLOCK_PCM_X_IMAGE_RESERVED) gcs_block_pcm_x_materialize_reserved_work(worker_id, &work); + else if (result == GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING) + gcs_block_pcm_x_finish_materialized_work(worker_id, &work); else gcs_block_pcm_x_stage_ready_work(worker_id, &work); } @@ -11219,9 +12787,124 @@ cluster_gcs_block_pcm_x_image_pump_tick(int worker_id) /* 49 only installs the holder-side revoke ledger and wakes exact registered * holders. It never copies a page, drops ownership, or synthesizes type 50 * inside the IC handler. */ +/* Log-once evidence for a repeating source-side REVOKE refusal: the live + * descriptor lifecycle blocking the transfer leg must name itself, because + * every refusal arm here is silent and the master keeps re-sending forever. + * A NULL site resets the streak (the revoke made progress). */ +static void +gcs_block_pcm_x_revoke_refusal_note_exact(const char *site, int own_result, + const ClusterPcmOwnSnapshot *snap, + const PcmXTicketRef *ref, uint64 image_id, + uint64 source_generation) +{ + gcs_block_pcm_x_revoke_refusal_note_exact_diagnosed(site, own_result, snap, ref, image_id, + source_generation, NULL); +} + +static void +gcs_block_pcm_x_revoke_refusal_note_exact_diagnosed( + const char *site, int own_result, const ClusterPcmOwnSnapshot *snap, const PcmXTicketRef *ref, + uint64 image_id, uint64 source_generation, const ClusterPcmOwnFinishRefusal *finish_refusal) +{ + static const char *last_site = NULL; + static uint32 last_flags = 0; + static uint64 last_request = 0; + static uint64 last_ticket = 0; + static uint64 last_image = 0; + static uint64 last_source_generation = 0; + static BufferTag last_tag; + static bool last_tag_valid = false; + static int last_result = 0; + static ClusterPcmOwnFinishRefusalReason last_finish_refusal + = CLUSTER_PCM_OWN_FINISH_REFUSAL_NONE; + static uint32 last_shared_refcount = 0; + static uint32 last_live_flags = 0; + static uint64 last_live_token = 0; + static bool last_bm_io_in_progress = false; + static bool logged = false; + uint32 flags = snap != NULL ? snap->flags : 0; + ClusterPcmOwnFinishRefusalReason finish_reason + = finish_refusal != NULL ? finish_refusal->reason : CLUSTER_PCM_OWN_FINISH_REFUSAL_NONE; + uint32 shared_refcount = finish_refusal != NULL ? finish_refusal->shared_refcount : 0; + uint32 live_flags = finish_refusal != NULL ? finish_refusal->live_flags : 0; + uint64 live_token = finish_refusal != NULL ? finish_refusal->live_token : 0; + bool bm_io_in_progress = finish_refusal != NULL && finish_refusal->bm_io_in_progress; + const BufferTag *tag = ref != NULL ? &ref->identity.tag : snap != NULL ? &snap->tag : NULL; + uint64 request_id = ref != NULL ? ref->identity.request_id : 0; + uint64 ticket_id = ref != NULL ? ref->handle.ticket_id : 0; + bool same_tag + = tag == NULL ? !last_tag_valid : last_tag_valid && BufferTagsEqual(tag, &last_tag); + + if (site == NULL) { + last_site = NULL; + last_tag_valid = false; + logged = false; + return; + } + if (site == last_site && own_result == last_result && flags == last_flags + && request_id == last_request && ticket_id == last_ticket && image_id == last_image + && source_generation == last_source_generation && finish_reason == last_finish_refusal + && shared_refcount == last_shared_refcount && live_flags == last_live_flags + && live_token == last_live_token && bm_io_in_progress == last_bm_io_in_progress + && same_tag) { + if (!logged) { + logged = true; + if (tag != NULL) + ereport( + LOG, + (errmsg("PCM-X source revoke is repeating a refusal at %s", site), + errdetail("result=%d epoch=%llu tag=%u/%u/%u/%d/%u " + "requester=%d procno=%u xid=%u request_id=%llu wait_seq=%llu " + "ticket=%llu queue_generation=%llu grant_generation=%llu " + "image_id=%llu source_generation=%llu pcm_state=%u " + "own_generation=%llu token=%llu own_flags=%u " + "finish_refusal=%u shared_refcount=%u " + "bm_io_in_progress=%s live_flags=%u live_token=%llu", + own_result, + (unsigned long long)(ref != NULL ? ref->identity.cluster_epoch : 0), + tag->spcOid, tag->dbOid, tag->relNumber, (int)tag->forkNum, + tag->blockNum, ref != NULL ? ref->identity.node_id : -1, + ref != NULL ? ref->identity.procno : 0, + ref != NULL ? ref->identity.xid : 0, (unsigned long long)request_id, + (unsigned long long)(ref != NULL ? ref->identity.wait_seq : 0), + (unsigned long long)ticket_id, + (unsigned long long)(ref != NULL ? ref->handle.queue_generation : 0), + (unsigned long long)(ref != NULL ? ref->grant_generation : 0), + (unsigned long long)image_id, (unsigned long long)source_generation, + (unsigned int)(snap != NULL ? snap->pcm_state : 0), + (unsigned long long)(snap != NULL ? snap->generation : 0), + (unsigned long long)(snap != NULL ? snap->reservation_token : 0), + (unsigned int)flags, (unsigned int)finish_reason, + (unsigned int)shared_refcount, bm_io_in_progress ? "true" : "false", + (unsigned int)live_flags, (unsigned long long)live_token))); + } + return; + } + last_site = site; + last_result = own_result; + last_flags = flags; + last_request = request_id; + last_ticket = ticket_id; + last_image = image_id; + last_source_generation = source_generation; + last_finish_refusal = finish_reason; + last_shared_refcount = shared_refcount; + last_live_flags = live_flags; + last_live_token = live_token; + last_bm_io_in_progress = bm_io_in_progress; + if (tag != NULL) { + last_tag = *tag; + last_tag_valid = true; + } else + last_tag_valid = false; + logged = false; +} + + static void cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const void *payload) { + PcmXRevokePayload revoke_frame; const PcmXRevokePayload *revoke; ClusterPcmOwnSnapshot own_snapshot; GcsBlockDedupKey image_key; @@ -11231,6 +12914,7 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi PcmXQueueResult progress_result; PcmXQueueResult result; uint64 current_epoch; + uint64 required_page_scn = 0; uint64 source_session; uint64 source_generation = 0; int buffer_id; @@ -11240,18 +12924,33 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi bool have_source_generation = false; bool new_reservation = false; - if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXRevokePayload) + if (env == NULL || payload == NULL + || (env->payload_length != sizeof(PcmXRevokePayload) + && env->payload_length != sizeof(PcmXRevokePayloadV2)) || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) return; source_node = (int32)env->source_node_id; - revoke = (const PcmXRevokePayload *)payload; + memset(&revoke_frame, 0, sizeof(revoke_frame)); + memcpy(&revoke_frame, payload, sizeof(revoke_frame)); + revoke = &revoke_frame; + if (env->payload_length == sizeof(PcmXRevokePayloadV2)) { + const PcmXRevokePayloadV2 *revoke_v2 = (const PcmXRevokePayloadV2 *)payload; + + if (source_node != cluster_node_id + && !cluster_sf_peer_supports_pcm_x_source_floor(source_node)) + return; + required_page_scn = revoke_v2->required_page_scn; + } current_epoch = cluster_epoch_get_current(); tag_master = cluster_gcs_lookup_master(revoke->ref.identity.tag); if (!cluster_gcs_pcm_x_revoke_ingress_valid(revoke, env->payload_length, source_node, current_epoch, tag_master, cluster_node_id) || !gcs_block_pcm_x_transfer_ingress_authorized(&revoke->ref.identity.tag, source_node, - current_epoch, &source_session)) + current_epoch, &source_session)) { + gcs_block_pcm_x_revoke_refusal_note_exact("ingress-auth", 0, NULL, &revoke->ref, + revoke->image_id, 0); return; + } if (!gcs_block_pcm_x_handler_tag_shard_exact(&revoke->ref.identity.tag)) return; worker_id = cluster_ic_tier1_my_data_channel(); @@ -11270,6 +12969,9 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi if (!gcs_block_pcm_x_ticket_ref_equal(&holder_progress.ref, &revoke->ref) || holder_progress.image.image_id != revoke->image_id) { cluster_pcm_x_stats_note_queue_result(PCM_X_QUEUE_STALE); + gcs_block_pcm_x_revoke_refusal_note_exact("holder-ledger-stale", (int)PCM_X_QUEUE_STALE, + NULL, &revoke->ref, revoke->image_id, + holder_progress.image.source_own_generation); return; } if ((holder_progress.flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0) { @@ -11295,6 +12997,8 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi } } else if (progress_result != PCM_X_QUEUE_NOT_FOUND) { cluster_pcm_x_stats_note_queue_result(progress_result); + gcs_block_pcm_x_revoke_refusal_note_exact("holder-progress", (int)progress_result, NULL, + &revoke->ref, revoke->image_id, 0); return; } @@ -11311,16 +13015,25 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi source_is_s = own_snapshot.pcm_state == (uint8)PCM_STATE_S; source_is_x = own_snapshot.pcm_state == (uint8)PCM_STATE_X; if (own_result != CLUSTER_PCM_OWN_OK || (!source_is_n && !source_is_s && !source_is_x) - || own_snapshot.flags != 0) + || own_snapshot.flags != 0) { + gcs_block_pcm_x_revoke_refusal_note_exact("ingress-snapshot", (int)own_result, + &own_snapshot, &revoke->ref, revoke->image_id, + own_snapshot.generation); return; + } source_generation = own_snapshot.generation; } - gcs_block_pcm_x_reserved_binding(revoke, source_generation, source_session, &reserved_binding); + gcs_block_pcm_x_reserved_binding(revoke, source_generation, source_session, required_page_scn, + &reserved_binding); image_result = cluster_gcs_block_dedup_pcm_x_reserve( worker_id, &image_key, &revoke->ref.identity.tag, &reserved_binding); if (image_result != GCS_BLOCK_PCM_X_IMAGE_RESERVED - && image_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) + && image_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) { + gcs_block_pcm_x_revoke_refusal_note_exact("image-reserve", (int)image_result, NULL, + &revoke->ref, revoke->image_id, + source_generation); return; + } new_reservation = image_result == GCS_BLOCK_PCM_X_IMAGE_RESERVED; if (!new_reservation) { image_result = cluster_gcs_block_dedup_pcm_x_rearm_exact( @@ -11332,15 +13045,22 @@ cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const voi return; } } - result = cluster_pcm_x_local_holder_revoke_apply_exact(revoke, source_node, source_session); + result = cluster_pcm_x_local_holder_revoke_apply_floor_exact(revoke, required_page_scn, + source_node, source_session); cluster_pcm_x_stats_note_queue_result(result); - if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + /* Ingress only installs/re-arms holder work. It does not prove the + * materializer advanced, so only a READY image resets the refusal streak. */ gcs_block_pcm_x_wake_registered_holders(&revoke->ref.identity.tag); - else if (new_reservation - && cluster_gcs_block_dedup_pcm_x_release_exact( - worker_id, &image_key, &revoke->ref.identity.tag, &reserved_binding) - != GCS_BLOCK_PCM_X_IMAGE_RELEASED) - cluster_pcm_x_runtime_fail_closed(); + } else { + gcs_block_pcm_x_revoke_refusal_note_exact("apply", (int)result, NULL, &revoke->ref, + revoke->image_id, source_generation); + if (new_reservation + && cluster_gcs_block_dedup_pcm_x_release_exact( + worker_id, &image_key, &revoke->ref.identity.tag, &reserved_binding, -1) + != GCS_BLOCK_PCM_X_IMAGE_RELEASED) + cluster_pcm_x_runtime_fail_closed(); + } } @@ -11349,11 +13069,17 @@ cluster_gcs_handle_pcm_x_image_ready_envelope(const ClusterICEnvelope *env, cons { const PcmXGrantPayload *image_ready; PcmXGrantPayload prepare; + GcsLostWriteVerdict floor_verdict; PcmXQueueResult result; + SCN required_page_scn; uint64 current_epoch; uint64 source_session; int32 source_node; int32 tag_master; + bool authorized; + bool diagnostic; + bool prepare_stage_result = false; + bool wire_valid; if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXGrantPayload) || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) @@ -11362,18 +13088,67 @@ cluster_gcs_handle_pcm_x_image_ready_envelope(const ClusterICEnvelope *env, cons image_ready = (const PcmXGrantPayload *)payload; current_epoch = cluster_epoch_get_current(); tag_master = cluster_gcs_lookup_master(image_ready->ref.identity.tag); - if (!cluster_gcs_pcm_x_image_ready_ingress_valid(image_ready, env->payload_length, source_node, - current_epoch, tag_master, cluster_node_id) - || !gcs_block_pcm_x_transfer_ingress_authorized(&image_ready->ref.identity.tag, source_node, - current_epoch, &source_session)) + diagnostic = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); + wire_valid = cluster_gcs_pcm_x_image_ready_ingress_valid( + image_ready, env->payload_length, source_node, current_epoch, tag_master, cluster_node_id); + authorized = wire_valid + && gcs_block_pcm_x_transfer_ingress_authorized( + &image_ready->ref.identity.tag, source_node, current_epoch, &source_session); + if (diagnostic) + ereport(LOG, (errmsg_internal("PCM-X IMAGE_READY master boundary: ingress"), + errdetail("wire_valid=%s authorized=%s source=%d local=%d tag_master=%d " + "epoch=%llu source_session=%llu request_id=%llu ticket=%llu " + "grant_generation=%llu image_source=%d image_id=%llu", + wire_valid ? "true" : "false", authorized ? "true" : "false", + source_node, cluster_node_id, tag_master, + (unsigned long long)current_epoch, + (unsigned long long)(authorized ? source_session : 0), + (unsigned long long)image_ready->ref.identity.request_id, + (unsigned long long)image_ready->ref.handle.ticket_id, + (unsigned long long)image_ready->ref.grant_generation, + image_ready->image.source_node, + (unsigned long long)image_ready->image.image_id))); + if (!wire_valid || !authorized) return; + required_page_scn = cluster_pcm_lock_pi_watermark_scn_query(image_ready->ref.identity.tag); + floor_verdict + = gcs_block_lost_write_verdict(required_page_scn, (SCN)image_ready->image.page_scn); + if (floor_verdict == GCS_LOST_WRITE_FAIL_STALE + || floor_verdict == GCS_LOST_WRITE_FAIL_ANOMALY) { + ereport(LOG, (errmsg_internal("PCM-X IMAGE_READY source stale before PREPARE: " + "source=%d request=%llu ticket=%llu image=%llu " + "image_scn=%llu required_scn=%llu verdict=%d", + source_node, + (unsigned long long)image_ready->ref.identity.request_id, + (unsigned long long)image_ready->ref.handle.ticket_id, + (unsigned long long)image_ready->image.image_id, + (unsigned long long)image_ready->image.page_scn, + (unsigned long long)required_page_scn, (int)floor_verdict))); + cluster_pcm_x_runtime_fail_closed(); + return; + } result = cluster_pcm_x_master_image_ready_exact(image_ready, source_node, source_session, &prepare); cluster_pcm_x_stats_note_queue_result(result); - if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) - (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, - prepare.ref.identity.node_id, &prepare, - sizeof(prepare)); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + gcs_block_pcm_x_revoke_refusal_note_exact(NULL, 0, NULL, NULL, 0, 0); + prepare_stage_result = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, + prepare.ref.identity.node_id, &prepare, + sizeof(prepare)); + } else + gcs_block_pcm_x_revoke_refusal_note_exact("image-ready-master-consume", (int)result, NULL, + &image_ready->ref, image_ready->image.image_id, + image_ready->image.source_own_generation); + if (diagnostic) + ereport(LOG, (errmsg_internal("PCM-X IMAGE_READY master boundary: consume"), + errdetail("result=%d prepare_stage_result=%s requester=%d source=%d " + "request_id=%llu ticket=%llu grant_generation=%llu image_id=%llu", + (int)result, prepare_stage_result ? "true" : "false", + image_ready->ref.identity.node_id, source_node, + (unsigned long long)image_ready->ref.identity.request_id, + (unsigned long long)image_ready->ref.handle.ticket_id, + (unsigned long long)image_ready->ref.grant_generation, + (unsigned long long)image_ready->image.image_id))); } @@ -11382,11 +13157,15 @@ cluster_gcs_handle_pcm_x_prepare_grant_envelope(const ClusterICEnvelope *env, co { const PcmXGrantPayload *prepare; PcmXLocalHandle leader; + PcmXQueueResult lookup_result; PcmXQueueResult result; uint64 current_epoch; uint64 source_session; int32 source_node; int32 tag_master; + bool authorized; + bool diagnostic; + bool wire_valid; if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXGrantPayload) || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) @@ -11395,16 +13174,51 @@ cluster_gcs_handle_pcm_x_prepare_grant_envelope(const ClusterICEnvelope *env, co prepare = (const PcmXGrantPayload *)payload; current_epoch = cluster_epoch_get_current(); tag_master = cluster_gcs_lookup_master(prepare->ref.identity.tag); - if (!cluster_gcs_pcm_x_prepare_grant_ingress_valid(prepare, env->payload_length, source_node, - current_epoch, tag_master, cluster_node_id) - || !gcs_block_pcm_x_transfer_ingress_authorized(&prepare->ref.identity.tag, source_node, - current_epoch, &source_session)) + diagnostic = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); + wire_valid = cluster_gcs_pcm_x_prepare_grant_ingress_valid( + prepare, env->payload_length, source_node, current_epoch, tag_master, cluster_node_id); + authorized = wire_valid + && gcs_block_pcm_x_transfer_ingress_authorized( + &prepare->ref.identity.tag, source_node, current_epoch, &source_session); + if (diagnostic) + ereport(LOG, (errmsg_internal("PCM-X PREPARE_GRANT requester boundary: ingress"), + errdetail("wire_valid=%s authorized=%s source=%d local=%d tag_master=%d " + "epoch=%llu source_session=%llu request_id=%llu ticket=%llu " + "grant_generation=%llu image_id=%llu", + wire_valid ? "true" : "false", authorized ? "true" : "false", + source_node, cluster_node_id, tag_master, + (unsigned long long)current_epoch, + (unsigned long long)(authorized ? source_session : 0), + (unsigned long long)prepare->ref.identity.request_id, + (unsigned long long)prepare->ref.handle.ticket_id, + (unsigned long long)prepare->ref.grant_generation, + (unsigned long long)prepare->image.image_id))); + if (!wire_valid || !authorized) return; - result = cluster_pcm_x_local_lookup_exact(&prepare->ref.identity, &leader); - if (result != PCM_X_QUEUE_OK) + lookup_result = cluster_pcm_x_local_lookup_exact(&prepare->ref.identity, &leader); + if (lookup_result != PCM_X_QUEUE_OK) { + if (diagnostic) + ereport(LOG, (errmsg_internal("PCM-X PREPARE_GRANT requester boundary: apply"), + errdetail("lookup_result=%d apply_result=-1 source=%d request_id=%llu " + "ticket=%llu grant_generation=%llu image_id=%llu", + (int)lookup_result, source_node, + (unsigned long long)prepare->ref.identity.request_id, + (unsigned long long)prepare->ref.handle.ticket_id, + (unsigned long long)prepare->ref.grant_generation, + (unsigned long long)prepare->image.image_id))); return; + } result = cluster_pcm_x_local_prepare_grant_exact(&leader, prepare, source_node, source_session); cluster_pcm_x_stats_note_queue_result(result); + if (diagnostic) + ereport(LOG, (errmsg_internal("PCM-X PREPARE_GRANT requester boundary: apply"), + errdetail("lookup_result=%d apply_result=%d source=%d request_id=%llu " + "ticket=%llu grant_generation=%llu image_id=%llu", + (int)lookup_result, (int)result, source_node, + (unsigned long long)prepare->ref.identity.request_id, + (unsigned long long)prepare->ref.handle.ticket_id, + (unsigned long long)prepare->ref.grant_generation, + (unsigned long long)prepare->image.image_id))); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) gcs_block_pcm_x_wake_requester(&prepare->ref.identity); } @@ -11413,29 +13227,47 @@ cluster_gcs_handle_pcm_x_prepare_grant_envelope(const ClusterICEnvelope *env, co static void cluster_gcs_handle_pcm_x_install_ready_envelope(const ClusterICEnvelope *env, const void *payload) { - const PcmXInstallReadyPayload *install_ready; + PcmXInstallReadyPayload frame; PcmXPhasePayload commit; PcmXQueueResult result; uint64 current_epoch; uint64 source_session; int32 source_node; int32 tag_master; - - if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXInstallReadyPayload) + bool rebase_wire_active = false; + bool source_rebase_capable = false; + + /* Both exact lengths are legal: the V1 104-byte frame normalizes to a + * zero rebase, the V2 112-byte frame carries the published grant base. */ + if (env == NULL || payload == NULL + || (env->payload_length != sizeof(PcmXInstallReadyPayload) + && env->payload_length != PCM_X_INSTALL_READY_V1_LEN) || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) return; source_node = (int32)env->source_node_id; - install_ready = (const PcmXInstallReadyPayload *)payload; + memset(&frame, 0, sizeof(frame)); + memcpy(&frame, payload, env->payload_length); current_epoch = cluster_epoch_get_current(); - tag_master = cluster_gcs_lookup_master(install_ready->ref.identity.tag); - if (!cluster_gcs_pcm_x_install_ready_ingress_valid(install_ready, env->payload_length, - source_node, current_epoch, tag_master, - cluster_node_id) - || !gcs_block_pcm_x_transfer_ingress_authorized( - &install_ready->ref.identity.tag, source_node, current_epoch, &source_session)) + tag_master = cluster_gcs_lookup_master(frame.ref.identity.tag); + /* Receiver-side V2 admission: the sender-side coverage gate is not an + * ingress invariant, so a V2 frame re-proves BOTH the activated + * formation-wide coverage and the source's REBASE capability. A + * SELF-loopback frame (master == requester node) has no HELLO record — + * exactly like gcs_block_pcm_x_source_capable, its capability is the + * local binary itself; the coverage flag still applies. */ + if (env->payload_length == sizeof(frame)) { + rebase_wire_active = cluster_pcm_x_runtime_snapshot().rebase_wire_active; + source_rebase_capable + = source_node == cluster_node_id || cluster_sf_peer_supports_pcm_x_rebase(source_node); + } + if (!cluster_gcs_pcm_x_install_ready_ingress_valid(&frame, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id, + rebase_wire_active, source_rebase_capable) + || !gcs_block_pcm_x_transfer_ingress_authorized(&frame.ref.identity.tag, source_node, + current_epoch, &source_session)) return; - result = cluster_pcm_x_master_install_ready_exact(install_ready, source_node, source_session, - &commit); + result = cluster_pcm_x_master_install_ready_exact(&frame, frame.rebased_own_generation, + source_node, source_session, &commit); cluster_pcm_x_stats_note_queue_result(result); if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_COMMIT_X, @@ -11484,18 +13316,27 @@ cluster_gcs_handle_pcm_x_final_ack_envelope(const ClusterICEnvelope *env, const PcmXGrdHandoffToken handoff; PcmXGrdHandoffResult handoff_result; PcmXMasterFinalAckToken token; + ClusterPcmWmProv wm_prov; + SCN watermark_scn; + bool wm_have; PcmXPhasePayload final_commit; PcmXQueueResult result; uint64 current_epoch; uint64 source_session; int32 source_node; int32 tag_master; + const char *fail_stage = "entry"; + bool authority_valid = false; if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXFinalAckPayload) || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) return; source_node = (int32)env->source_node_id; final_ack = (const PcmXFinalAckPayload *)payload; + memset(&authority, 0, sizeof(authority)); + memset(&handoff, 0, sizeof(handoff)); + memset(&token, 0, sizeof(token)); + handoff_result = PCM_X_GRD_HANDOFF_INVALID; current_epoch = cluster_epoch_get_current(); tag_master = cluster_gcs_lookup_master(final_ack->ref.identity.tag); if (!cluster_gcs_pcm_x_final_ack_ingress_valid(final_ack, env->payload_length, source_node, @@ -11507,33 +13348,70 @@ cluster_gcs_handle_pcm_x_final_ack_envelope(const ClusterICEnvelope *env, const &token); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { - gcs_block_pcm_x_master_drive_fail_closed(result); - return; + fail_stage = "prepare"; + goto final_ack_fail_closed; } - if (!cluster_pcm_lock_authority_snapshot(final_ack->ref.identity.tag, &authority) - || !cluster_gcs_pcm_x_grd_handoff_token_build(&token, &authority, &handoff)) { - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); - return; + authority_valid = cluster_pcm_lock_authority_snapshot(final_ack->ref.identity.tag, &authority); + if (!authority_valid) { + fail_stage = "authority-snapshot"; + goto final_ack_fail_closed; + } + if (!cluster_gcs_pcm_x_grd_handoff_token_build(&token, &authority, &handoff)) { + fail_stage = "handoff-token"; + goto final_ack_fail_closed; } handoff_result = cluster_pcm_lock_queue_handoff_x_exact(&handoff); if (handoff_result != PCM_X_GRD_HANDOFF_OK && handoff_result != PCM_X_GRD_HANDOFF_DUPLICATE) { - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); - return; + fail_stage = "grd-handoff"; + goto final_ack_fail_closed; } result = cluster_pcm_x_master_final_ack_finalize_exact(&token, &final_commit); cluster_pcm_x_stats_note_queue_result(result); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { /* GRD already committed; retaining the engine/image evidence and * closing the runtime is the only 8.A-safe outcome. */ - (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, - PCM_X_RUNTIME_RECOVERY_BLOCKED); - return; + fail_stage = "finalize"; + goto final_ack_fail_closed; } (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK, final_commit.ref.identity.node_id, &final_commit, sizeof(final_commit)); + return; + +final_ack_fail_closed: + watermark_scn = cluster_pcm_lock_pi_watermark_scn_query(final_ack->ref.identity.tag); + wm_have = cluster_pcm_lock_pi_watermark_prov_query(final_ack->ref.identity.tag, &wm_prov); + ereport( + LOG, + (errmsg("PCM-X FINAL_ACK fail-closed at %s", fail_stage), + errdetail( + "result=%d handoff_result=%d authority_valid=%d " + "tag=%u/%u/%u/%d/%u requester=%d procno=%u request_id=%llu " + "ticket=%llu grant=%llu image=%llu committed_generation=%llu " + "authority_state=%u x_holder=%d s_holders=0x%x " + "master_holder=%u pending_x_node=%d pending_x_value=%llu source=%u " + "image_page_scn=%llu watermark_scn=%llu " + "wm_src=%s wm_have=%d wm_table_full=%d wm_sender=%d " + "wm_request_id=%llu wm_epoch=%llu wm_old_scn=%llu wm_new_scn=%llu " + "wm_matches_current=%d", + (int)result, (int)handoff_result, authority_valid ? 1 : 0, + final_ack->ref.identity.tag.spcOid, final_ack->ref.identity.tag.dbOid, + final_ack->ref.identity.tag.relNumber, (int)final_ack->ref.identity.tag.forkNum, + final_ack->ref.identity.tag.blockNum, final_ack->ref.identity.node_id, + final_ack->ref.identity.procno, (unsigned long long)final_ack->ref.identity.request_id, + (unsigned long long)final_ack->ref.handle.ticket_id, + (unsigned long long)final_ack->ref.grant_generation, + (unsigned long long)final_ack->image_id, + (unsigned long long)final_ack->committed_own_generation, (unsigned int)authority.state, + authority.x_holder_node, authority.s_holders_bitmap, authority.master_holder.node_id, + authority.pending_x_requester_node, (unsigned long long)authority.pending_x_since_lsn, + token.image.source_node, (unsigned long long)handoff.page_scn, + (unsigned long long)watermark_scn, cluster_pcm_wm_src_text(wm_prov.source), + wm_have ? 1 : 0, wm_prov.table_full ? 1 : 0, wm_prov.sender_node, + (unsigned long long)wm_prov.request_id, (unsigned long long)wm_prov.epoch, + (unsigned long long)wm_prov.old_scn, (unsigned long long)wm_prov.new_scn, + wm_have && wm_prov.new_scn == watermark_scn ? 1 : 0))); + cluster_pcm_x_runtime_fail_closed(); } @@ -11654,11 +13532,17 @@ gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, { GcsBlockPcmXImageBinding binding; GcsBlockDedupKey key; + GcsBlockPcmXImageResult image_result; PcmXLocalHolderProgress progress; + PcmXLocalDrainCertificate certificate; PcmXQueueResult progress_result; PcmXQueueResult result; + ClusterPcmOwnSelfHandoffSample handoff_sample; + ClusterPcmOwnResult handoff_result; + ClusterPcmOwnResult own_result; ClusterPcmXRevokeFinishMode finish_mode; uint64 holder_image_id = 0; + bool certificate_exact; bool holder_image = false; bool holder_ref = false; bool self_source_handoff = false; @@ -11671,9 +13555,9 @@ gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, } else if (progress_result != PCM_X_QUEUE_OK && progress_result != PCM_X_QUEUE_NOT_FOUND) return progress_result; - result = cluster_pcm_x_local_drain_poll_exact(poll, authenticated_master_node, - authenticated_master_session); - if (result != PCM_X_QUEUE_OK) + result = cluster_pcm_x_local_drain_poll_certificate_exact( + poll, authenticated_master_node, authenticated_master_session, &certificate); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) return result; /* DRAIN mutates the holder leg. Capture the image after that transition so @@ -11695,6 +13579,7 @@ gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, binding.identity.ref = progress.ref; binding.identity.image = progress.image; binding.master_session = authenticated_master_session; + binding.required_page_scn = progress.required_page_scn; holder_image = true; } else if (progress_result != PCM_X_QUEUE_OK && progress_result != PCM_X_QUEUE_NOT_FOUND) { /* Local DRAIN is already durable; losing its exact image evidence cannot @@ -11707,43 +13592,108 @@ gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, return PCM_X_QUEUE_CORRUPT; } if (!holder_image) - return PCM_X_QUEUE_OK; - self_source_handoff = binding.identity.image.source_node == (uint32)cluster_node_id - && poll->ref.identity.node_id == cluster_node_id; - if (self_source_handoff - && cluster_bufmgr_pcm_own_self_handoff_x_exact(&poll->ref.identity.tag, - binding.identity.image.source_own_generation) - != CLUSTER_PCM_OWN_OK) { - /* FINAL makes the in-place X descriptor authoritative before DRAIN. - * Preserve the immutable record if that exact proof is missing. */ + return result; + image_result = cluster_gcs_block_dedup_pcm_x_drain_status_exact( + worker_id, &key, &poll->ref.identity.tag, &binding); + if (image_result == GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) + return result; + if (image_result != GCS_BLOCK_PCM_X_IMAGE_NOT_READY) { cluster_pcm_x_runtime_fail_closed(); return PCM_X_QUEUE_CORRUPT; } - if (cluster_gcs_block_dedup_pcm_x_release_exact(worker_id, &key, &poll->ref.identity.tag, - &binding) - != GCS_BLOCK_PCM_X_IMAGE_RELEASED) { + self_source_handoff = binding.identity.image.source_node == (uint32)cluster_node_id + && poll->ref.identity.node_id == cluster_node_id; + if (self_source_handoff) { + /* The exact completion certificate - the writer-round protocol ledger + * captured under the drain's own tag lock - is the release authority. + * The live descriptor is NOT: its per-buf_id generation carries no + * BufferTag lineage, and the tag may legitimately have moved on + * (X->S downgrade for a reader, a later lifecycle) or been evicted + * since FINAL_ACK. The descriptor probe below only reports a + * structurally malformed flags/token shape. */ + certificate_exact + = certificate.valid && gcs_block_pcm_x_ticket_ref_equal(&certificate.ref, &poll->ref) + && certificate.image.image_id == binding.identity.image.image_id + && certificate.image.source_own_generation + == binding.identity.image.source_own_generation + && certificate.image.source_node == binding.identity.image.source_node + && certificate.master_node == authenticated_master_node + && certificate.master_session_incarnation == authenticated_master_session + && certificate.committed_own_generation + == binding.identity.image.source_own_generation + 1; + handoff_result + = cluster_bufmgr_pcm_own_self_handoff_probe(&poll->ref.identity.tag, &handoff_sample); + if (handoff_result == CLUSTER_PCM_OWN_CORRUPT) { + ereport(LOG, (errmsg("PCM-X self-source DRAIN found a malformed descriptor lifecycle"), + errdetail("own_flags=%u own_token=%llu live_gen=%llu pcm_state=%u " + "buffer_type=%u bm_valid=%d found=%d", + (unsigned int)handoff_sample.own_flags, + (unsigned long long)handoff_sample.live_token, + (unsigned long long)handoff_sample.live_generation, + (unsigned int)handoff_sample.pcm_state, + (unsigned int)handoff_sample.buffer_type, + handoff_sample.bm_valid ? 1 : 0, + handoff_sample.buffer_found ? 1 : 0))); + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + if (!certificate_exact) { + /* Local DRAIN is already durable, and a duplicate poll cannot fabricate + * this certificate. Preserve the immutable record and stop the runtime + * for formation recovery instead of retrying into a false ACK. */ + ereport(LOG, + (errmsg("PCM-X self-source DRAIN certificate is not exact"), + errdetail("valid=%d cert_committed=%llu cert_image=%llu cert_source_gen=%llu " + "image=%llu source_gen=%llu source_node=%u live_gen=%llu " + "own_flags=%u pcm_state=%u found=%d", + certificate.valid ? 1 : 0, + (unsigned long long)certificate.committed_own_generation, + (unsigned long long)certificate.image.image_id, + (unsigned long long)certificate.image.source_own_generation, + (unsigned long long)binding.identity.image.image_id, + (unsigned long long)binding.identity.image.source_own_generation, + (unsigned int)binding.identity.image.source_node, + (unsigned long long)handoff_sample.live_generation, + (unsigned int)handoff_sample.own_flags, + (unsigned int)handoff_sample.pcm_state, + handoff_sample.buffer_found ? 1 : 0))); + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + } + if (!self_source_handoff) { + finish_mode = cluster_pcm_x_revoke_finish_mode(&poll->ref.identity.tag, 0); + if (finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_RETAIN) { + /* The A-record remains the retry authority until the conditional + * descriptor release succeeds. A busy BufferContent lock is normal + * DATA-worker contention, not corruption and not a reason to erase + * immutable evidence. */ + own_result = cluster_bufmgr_pcm_own_release_retained_image( + &poll->ref.identity.tag, binding.identity.image.source_own_generation); + gcs_block_pcm_x_note_own_result(own_result); + if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY) + return PCM_X_QUEUE_NOT_READY; + if (own_result != CLUSTER_PCM_OWN_OK) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + } else if (finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_DROP) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + } + image_result = cluster_gcs_block_dedup_pcm_x_release_exact( + worker_id, &key, &poll->ref.identity.tag, &binding, authenticated_master_node); + if (image_result != GCS_BLOCK_PCM_X_IMAGE_RELEASED + && image_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) { cluster_pcm_x_runtime_fail_closed(); return PCM_X_QUEUE_CORRUPT; } if (self_source_handoff) { if (ClusterGcsBlock != NULL) pg_atomic_fetch_add_u64(&ClusterGcsBlock->pcm_x_self_handoff_drain_count, 1); - return PCM_X_QUEUE_OK; - } - finish_mode = cluster_pcm_x_revoke_finish_mode(&poll->ref.identity.tag, 0); - if (finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_DROP) - return PCM_X_QUEUE_OK; - if (finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_RETAIN) { - cluster_pcm_x_runtime_fail_closed(); - return PCM_X_QUEUE_CORRUPT; } - if (cluster_bufmgr_pcm_own_release_retained_image(&poll->ref.identity.tag, - binding.identity.image.source_own_generation) - != CLUSTER_PCM_OWN_OK) { - cluster_pcm_x_runtime_fail_closed(); - return PCM_X_QUEUE_CORRUPT; - } - return PCM_X_QUEUE_OK; + return result; } @@ -11934,6 +13884,8 @@ cluster_gcs_pcm_x_terminal_kick(const PcmXTicketRef *ref) if (ref->grant_generation == 0) { result = gcs_block_pcm_x_cancel_terminal_cleanup_exact(ref); cluster_pcm_x_stats_note_queue_result(result); + cluster_pcm_x_terminal_note(PCM_X_TERMINAL_NOTE_CANCEL_CLEANUP, (uint32)result, + ref->handle.ticket_id); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { gcs_block_pcm_x_master_drive_fail_closed(result); return; @@ -11971,6 +13923,8 @@ cluster_gcs_pcm_x_terminal_kick(const PcmXTicketRef *ref) /* All ACKs may already be present. Detach first; global ticket order is * enforced inside the engine and the periodic oldest-ticket scan retries. */ result = cluster_pcm_x_master_detach_terminal_exact(ref); + cluster_pcm_x_terminal_note(PCM_X_TERMINAL_NOTE_DETACH, (uint32)result, + ref->handle.ticket_id); if (result == PCM_X_QUEUE_OK) { gcs_block_pcm_x_master_drive_tag(&ref->identity.tag, ref->identity.cluster_epoch); return; @@ -12021,8 +13975,13 @@ cluster_gcs_pcm_x_terminal_kick(const PcmXTicketRef *ref) ref, (PcmXTerminalLegKind)kind, node, responder_session, &token); if (result == PCM_X_QUEUE_STALE || result == PCM_X_QUEUE_NOT_READY) continue; - if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { + cluster_pcm_x_terminal_note(kind == PCM_X_TERMINAL_LEG_DRAIN + ? PCM_X_TERMINAL_NOTE_ARM_DRAIN + : PCM_X_TERMINAL_NOTE_ARM_RETIRE, + (uint32)result, ref->handle.ticket_id); return; + } if (kind == PCM_X_TERMINAL_LEG_DRAIN) { memset(&poll, 0, sizeof(poll)); poll.ref = *ref; @@ -12030,8 +13989,13 @@ cluster_gcs_pcm_x_terminal_kick(const PcmXTicketRef *ref) /* Even self DRAIN is application traffic. Dispatch it through * the tag shard so holder-byte release and terminal publication * remain one single-consumer action. */ - (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_DRAIN_POLL, node, &poll, - sizeof(poll)); + cluster_pcm_x_terminal_note( + PCM_X_TERMINAL_NOTE_DRAIN_STAGE, + cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_DRAIN_POLL, node, &poll, + sizeof(poll)) + ? (uint32)PCM_X_QUEUE_OK + : (uint32)PCM_X_QUEUE_NOT_READY, + ref->handle.ticket_id); return; } else { runtime = cluster_pcm_x_runtime_snapshot(); @@ -12041,15 +14005,24 @@ cluster_gcs_pcm_x_terminal_kick(const PcmXTicketRef *ref) retire.retire_through_ticket_id = ref->handle.ticket_id; retire.sender_node = node; if (node != cluster_node_id) { - (void)gcs_block_pcm_x_stage_retire_up_to(node, &retire, &ref->identity.tag); + cluster_pcm_x_terminal_note( + PCM_X_TERMINAL_NOTE_RETIRE_STAGE, + gcs_block_pcm_x_stage_retire_up_to(node, &retire, &ref->identity.tag) + ? (uint32)PCM_X_QUEUE_OK + : (uint32)PCM_X_QUEUE_NOT_READY, + ref->handle.ticket_id); return; } result = gcs_block_pcm_x_local_retire_apply_and_wake(&retire, cluster_node_id, responder_session); + cluster_pcm_x_terminal_note(PCM_X_TERMINAL_NOTE_RETIRE_LOCAL, (uint32)result, + ref->handle.ticket_id); if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) return; result = cluster_pcm_x_master_retire_ack_resolve_exact( &retire, cluster_node_id, responder_session, &resolved); + cluster_pcm_x_terminal_note(PCM_X_TERMINAL_NOTE_RETIRE_ACK_RESOLVE, + (uint32)result, ref->handle.ticket_id); } if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) return; @@ -12070,24 +14043,40 @@ gcs_block_pcm_x_master_drive_retry_tick(void) { static Size cursor = 0; BufferTag tag; + PcmXQueueResult result; + Size cursor_before; uint64 cluster_epoch; - if (cluster_pcm_x_master_drive_work_next(&cursor, PCM_X_MASTER_DRIVE_SCAN_BUDGET, &tag, - &cluster_epoch) - == PCM_X_QUEUE_OK) + cursor_before = cursor; + result = cluster_pcm_x_master_drive_work_next(&cursor, PCM_X_MASTER_DRIVE_SCAN_BUDGET, &tag, + &cluster_epoch); + if (result == PCM_X_QUEUE_OK) gcs_block_pcm_x_master_drive_tag(&tag, cluster_epoch); + else + gcs_block_pcm_x_master_retry_observe("work-next", result, -1, cursor_before, cursor, NULL, + cluster_epoch); } +#define GCS_BLOCK_PCM_X_TERMINAL_TICK_BUDGET 8 + static void gcs_block_pcm_x_terminal_retry_tick(void) { PcmXTicketRef ref; - PcmXQueueResult result; - - result = cluster_pcm_x_master_terminal_work_next(&ref); - if (result == PCM_X_QUEUE_OK) + uint64 after = 0; + int kicks; + + /* The oldest ticket keeps frontier priority, but a stuck head must not + * starve younger terminal tickets: the head's own RETIRE preflight can be + * waiting for exactly the younger ticket's DRAIN evidence (cross-lane + * holder interlock), so each tick walks the terminal set in id order. */ + for (kicks = 0; kicks < GCS_BLOCK_PCM_X_TERMINAL_TICK_BUDGET; kicks++) { + if (cluster_pcm_x_master_terminal_work_next_after(after, &ref) != PCM_X_QUEUE_OK) + return; cluster_gcs_pcm_x_terminal_kick(&ref); + after = ref.handle.ticket_id; + } } @@ -12928,8 +14917,11 @@ gcs_block_pi_discard_master_apply(BufferTag tag, SCN written_scn) void cluster_gcs_block_pi_discard_drain(void) { + bool data_plane; + if (ClusterGcsBlock == NULL || !cluster_past_image) return; + data_plane = cluster_gcs_block_family_on_data_plane(); for (;;) { BufferTag tag; @@ -12945,10 +14937,39 @@ cluster_gcs_block_pi_discard_drain(void) slot = ClusterGcsBlock->pi_note_drain_seq % CLUSTER_GCS_PI_NOTE_RING_SIZE; tag = ClusterGcsBlock->pi_note_ring[slot].tag; page_scn = ClusterGcsBlock->pi_note_ring[slot].page_scn; - ClusterGcsBlock->pi_note_drain_seq++; + if (!data_plane) + ClusterGcsBlock->pi_note_drain_seq++; SpinLockRelease(&ClusterGcsBlock->pi_note_lock); master_node = cluster_gcs_lookup_master(tag); + if (data_plane) { + GcsBlockInvalidateAckPayload note; + int worker; + + if (master_node < 0 || master_node >= 32) + break; + memset(¬e, 0, sizeof(note)); + note.request_id = 0; /* unsolicited: diverted before slot logic */ + note.epoch = cluster_epoch_get_current(); + note.tag = tag; + note.sender_node = cluster_node_id; + note.ack_status = GCS_BLOCK_INVALIDATE_ACK_STATUS_PI_DURABLE_NOTE; + GcsBlockInvalidateAckPayloadSetPageScn(¬e, page_scn); + note.checksum = gcs_block_compute_invalidate_ack_checksum(¬e); + worker = cluster_lms_shard_for_tag(&tag, cluster_lms_workers); + if (!cluster_lms_outbound_enqueue(worker, PGRAC_IC_MSG_GCS_BLOCK_INVALIDATE_ACK, + (uint32)master_node, ¬e, sizeof(note))) + break; + + /* The staged ring owns a byte copy now. Advancing afterwards gives + * at-least-once delivery across a crash between these two operations; + * a duplicate status-3 apply is idempotent. Never take the outbound + * LWLock while holding this spinlock. */ + SpinLockAcquire(&ClusterGcsBlock->pi_note_lock); + ClusterGcsBlock->pi_note_drain_seq++; + SpinLockRelease(&ClusterGcsBlock->pi_note_lock); + continue; + } if (master_node == cluster_node_id) { gcs_block_pi_discard_master_apply(tag, page_scn); } else if (master_node >= 0 && master_node < 32) { @@ -13158,6 +15179,234 @@ cluster_gcs_block_local_x_upgrade(BufferTag tag) * holder would). * ============================================================ */ +/* + * Break the mirror-N GRANT_PENDING side of the pin/INVALIDATE cycle without + * touching another backend's ownership token. A backend reserves its GCS + * slot only after it has installed the descriptor's single GRANT_PENDING + * reservation, so at most one live N->S slot can match this tag. The current + * authenticated master INVALIDATE is sufficient negative authority for that + * exact attempt: publish a local DENIED_PENDING_X and let the owning backend + * perform the established token-exact abort/rearm sequence. + * + * Direct-land attempts are deliberately excluded while their target is live; + * their lane has its own LMON abort protocol. A later INVALIDATE retry can + * wake the slot after that protocol reaches ABORTED. First-reply-wins remains + * intact because reply_received is tested and set under the backend block's + * exclusive lock. + */ +static bool +gcs_block_wake_local_pending_s_request(const GcsBlockInvalidatePayload *inv) +{ + int backend_idx; + + if (inv == NULL || gcs_block_backend_blocks == NULL || ClusterGcsBlock == NULL) + return false; + + for (backend_idx = 0; backend_idx < MaxBackends; backend_idx++) { + ClusterGcsBlockBackendBlock *blk = &gcs_block_backend_blocks[backend_idx]; + int slot_idx; + + LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); + for (slot_idx = 0; slot_idx < MAX_OUTSTANDING_BLOCK_REQUESTS_PER_BACKEND; slot_idx++) { + ClusterGcsBlockOutstandingSlot *slot = &blk->slots[slot_idx]; + GcsBlockReplyHeader *hdr; + + if (!GcsBlockLocalPendingSDenialMatches( + slot->in_use, slot->reply_received, slot->stale, slot->transition_id, + &slot->tag, slot->request_epoch, slot->expected_master_node, slot->direct_state, + slot->direct_target_prepared, &inv->tag, inv->epoch, inv->master_node)) + continue; + + hdr = &slot->reply_header; + memset(hdr, 0, sizeof(*hdr)); + hdr->request_id = slot->request_id; + hdr->epoch = inv->epoch; + hdr->sender_node = inv->master_node; + hdr->requester_backend_id = backend_idx + 1; + hdr->transition_id = slot->transition_id; + hdr->status = (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X; + GcsBlockReplyHeaderSetForwardingMasterNode(hdr, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + memset(slot->reply_block_data, 0, sizeof(slot->reply_block_data)); + hdr->checksum = gcs_block_compute_checksum(slot->reply_block_data); + slot->reply_sf_dep_valid = false; + slot->reply_sf_flags = 0; + cluster_sf_dep_vec_reset(&slot->reply_sf_dep_vec); + slot->reply_undo_trailer_valid = false; + slot->reply_undo_tt_generation = 0; + slot->reply_undo_authority_scn = 0; + slot->reply_received = true; + ConditionVariableSignal(&slot->reply_cv); + pg_atomic_fetch_add_u64(&ClusterGcsBlock->starvation_denied_pending_x_count, 1); + LWLockRelease(&blk->lock.lock); + return true; + } + LWLockRelease(&blk->lock.lock); + } + return false; +} + +/* Observation-only rejection bits for the closest local N->S request slot. + * They must never feed protocol control flow; the authority predicate remains + * GcsBlockLocalPendingSDenialMatches(). */ +#define GCS_BLOCK_PENDING_OBS_NO_SLOT UINT32_C(0x001) +#define GCS_BLOCK_PENDING_OBS_REPLY UINT32_C(0x002) +#define GCS_BLOCK_PENDING_OBS_STALE UINT32_C(0x004) +#define GCS_BLOCK_PENDING_OBS_TRANSITION UINT32_C(0x008) +#define GCS_BLOCK_PENDING_OBS_TAG UINT32_C(0x010) +#define GCS_BLOCK_PENDING_OBS_EPOCH UINT32_C(0x020) +#define GCS_BLOCK_PENDING_OBS_MASTER UINT32_C(0x040) +#define GCS_BLOCK_PENDING_OBS_DIRECT_STATE UINT32_C(0x080) +#define GCS_BLOCK_PENDING_OBS_DIRECT_PREPARED UINT32_C(0x100) + +typedef struct GcsBlockGrantPendingObservation { + bool valid; + uint64 invalidate_request_id; + uint64 invalidate_epoch; + BufferTag tag; + int32 invalidate_master; + int own_result; + int buffer_id; + uint8 own_state; + uint64 own_generation; + uint64 own_token; + uint32 own_flags; + bool woke_local; + int slot_backend; + int slot_index; + uint64 slot_request_id; + uint64 slot_epoch; + int32 slot_master; + uint8 slot_transition; + bool slot_reply_received; + bool slot_stale; + int slot_direct_state; + bool slot_direct_prepared; + uint32 reject_mask; +} GcsBlockGrantPendingObservation; + +/* LMON is single-threaded. A tiny process-local cache suppresses identical + * BUSY retries while retaining the first sighting and every identity/state + * change for the exact INVALIDATE. */ +static void +gcs_block_observe_grant_pending_invalidate(const GcsBlockInvalidatePayload *inv, bool woke_local) +{ + static GcsBlockGrantPendingObservation cache[8]; + static uint32 next_cache_slot = 0; + GcsBlockGrantPendingObservation obs; + ClusterPcmOwnSnapshot own; + int backend_idx; + int candidate_priority = 0; + int cache_idx = -1; + int i; + + if (inv == NULL) + return; + memset(&obs, 0, sizeof(obs)); + memset(&own, 0, sizeof(own)); + obs.valid = true; + obs.invalidate_request_id = inv->request_id; + obs.invalidate_epoch = inv->epoch; + obs.tag = inv->tag; + obs.invalidate_master = inv->master_node; + obs.buffer_id = -1; + obs.slot_backend = -1; + obs.slot_index = -1; + obs.slot_master = -1; + obs.woke_local = woke_local; + obs.own_result = (int)cluster_bufmgr_pcm_own_snapshot_by_tag(&inv->tag, &obs.buffer_id, &own); + obs.own_state = own.pcm_state; + obs.own_generation = own.generation; + obs.own_token = own.reservation_token; + obs.own_flags = own.flags; + + if (gcs_block_backend_blocks != NULL) { + for (backend_idx = 0; backend_idx < MaxBackends; backend_idx++) { + ClusterGcsBlockBackendBlock *blk = &gcs_block_backend_blocks[backend_idx]; + int slot_idx; + + LWLockAcquire(&blk->lock.lock, LW_SHARED); + for (slot_idx = 0; slot_idx < MAX_OUTSTANDING_BLOCK_REQUESTS_PER_BACKEND; slot_idx++) { + ClusterGcsBlockOutstandingSlot *slot = &blk->slots[slot_idx]; + int priority; + + if (!slot->in_use) + continue; + priority = BufferTagsEqual(&slot->tag, &inv->tag) ? 2 : 1; + if (priority <= candidate_priority) + continue; + candidate_priority = priority; + obs.slot_backend = backend_idx; + obs.slot_index = slot_idx; + obs.slot_request_id = slot->request_id; + obs.slot_epoch = slot->request_epoch; + obs.slot_master = slot->expected_master_node; + obs.slot_transition = slot->transition_id; + obs.slot_reply_received = slot->reply_received; + obs.slot_stale = slot->stale; + obs.slot_direct_state = (int)slot->direct_state; + obs.slot_direct_prepared = slot->direct_target_prepared; + obs.reject_mask = 0; + if (slot->reply_received) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_REPLY; + if (slot->stale) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_STALE; + if (slot->transition_id != (uint8)PCM_TRANS_N_TO_S) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_TRANSITION; + if (!BufferTagsEqual(&slot->tag, &inv->tag)) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_TAG; + if (slot->request_epoch != inv->epoch) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_EPOCH; + if (slot->expected_master_node != inv->master_node) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_MASTER; + if (slot->direct_state != GCS_BLOCK_DIRECT_UNARMED + && slot->direct_state != GCS_BLOCK_DIRECT_ABORTED) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_DIRECT_STATE; + if (slot->direct_target_prepared) + obs.reject_mask |= GCS_BLOCK_PENDING_OBS_DIRECT_PREPARED; + } + LWLockRelease(&blk->lock.lock); + if (candidate_priority == 2) + break; + } + } + if (candidate_priority == 0) + obs.reject_mask = GCS_BLOCK_PENDING_OBS_NO_SLOT; + + for (i = 0; i < lengthof(cache); i++) { + if (cache[i].valid && cache[i].invalidate_request_id == obs.invalidate_request_id + && cache[i].invalidate_epoch == obs.invalidate_epoch + && cache[i].invalidate_master == obs.invalidate_master + && BufferTagsEqual(&cache[i].tag, &obs.tag)) { + cache_idx = i; + break; + } + } + if (cache_idx >= 0 && memcmp(&cache[cache_idx], &obs, sizeof(obs)) == 0) + return; + if (cache_idx < 0) { + cache_idx = (int)(next_cache_slot % lengthof(cache)); + next_cache_slot++; + } + cache[cache_idx] = obs; + + elog(LOG, + "cluster PCM grant-pending invalidate observation: invalidate_request_id=%llu " + "epoch=%llu master=%d x_requester=%u rel=%u fork=%d blk=%u own_result=%d " + "buffer=%d own_state=%u own_generation=%llu own_token=%llu own_flags=0x%x " + "woke_local=%d slot_backend=%d slot_index=%d slot_request_id=%llu " + "slot_epoch=%llu slot_master=%d slot_transition=%u reply_received=%d stale=%d " + "direct_state=%d direct_prepared=%d reject_mask=0x%x", + (unsigned long long)obs.invalidate_request_id, (unsigned long long)obs.invalidate_epoch, + obs.invalidate_master, (unsigned int)inv->invalidating_for_x_node, inv->tag.relNumber, + (int)inv->tag.forkNum, inv->tag.blockNum, obs.own_result, obs.buffer_id, obs.own_state, + (unsigned long long)obs.own_generation, (unsigned long long)obs.own_token, obs.own_flags, + obs.woke_local ? 1 : 0, obs.slot_backend, obs.slot_index, + (unsigned long long)obs.slot_request_id, (unsigned long long)obs.slot_epoch, + obs.slot_master, obs.slot_transition, obs.slot_reply_received ? 1 : 0, + obs.slot_stale ? 1 : 0, obs.slot_direct_state, obs.slot_direct_prepared ? 1 : 0, + obs.reject_mask); +} + /* * gcs_block_invalidate_execute — apply one INVALIDATE directive + ACK. * @@ -13169,7 +15418,7 @@ static bool gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) { GcsBlockInvalidateAckPayload ack; - ClusterPcmOwnResult own_result; + ClusterPcmOwnResult own_result = CLUSTER_PCM_OWN_INVALID; PcmXQueueResult queue_result; PcmLockMode pre_state; XLogRecPtr page_lsn = InvalidXLogRecPtr; @@ -13178,6 +15427,7 @@ gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) uint64 master_session = 0; uint8 ack_status = 0; /* OK */ bool kept_pi = false; /* spec-6.12h D-h2 — drop converted to a PI */ + bool woke_local = false; uint64 current_epoch = cluster_epoch_get_current(); if (inv->epoch != current_epoch) { @@ -13204,12 +15454,17 @@ gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) * LockBuffer finalize then sets pcm_state=X; acking already_invalidated * here would let the master clear this node's holder bit and re-grant X * elsewhere, stranding the just-finalized stale X (double X holder, - * Rule 8.A). Return false so the caller PARKS the directive (the round-5 - * A2 park lot); the LMS-loop retry re-runs once the grant finalized - * (PENDING cleared), and then sees the real X/S and invalidates it. + * Rule 8.A). A BUSY-capable master gets a negative ACK and retries; + * legacy peers use the round-5 park lot. Either route re-runs only after + * the in-flight grant has finalized or its owner has aborted PENDING. */ if (cluster_bufmgr_block_grant_pending(inv->tag)) { cluster_pcm_note_invalidate_parked_grant_pending(); + /* The master-side denial may be queued behind this same-tag BUSY + * loop. Deliver its exact local equivalent now so the reservation + * owner clears GRANT_PENDING before the next INVALIDATE retry. */ + woke_local = gcs_block_wake_local_pending_s_request(inv); + gcs_block_observe_grant_pending_invalidate(inv, woke_local); /* * Ruling ② — a BUSY-capable master gets the negative ACK RIGHT * NOW instead of a silent park: it aborts the round (clears @@ -13219,7 +15474,8 @@ gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) * very pending_x). Nothing local changed; terminal for THIS * directive. An old master falls back to the round-5 park. */ - if (cluster_sf_peer_supports_gcs_inval_busy(inv->master_node)) { + if (inv->master_node == cluster_node_id + || cluster_sf_peer_supports_gcs_inval_busy(inv->master_node)) { pg_atomic_fetch_add_u64(&ClusterGcsBlock->invalidate_busy_sent_count, 1); ack_status = (uint8)GCS_BLOCK_INVALIDATE_ACK_STATUS_RETRYABLE_BUSY; goto send_ack; @@ -13311,6 +15567,22 @@ gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) } /* FALLTHROUGH */ case CLUSTER_BUFMGR_GCS_DROP_STALE: + if (pre_state == PCM_LOCK_MODE_S) { + static bool pcm_x_queue_invalidate_refusal_logged = false; + + if (!pcm_x_queue_invalidate_refusal_logged) { + pcm_x_queue_invalidate_refusal_logged = true; + ereport(LOG, (errmsg_internal("PCM-X queue INVALIDATE passive-S release refused"), + errdetail_internal( + "epoch=%llu tag=%u/%u/%u/%d/%u requester=%u " + "master=%d request_id=%llu queue_result=%d own_result=%d", + (unsigned long long)inv->epoch, inv->tag.spcOid, inv->tag.dbOid, + inv->tag.relNumber, (int)inv->tag.forkNum, inv->tag.blockNum, + (unsigned int)inv->invalidating_for_x_node, inv->master_node, + (unsigned long long)inv->request_id, (int)queue_result, + (int)own_result))); + } + } /* GCS serve-stall round-5 (A2): nothing changed, no ACK — the * caller parks the directive and the LMS loop retries it. STALE is * unreachable here (the invalidate wrapper passes no expected_lsn, so @@ -13322,7 +15594,8 @@ gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) * is often itself waiting behind the master's pending_x, so parking * only burns the master's timeout). Nothing local changed. */ - if (cluster_sf_peer_supports_gcs_inval_busy(inv->master_node)) { + if (inv->master_node == cluster_node_id + || cluster_sf_peer_supports_gcs_inval_busy(inv->master_node)) { pg_atomic_fetch_add_u64(&ClusterGcsBlock->invalidate_busy_sent_count, 1); ack_status = (uint8)GCS_BLOCK_INVALIDATE_ACK_STATUS_RETRYABLE_BUSY; goto send_ack; @@ -13477,12 +15750,11 @@ cluster_gcs_block_invalidate_park_tick(void) * window point instead. Same force-behavior inject pattern as * cluster-gcs-block-duplicate-grant-reply / -stale-ship. * - * With GRANT_PENDING staged the handler parks (returns false, bumps - * pcm.invalidate_parked_grant_pending_count) BEFORE any wire send, so the - * synthetic request_id/master_node never reach the ACK path. Without the - * park fix it would have acked already_invalidated (the W3 defect) — the - * ACK then goes to a stale request_id slot and is rejected (HC100), so - * even the defect arm cannot corrupt master state from this shim. + * With GRANT_PENDING staged the handler refuses a positive ACK and bumps + * pcm.invalidate_parked_grant_pending_count. Modern/self masters receive + * RETRYABLE_BUSY; legacy peers park. The synthetic request id cannot match + * a real invalidate slot, so even this deterministic test arm cannot mutate + * master holder state. * * Caller (bufmgr LockBuffer) holds the buffer's content lock; the handler's * park path takes only the mapping partition (SHARED) + header spinlock — @@ -13706,9 +15978,11 @@ cluster_gcs_handle_block_invalidate_ack_envelope(const ClusterICEnvelope *env, c * Off-epoch notes are dropped (fail-safe: the PI merely lingers). */ if (ack->ack_status == GCS_BLOCK_INVALIDATE_ACK_STATUS_PI_DURABLE_NOTE) { - if (ack->epoch == cluster_epoch_get_current()) + if (ack->epoch == cluster_epoch_get_current()) { + pg_atomic_fetch_add_u64(&ClusterGcsBlock->pi_durable_note_apply_count, 1); gcs_block_pi_discard_master_apply(ack->tag, GcsBlockInvalidateAckPayloadGetPageScn(ack)); + } return; } if (ack->ack_status == GCS_BLOCK_INVALIDATE_ACK_STATUS_PI_KEPT_NOTE) { @@ -13724,9 +15998,21 @@ cluster_gcs_handle_block_invalidate_ack_envelope(const ClusterICEnvelope *env, c cluster_pcm_x_stats_note_queue_result(queue_result); if (queue_result == PCM_X_QUEUE_OK) queue_positive = true; - else if (queue_result == PCM_X_QUEUE_BAD_STATE || queue_result == PCM_X_QUEUE_CORRUPT - || queue_result == PCM_X_QUEUE_COUNTER_EXHAUSTED - || queue_result == PCM_X_QUEUE_INVALID) + else if (queue_result == PCM_X_QUEUE_BUSY) { + uint64 now_ms = (uint64)(GetCurrentTimestamp() / 1000); + uint64 retry_delay_ms = gcs_block_pcm_x_invalidate_busy_retry_delay_ms(&queue_snapshot); + + queue_result = cluster_pcm_x_master_invalidate_busy_backoff_exact( + &queue_snapshot, ack->sender_node, now_ms, retry_delay_ms, &queue_updated); + cluster_pcm_x_stats_note_queue_result(queue_result); + if (queue_result == PCM_X_QUEUE_OK || queue_result == PCM_X_QUEUE_DUPLICATE) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->invalidate_busy_received_count, 1); + else + gcs_block_pcm_x_master_drive_fail_closed(queue_result); + return; + } else if (queue_result == PCM_X_QUEUE_BAD_STATE || queue_result == PCM_X_QUEUE_CORRUPT + || queue_result == PCM_X_QUEUE_COUNTER_EXHAUSTED + || queue_result == PCM_X_QUEUE_INVALID) gcs_block_pcm_x_master_drive_fail_closed(queue_result); if (!queue_positive) return; @@ -14467,6 +16753,12 @@ cluster_gcs_get_pi_watermark_retire_count(void) return ClusterGcsBlock ? pg_atomic_read_u64(&ClusterGcsBlock->pi_watermark_retire_count) : 0; } +uint64 +cluster_gcs_get_pi_durable_note_apply_count(void) +{ + return ClusterGcsBlock ? pg_atomic_read_u64(&ClusterGcsBlock->pi_durable_note_apply_count) : 0; +} + uint64 cluster_gcs_get_lost_write_detected_count(void) { diff --git a/src/backend/cluster/cluster_gcs_block_dedup.c b/src/backend/cluster/cluster_gcs_block_dedup.c index eb687b0b8b..6600794a45 100644 --- a/src/backend/cluster/cluster_gcs_block_dedup.c +++ b/src/backend/cluster/cluster_gcs_block_dedup.c @@ -397,6 +397,31 @@ dedup_pcm_x_note_failclosed(ClusterGcsBlockDedupShard *shard) pg_atomic_fetch_add_u64(&shard->pcm_x_failclosed_count, 1); } +/* The generic smart-fusion metadata cell is eight bytes and PCM-X entry + * kinds never use it. Keep the exact revoke token there without changing + * the fixed shared-memory entry layout. */ +static uint64 +dedup_pcm_x_reservation_token_get(const GcsBlockDedupEntry *entry) +{ + uint64 token; + + memcpy(&token, &entry->has_sf_dep, sizeof(token)); + return token; +} + +static void +dedup_pcm_x_reservation_token_set(GcsBlockDedupEntry *entry, uint64 token) +{ + memcpy(&entry->has_sf_dep, &token, sizeof(token)); +} + +static bool +dedup_pcm_x_source_state_valid(uint8 pcm_state) +{ + return pcm_state == (uint8)PCM_STATE_N || pcm_state == (uint8)PCM_STATE_S + || pcm_state == (uint8)PCM_STATE_X; +} + static uint32 dedup_pcm_x_block_checksum(const char *block_data) { @@ -452,6 +477,24 @@ dedup_pcm_x_binding_valid(const GcsBlockDedupKey *key, const BufferTag *tag, return true; } +/* Generic entries retain the established signed TTL meaning of + * pinned_lifetime_us. PCM-X entries are excluded from generic GC, so that + * same fixed 8-byte cell carries their exact required source SCN without + * changing the 8472-byte entry ABI. */ +static uint64 +dedup_pcm_x_required_page_scn_get(const GcsBlockDedupEntry *entry) +{ + return entry != NULL ? (uint64)entry->pinned_lifetime_us : 0; +} + +static void +dedup_pcm_x_binding_from_entry(const GcsBlockDedupEntry *entry, GcsBlockPcmXImageBinding *binding) +{ + binding->identity = entry->payload_meta.pcm_x_identity; + binding->master_session = entry->pcm_x_master_session; + binding->required_page_scn = dedup_pcm_x_required_page_scn_get(entry); +} + static bool dedup_pcm_x_reservation_equal(const GcsBlockDedupEntry *entry, const GcsBlockPcmXImageBinding *binding) @@ -460,6 +503,7 @@ dedup_pcm_x_reservation_equal(const GcsBlockDedupEntry *entry, const GcsBlockPcmXImageIdentity *incoming = &binding->identity; return entry->pcm_x_master_session == binding->master_session + && dedup_pcm_x_required_page_scn_get(entry) == binding->required_page_scn && memcmp(&stored->ref, &incoming->ref, sizeof(stored->ref)) == 0 && stored->image.image_id == incoming->image.image_id && stored->image.source_own_generation == incoming->image.source_own_generation @@ -504,8 +548,7 @@ dedup_pcm_x_entry_payload_valid(const GcsBlockDedupKey *key, const BufferTag *ta { GcsBlockPcmXImageBinding binding; - binding.identity = entry->payload_meta.pcm_x_identity; - binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &binding); return entry->transition_id == (uint8)PCM_TRANS_N_TO_S && entry->status == (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER && dedup_pcm_x_ready_payload_valid(key, tag, &binding, &entry->reply_header, @@ -522,6 +565,17 @@ dedup_pcm_x_entry_ready_valid(const GcsBlockDedupKey *key, const BufferTag *tag, } +static bool +dedup_pcm_x_entry_drained_valid(const GcsBlockDedupKey *key, const BufferTag *tag, + const GcsBlockDedupEntry *entry) +{ + return entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED && entry->request_flags > 0 + && entry->request_flags <= (uint8)PCM_X_PROTOCOL_NODE_LIMIT + && dedup_pcm_x_reservation_token_get(entry) == 0 + && dedup_pcm_x_entry_payload_valid(key, tag, entry); +} + + /* ============================================================ * Public API. * ============================================================ */ @@ -689,6 +743,206 @@ cluster_gcs_block_dedup_lookup_or_register(int worker_id, const GcsBlockDedupKey return result; } +static bool +dedup_pending_x_denial_is_exact(const GcsBlockDedupEntry *entry) +{ + const GcsBlockReplyHeader *header = &entry->reply_header; + + return entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_GENERIC + && entry->transition_id == (uint8)PCM_TRANS_N_TO_S + && entry->status == (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X + && entry->completed_at_ts != 0 && header->request_id == entry->key.request_id + && header->epoch == entry->key.cluster_epoch && header->sender_node == cluster_node_id + && header->requester_backend_id == entry->key.requester_backend_id + && header->transition_id == (uint8)PCM_TRANS_N_TO_S + && header->status == (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X + && GcsBlockReplyHeaderGetForwardingMasterNode(header) + == GCS_BLOCK_REPLY_NO_FORWARDING_MASTER; +} + +static bool +dedup_pending_x_entry_has_legacy_s_right(const GcsBlockDedupEntry *entry) +{ + GcsBlockReplyStatus status; + + if (entry->completed_at_ts == 0) + return true; + status = (GcsBlockReplyStatus)entry->status; + return status == GCS_BLOCK_REPLY_GRANTED || status == GCS_BLOCK_REPLY_GRANTED_STORAGE_FALLBACK + || status == GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER + || status == GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER + || status == GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE; +} + +static void +dedup_pending_x_install_denial(GcsBlockDedupEntry *entry) +{ + GcsBlockReplyHeader denial; + + memset(&denial, 0, sizeof(denial)); + denial.request_id = entry->key.request_id; + denial.epoch = entry->key.cluster_epoch; + denial.sender_node = cluster_node_id; + denial.requester_backend_id = entry->key.requester_backend_id; + denial.transition_id = (uint8)PCM_TRANS_N_TO_S; + denial.status = (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X; + GcsBlockReplyHeaderSetForwardingMasterNode(&denial, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + + entry->status = (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X; + entry->reply_header = denial; + entry->has_sf_dep = false; + entry->sf_flags = 0; + entry->sf_dep_count = 0; + cluster_sf_dep_vec_reset(&entry->payload_meta.sf_dep_vec); + memset(entry->block_data, 0, sizeof(entry->block_data)); + entry->completed_at_ts = GetCurrentTimestamp(); + entry->done_at_ts = 0; +} + +/* PCM-X queue arbitration must revoke the right of an older legacy reader + * before type-49 can wait on that reader's GRANT_PENDING reservation. The + * scan returns at most one newly terminated identity per call so the caller + * can send it without a bounded/sentinel array; after all live rights are + * gone it returns one unacknowledged cached denial for loss recovery. */ +GcsBlockPendingXDenyResult +cluster_gcs_block_dedup_pending_x_deny_next(int worker_id, const BufferTag *tag, + GcsBlockDedupEntry *denied_out) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + HASH_SEQ_STATUS scan; + GcsBlockDedupEntry *entry; + GcsBlockDedupEntry replay; + bool have_replay = false; + + Assert(tag != NULL); + Assert(denied_out != NULL); + if (tag == NULL || denied_out == NULL) + return GCS_BLOCK_PENDING_X_DENY_INVALID; + memset(denied_out, 0, sizeof(*denied_out)); + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PENDING_X_DENY_INVALID; + + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + hash_seq_init(&scan, htab); + while ((entry = (GcsBlockDedupEntry *)hash_seq_search(&scan)) != NULL) { + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC + || entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 || entry->done_at_ts != 0) + continue; + + if (dedup_pending_x_denial_is_exact(entry)) { + if (!have_replay) { + replay = *entry; + have_replay = true; + } + continue; + } + if (entry->status == (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X + && entry->completed_at_ts != 0) { + dedup_pcm_x_note_failclosed(shard); + hash_seq_term(&scan); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PENDING_X_DENY_INVALID; + } + if (!dedup_pending_x_entry_has_legacy_s_right(entry)) + continue; + + dedup_pending_x_install_denial(entry); + *denied_out = *entry; + hash_seq_term(&scan); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PENDING_X_DENY_NEW; + } + + if (have_replay) + *denied_out = replay; + LWLockRelease(&shard->lock.lock); + return have_replay ? GCS_BLOCK_PENDING_X_DENY_REPLAY : GCS_BLOCK_PENDING_X_DENY_NOT_FOUND; +} + +GcsBlockPendingXDenyResult +cluster_gcs_block_dedup_pending_x_deny_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, uint8 transition_id, + GcsBlockDedupEntry *denied_out) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + bool found = false; + + Assert(key != NULL); + Assert(tag != NULL); + Assert(denied_out != NULL); + if (key == NULL || tag == NULL || denied_out == NULL + || transition_id != (uint8)PCM_TRANS_N_TO_S) + return GCS_BLOCK_PENDING_X_DENY_INVALID; + memset(denied_out, 0, sizeof(*denied_out)); + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PENDING_X_DENY_INVALID; + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (!found || entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 || entry->transition_id != transition_id) { + if (found && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC) + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PENDING_X_DENY_INVALID; + } + if (dedup_pending_x_denial_is_exact(entry)) { + *denied_out = *entry; + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PENDING_X_DENY_REPLAY; + } + if (entry->status == (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X && entry->completed_at_ts != 0) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PENDING_X_DENY_INVALID; + } + + dedup_pending_x_install_denial(entry); + *denied_out = *entry; + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PENDING_X_DENY_NEW; +} + +bool +cluster_gcs_block_dedup_set_request_flags_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, uint8 transition_id, + uint8 request_flags) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + bool found = false; + bool updated = false; + + Assert(key != NULL); + Assert(tag != NULL); + if (key == NULL || tag == NULL || (request_flags & ~GCS_BLOCK_DEDUP_REQUEST_F_VALID_MASK) != 0) + return false; + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return false; + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (found && entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_GENERIC + && memcmp(&entry->tag, tag, sizeof(*tag)) == 0 && entry->transition_id == transition_id) { + uint8 pinned_flags = GCS_BLOCK_DEDUP_REQUEST_F_PINNED | request_flags; + + if (entry->request_flags == 0) + entry->request_flags = pinned_flags; + updated = entry->request_flags == pinned_flags; + } else if (found && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC) + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return updated; +} + GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_reserve(int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, @@ -710,13 +964,21 @@ cluster_gcs_block_dedup_pcm_x_reserve(int worker_id, const GcsBlockDedupKey *key LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); if (found) { + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED + && !dedup_pcm_x_entry_drained_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } if ((entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED || entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE - || entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) + || entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED + || entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) && entry->transition_id == (uint8)PCM_TRANS_N_TO_S && memcmp(&entry->tag, tag, sizeof(*tag)) == 0 && dedup_pcm_x_reservation_equal(entry, reserved_binding)) { - dedup_pcm_x_work_pending[worker_id] = true; + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) + dedup_pcm_x_work_pending[worker_id] = true; LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_DUPLICATE; } @@ -742,6 +1004,7 @@ cluster_gcs_block_dedup_pcm_x_reserve(int worker_id, const GcsBlockDedupKey *key entry->transition_id = (uint8)PCM_TRANS_N_TO_S; entry->entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED; entry->pcm_x_master_session = reserved_binding->master_session; + entry->pinned_lifetime_us = (int64)reserved_binding->required_page_scn; entry->payload_meta.pcm_x_identity = reserved_binding->identity; entry->registered_at_ts = GetCurrentTimestamp(); pg_atomic_fetch_add_u32(&shard->entry_count, 1); @@ -754,6 +1017,7 @@ GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_materialize(int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, const GcsBlockPcmXImageBinding *ready_binding, + uint64 reservation_token, uint8 source_pcm_state, const GcsBlockReplyHeader *reply_header, const char *block_data) { @@ -765,7 +1029,8 @@ cluster_gcs_block_dedup_pcm_x_materialize(int worker_id, const GcsBlockDedupKey shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); if (shard == NULL) return GCS_BLOCK_PCM_X_IMAGE_FULL; - if (!dedup_pcm_x_ready_payload_valid(key, tag, ready_binding, reply_header, block_data)) { + if (reservation_token == 0 || !dedup_pcm_x_source_state_valid(source_pcm_state) + || !dedup_pcm_x_ready_payload_valid(key, tag, ready_binding, reply_header, block_data)) { dedup_pcm_x_note_failclosed(shard); return GCS_BLOCK_PCM_X_IMAGE_INVALID; } @@ -781,12 +1046,14 @@ cluster_gcs_block_dedup_pcm_x_materialize(int worker_id, const GcsBlockDedupKey || entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) { GcsBlockPcmXImageBinding stored_binding; - stored_binding.identity = entry->payload_meta.pcm_x_identity; - stored_binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &stored_binding); if (entry->transition_id == (uint8)PCM_TRANS_N_TO_S && memcmp(&entry->tag, tag, sizeof(*tag)) == 0 && GcsBlockPcmXImageBindingEqual(&stored_binding, ready_binding) && dedup_pcm_x_entry_payload_valid(key, tag, entry) + && (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + || (dedup_pcm_x_reservation_token_get(entry) == reservation_token + && entry->request_flags == source_pcm_state)) && memcmp(&entry->reply_header, reply_header, sizeof(*reply_header)) == 0 && memcmp(entry->block_data, block_data, GCS_BLOCK_DATA_SIZE) == 0) { LWLockRelease(&shard->lock.lock); @@ -811,17 +1078,20 @@ cluster_gcs_block_dedup_pcm_x_materialize(int worker_id, const GcsBlockDedupKey entry->reply_header = *reply_header; entry->status = (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER; + entry->request_flags = source_pcm_state; + dedup_pcm_x_reservation_token_set(entry, reservation_token); entry->payload_meta.pcm_x_identity = ready_binding->identity; memcpy(entry->block_data, block_data, GCS_BLOCK_DATA_SIZE); entry->completed_at_ts = GetCurrentTimestamp(); - stored_binding.identity = entry->payload_meta.pcm_x_identity; - stored_binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &stored_binding); if (!dedup_pcm_x_ready_payload_valid(key, tag, &stored_binding, &entry->reply_header, entry->block_data)) { memset(&entry->reply_header, 0, sizeof(entry->reply_header)); memset(entry->block_data, 0, GCS_BLOCK_DATA_SIZE); entry->payload_meta.pcm_x_identity = reserved_identity; entry->status = 0; + entry->request_flags = 0; + dedup_pcm_x_reservation_token_set(entry, 0); entry->completed_at_ts = 0; dedup_pcm_x_note_failclosed(shard); LWLockRelease(&shard->lock.lock); @@ -829,6 +1099,7 @@ cluster_gcs_block_dedup_pcm_x_materialize(int worker_id, const GcsBlockDedupKey } } entry->entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED; + dedup_pcm_x_work_pending[worker_id] = true; pg_atomic_fetch_add_u64(&shard->pcm_x_stage_count, 1); LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_STORED; @@ -861,8 +1132,7 @@ cluster_gcs_block_dedup_pcm_x_publish_ready_exact(int worker_id, const GcsBlockD LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; } - stored_binding.identity = entry->payload_meta.pcm_x_identity; - stored_binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &stored_binding); if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 || !GcsBlockPcmXImageBindingEqual(&stored_binding, ready_binding) @@ -882,6 +1152,8 @@ cluster_gcs_block_dedup_pcm_x_publish_ready_exact(int worker_id, const GcsBlockD } entry->entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE; + entry->request_flags = 0; + dedup_pcm_x_reservation_token_set(entry, 0); dedup_pcm_x_work_pending[worker_id] = true; LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_STORED; @@ -920,6 +1192,19 @@ cluster_gcs_block_dedup_pcm_x_lookup(int worker_id, const GcsBlockDedupKey *key, LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) { + dedup_pcm_x_binding_from_entry(entry, &stored_binding); + if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 + || !GcsBlockPcmXImageBindingEqual(&stored_binding, expected_binding) + || !dedup_pcm_x_entry_drained_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; + } if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 || (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED @@ -929,8 +1214,7 @@ cluster_gcs_block_dedup_pcm_x_lookup(int worker_id, const GcsBlockDedupKey *key, LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_STALE; } - stored_binding.identity = entry->payload_meta.pcm_x_identity; - stored_binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &stored_binding); if (!GcsBlockPcmXImageBindingEqual(&stored_binding, expected_binding)) { dedup_pcm_x_note_failclosed(shard); LWLockRelease(&shard->lock.lock); @@ -963,10 +1247,59 @@ cluster_gcs_block_dedup_pcm_x_lookup(int worker_id, const GcsBlockDedupKey *key, return GCS_BLOCK_PCM_X_IMAGE_REPLAY; } +GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_drain_status_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *binding) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + GcsBlockPcmXImageBinding stored_binding; + bool found = false; + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PCM_X_IMAGE_FULL; + if (!dedup_pcm_x_binding_valid(key, tag, binding, false)) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + + LWLockAcquire(&shard->lock.lock, LW_SHARED); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (!found) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; + } + dedup_pcm_x_binding_from_entry(entry, &stored_binding); + if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 + || !GcsBlockPcmXImageBindingEqual(&stored_binding, binding) + || (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) + || (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + ? !dedup_pcm_x_entry_ready_valid(key, tag, entry) + : !dedup_pcm_x_entry_drained_valid(key, tag, entry))) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) { + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_DUPLICATE; + } + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_READY; +} + + GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_release_exact(int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, - const GcsBlockPcmXImageBinding *binding) + const GcsBlockPcmXImageBinding *binding, + int32 drained_master_node) { ClusterGcsBlockDedupShard *shard; HTAB *htab = NULL; @@ -985,6 +1318,10 @@ cluster_gcs_block_dedup_pcm_x_release_exact(int worker_id, const GcsBlockDedupKe dedup_pcm_x_note_failclosed(shard); return GCS_BLOCK_PCM_X_IMAGE_INVALID; } + if (drained_master_node < -1 || drained_master_node >= PCM_X_PROTOCOL_NODE_LIMIT) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); @@ -993,21 +1330,62 @@ cluster_gcs_block_dedup_pcm_x_release_exact(int worker_id, const GcsBlockDedupKe LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; } - stored_binding.identity = entry->payload_meta.pcm_x_identity; - stored_binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &stored_binding); if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 || (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE - && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) || !GcsBlockPcmXImageBindingEqual(&stored_binding, binding)) { dedup_pcm_x_note_failclosed(shard); LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_STALE; } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) { + if (!dedup_pcm_x_entry_drained_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + if (drained_master_node < 0 || entry->request_flags != (uint8)(drained_master_node + 1)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_DUPLICATE; + } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE) { + if (drained_master_node < 0) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + if (!dedup_pcm_x_entry_ready_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + entry->entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED; + entry->request_flags = (uint8)(drained_master_node + 1); + entry->done_at_ts = 0; + pg_atomic_fetch_add_u64(&shard->pcm_x_release_count, 1); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_RELEASED; + } + if (drained_master_node != -1) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } (void)hash_search(htab, key, HASH_REMOVE, &found); - Assert(found); + if (!found) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } pg_atomic_fetch_sub_u32(&shard->entry_count, 1); pg_atomic_fetch_add_u64(&shard->pcm_x_release_count, 1); LWLockRelease(&shard->lock.lock); @@ -1015,13 +1393,111 @@ cluster_gcs_block_dedup_pcm_x_release_exact(int worker_id, const GcsBlockDedupKe } +bool +cluster_gcs_block_dedup_pcm_x_retire_up_to(uint64 cluster_epoch, int32 authenticated_master_node, + uint64 authenticated_master_session, + uint64 retire_through_ticket_id) +{ + int s; + + if (authenticated_master_node < 0 || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_master_session == 0 || retire_through_ticket_id == 0 + || cluster_gcs_block_dedup_shards == NULL) + return false; + + for (s = 0; s < cluster_gcs_block_dedup_n_shards; s++) { + ClusterGcsBlockDedupShard *shard = &cluster_gcs_block_dedup_shards[s]; + HTAB *htab = cluster_gcs_block_dedup_htabs[s]; + HASH_SEQ_STATUS scan; + GcsBlockDedupEntry *entry; + int removed = 0; + + if (htab == NULL) + continue; + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + hash_seq_init(&scan, htab); + while ((entry = (GcsBlockDedupEntry *)hash_seq_search(&scan)) != NULL) { + const PcmXTicketRef *ref; + + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) + continue; + if (!dedup_pcm_x_entry_drained_valid(&entry->key, &entry->tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + hash_seq_term(&scan); + LWLockRelease(&shard->lock.lock); + return false; + } + ref = &entry->payload_meta.pcm_x_identity.ref; + if (ref->identity.cluster_epoch == cluster_epoch + && entry->request_flags == (uint8)(authenticated_master_node + 1) + && entry->pcm_x_master_session == authenticated_master_session + && ref->handle.ticket_id <= retire_through_ticket_id) { + (void)hash_search(htab, &entry->key, HASH_REMOVE, NULL); + removed++; + } + } + if (removed > 0) + pg_atomic_fetch_sub_u32(&shard->entry_count, (uint32)removed); + LWLockRelease(&shard->lock.lock); + } + return true; +} + + +GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_preserve_finish_error_exact( + int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, + const GcsBlockPcmXImageBinding *binding, uint64 reservation_token, uint8 source_pcm_state) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + GcsBlockPcmXImageBinding stored_binding; + bool found = false; + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PCM_X_IMAGE_FULL; + if (reservation_token == 0 || !dedup_pcm_x_source_state_valid(source_pcm_state) + || !dedup_pcm_x_binding_valid(key, tag, binding, false)) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + + LWLockAcquire(&shard->lock.lock, LW_SHARED); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (!found) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; + } + dedup_pcm_x_binding_from_entry(entry, &stored_binding); + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED + || entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 + || !GcsBlockPcmXImageBindingEqual(&stored_binding, binding) + || !dedup_pcm_x_entry_payload_valid(key, tag, entry) + || dedup_pcm_x_reservation_token_get(entry) != reservation_token + || entry->request_flags != source_pcm_state) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING; +} + + static void dedup_pcm_x_copy_work(const GcsBlockDedupEntry *entry, GcsBlockPcmXImageWork *work) { memset(work, 0, sizeof(*work)); work->key = entry->key; - work->binding.identity = entry->payload_meta.pcm_x_identity; - work->binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &work->binding); + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) { + work->reservation_token = dedup_pcm_x_reservation_token_get(entry); + work->source_pcm_state = entry->request_flags; + } work->tag = entry->tag; work->entry_kind = entry->entry_kind; } @@ -1060,11 +1536,11 @@ dedup_pcm_x_consider_work(const GcsBlockDedupEntry *entry, const GcsBlockDedupKe } -/* Return at most one immutable work token per LMS tick. RESERVED wins the - * first mixed-class tick, then mixed-class selections alternate so neither - * fresh byte materialization nor READY replay can starve. Within each class - * a process-local exact-key cursor prevents one pinned holder from - * monopolizing every tick. */ +/* Return at most one immutable work token per LMS tick. Admission work + * includes fresh RESERVED entries and commit-only retries for immutable + * MATERIALIZED_UNCOMMITTED entries; the common exact-key cursor prevents a + * contended commit from monopolizing the worker. Admission wins the first + * mixed-class tick, then selections alternate with READY replay. */ GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_next_work(int worker_id, GcsBlockPcmXImageWork *work_out) { @@ -1099,24 +1575,29 @@ cluster_gcs_block_dedup_pcm_x_next_work(int worker_id, GcsBlockPcmXImageWork *wo hash_seq_init(&scan, htab); while ((entry = (GcsBlockDedupEntry *)hash_seq_search(&scan)) != NULL) { if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED - && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE) + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) continue; /* A READY entry admitted to the outbound ring sleeps until an exact * type-49 retransmit clears this marker. */ if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE && entry->done_at_ts != 0) continue; - binding.identity = entry->payload_meta.pcm_x_identity; - binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &binding); if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S || (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED && !dedup_pcm_x_binding_valid(&entry->key, &entry->tag, &binding, true)) + || (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED + && (!dedup_pcm_x_entry_payload_valid(&entry->key, &entry->tag, entry) + || dedup_pcm_x_reservation_token_get(entry) == 0 + || !dedup_pcm_x_source_state_valid(entry->request_flags))) || (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE && !dedup_pcm_x_entry_ready_valid(&entry->key, &entry->tag, entry))) { dedup_pcm_x_note_failclosed(shard); result = GCS_BLOCK_PCM_X_IMAGE_INVALID; break; } - if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED) { + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED + || entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) { dedup_pcm_x_consider_work(entry, &dedup_pcm_x_work_cursor[worker_id], dedup_pcm_x_work_cursor_valid[worker_id], &reserved_after, &have_reserved_after, &reserved_wrap, &have_reserved_wrap); @@ -1136,7 +1617,9 @@ cluster_gcs_block_dedup_pcm_x_next_work(int worker_id, GcsBlockPcmXImageWork *wo if (have_reserved && !choose_ready) { *work_out = have_reserved_after ? reserved_after : reserved_wrap; - result = GCS_BLOCK_PCM_X_IMAGE_RESERVED; + result = work_out->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED + ? GCS_BLOCK_PCM_X_IMAGE_RESERVED + : GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING; } else if (have_ready) { *work_out = have_ready_after ? ready_after : ready_wrap; result = GCS_BLOCK_PCM_X_IMAGE_REPLAY; @@ -1182,8 +1665,7 @@ cluster_gcs_block_dedup_pcm_x_mark_staged_exact(int worker_id, const GcsBlockDed LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; } - stored_binding.identity = entry->payload_meta.pcm_x_identity; - stored_binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &stored_binding); if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE || entry->transition_id != (uint8)PCM_TRANS_N_TO_S || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 @@ -1232,8 +1714,7 @@ cluster_gcs_block_dedup_pcm_x_unmark_staged_exact(int worker_id, const GcsBlockD LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; } - stored_binding.identity = entry->payload_meta.pcm_x_identity; - stored_binding.master_session = entry->pcm_x_master_session; + dedup_pcm_x_binding_from_entry(entry, &stored_binding); if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE || entry->transition_id != (uint8)PCM_TRANS_N_TO_S || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 @@ -1282,7 +1763,8 @@ cluster_gcs_block_dedup_pcm_x_rearm_exact(int worker_id, const GcsBlockDedupKey return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; } if ((entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED - && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE) + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) || entry->transition_id != (uint8)PCM_TRANS_N_TO_S || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 || !dedup_pcm_x_reservation_equal(entry, reserved_binding)) { @@ -1295,6 +1777,15 @@ cluster_gcs_block_dedup_pcm_x_rearm_exact(int worker_id, const GcsBlockDedupKey LWLockRelease(&shard->lock.lock); return GCS_BLOCK_PCM_X_IMAGE_NOT_READY; } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED) { + if (!dedup_pcm_x_entry_drained_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_READY; + } if (!dedup_pcm_x_entry_ready_valid(key, tag, entry)) { dedup_pcm_x_note_failclosed(shard); LWLockRelease(&shard->lock.lock); @@ -1462,6 +1953,14 @@ cluster_gcs_block_dedup_install_reply_ex(int worker_id, const GcsBlockDedupKey * LWLockRelease(&shard->lock.lock); return; } + /* A queue-kind pending-X claim has already terminated this exact legacy + * reader. Any asynchronous old producer has permanently lost the right + * to restore GRANTED/FORWARD/page bytes; preserve the cached denial for + * retransmit until exact DONE. */ + if (dedup_pending_x_denial_is_exact(entry)) { + LWLockRelease(&shard->lock.lock); + return; + } entry->status = (uint8)status; entry->reply_header = *header; diff --git a/src/backend/cluster/cluster_gcs_block_shard.c b/src/backend/cluster/cluster_gcs_block_shard.c index 73d1715359..e957e8df4c 100644 --- a/src/backend/cluster/cluster_gcs_block_shard.c +++ b/src/backend/cluster/cluster_gcs_block_shard.c @@ -142,14 +142,24 @@ cluster_gcs_block_payload_shard(uint8 msg_type, const void *payload, uint16 payl pcm_x_expected_len = sizeof(PcmXBlockerChunkPayload); break; case PGRAC_IC_MSG_PCM_X_REVOKE: - pcm_x_expected_len = sizeof(PcmXRevokePayload); + /* Source-floor V2 appends one SCN to the byte-identical V1 prefix. */ + if (payload_len != sizeof(PcmXRevokePayload) && payload_len != sizeof(PcmXRevokePayloadV2)) + return -1; + memcpy(&pcm_x_tag, payload, sizeof(pcm_x_tag)); + tag = &pcm_x_tag; break; case PGRAC_IC_MSG_PCM_X_IMAGE_READY: case PGRAC_IC_MSG_PCM_X_PREPARE_GRANT: pcm_x_expected_len = sizeof(PcmXGrantPayload); break; case PGRAC_IC_MSG_PCM_X_INSTALL_READY: - pcm_x_expected_len = sizeof(PcmXInstallReadyPayload); + /* A' rebase: the V1 104-byte and V2 112-byte exact frames are both + * legal; the tag prefix is identical, so the shard key is too. */ + if (payload_len != sizeof(PcmXInstallReadyPayload) + && payload_len != PCM_X_INSTALL_READY_V1_LEN) + return -1; + memcpy(&pcm_x_tag, payload, sizeof(pcm_x_tag)); + tag = &pcm_x_tag; break; case PGRAC_IC_MSG_PCM_X_FINAL_ACK: pcm_x_expected_len = sizeof(PcmXFinalAckPayload); diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 5232cf7633..9999046168 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -800,8 +800,9 @@ int cluster_gcs_block_recovery_wait_ms = 200; * DENIED_INVALIDATE_TIMEOUT reply (status 11) → 53R91 SQLSTATE. * cluster_gcs_block_starvation_backoff_ms (HC117): reader backoff * base for DENIED_PENDING_X retry loop; backoff = base × 2^attempt. - * cluster_gcs_block_starvation_max_retries (HC117): reader retry - * budget; exhaustion → 53R92 SQLSTATE. + * cluster_gcs_block_starvation_max_retries (HC117): retained as a + * configuration-compatibility surface. Queue-arbitrated reader retry is + * now unbounded/cancelable and does not consume this legacy budget. */ int cluster_gcs_block_invalidate_ack_timeout_ms = 1500; int cluster_gcs_block_starvation_backoff_ms = 100; @@ -4204,10 +4205,12 @@ cluster_init_guc(void) "2^attempt. HC117. PGC_SUSET."), &cluster_gcs_block_starvation_backoff_ms, 100, 1, 60000, PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomIntVariable( - "cluster.gcs_block_starvation_max_retries", gettext_noop("S barrier reader retry budget."), - gettext_noop("Reader DENIED_PENDING_X retry budget. Budget exhausted → " - "ereport(53R92); upper-layer transaction may retry the " - "whole statement. HC117. PGC_SUSET."), + "cluster.gcs_block_starvation_max_retries", + gettext_noop("Legacy S barrier reader retry budget compatibility setting."), + gettext_noop("Retained for configuration compatibility. DENIED_PENDING_X now " + "exact-aborts its old reservation and retries with fresh identities " + "until success or query cancellation; this value no longer causes " + "a client error. HC117. PGC_SUSET."), &cluster_gcs_block_starvation_max_retries, 8, 0, 64, PGC_SUSET, 0, NULL, NULL, NULL); /* diff --git a/src/backend/cluster/cluster_ic.c b/src/backend/cluster/cluster_ic.c index 738681173e..86a3dc6e91 100644 --- a/src/backend/cluster/cluster_ic.c +++ b/src/backend/cluster/cluster_ic.c @@ -663,6 +663,12 @@ cluster_ic_build_hello(uint8 out_buf[PGRAC_IC_HELLO_BYTES], uint16 hello_version /* PCM-X conversion is a protocol capability. Every opcode has a distinct * registered DATA-plane type, so mixed-version senders gate on this bit. */ capabilities |= PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1; + /* A' rebase: V2 INSTALL_READY understanding, same unconditional protocol + * discipline; activation is formation-wide (see cluster_ic.h). */ + capabilities |= PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1; + /* P0-20: V2 REVOKE source-floor carrier. Senders still gate the extended + * frame on the selected peer's current verified connection. */ + capabilities |= PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1; if (capabilities != 0) ic_le_write_uint32(out_buf + PGRAC_IC_HELLO_CAPABILITIES_OFFSET, capabilities); diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index c040c0fe98..febcc74332 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -538,8 +538,7 @@ static ClusterInjectPoint cluster_injection_points[] = { * * cluster-gcs-block-starvation-force-denied: * Fires inside master N→S handler. SKIP forces DENIED_PENDING_X - * reply unconditionally so TAP can drive reader backoff retry + - * 53R92 ERRCODE_CLUSTER_GCS_BLOCK_STARVATION_EXHAUSTED. + * reply so TAP can drive exact abort + fresh-identity backoff retry. * * cluster-catalog-services-ready-force-closed: * Fires inside cluster_catalog_services_ready (spec-6.14 D8). SKIP @@ -607,6 +606,17 @@ static ClusterInjectPoint cluster_injection_points[] = { */ { .name = "cluster-pcm-writer-cached-x-stall" }, { .name = "cluster-pcm-restore-aba-window" }, + /* + * PCM-X retained-image finish FlushBuffer boundary (t/400). + * + * The finish path first proves its exact REVOKING lifecycle while holding + * caller pin + content EXCLUSIVE. An armed point makes the unchanged page + * dirty so this otherwise race-dependent FlushBuffer leg is deterministic. + * The point itself fires only inside that exact FlushBuffer call, just + * before smgrwrite; SKIP_N raises the fail-closed test ERROR there, while + * SLEEP:0 supplies a success-path observation without changing behavior. + */ + { .name = "cluster-pcm-x-retain-flush-error" }, /* * cluster-pcm-grant-finalize-window (W3): * Fires in LockBuffer AFTER a real X acquire installed the grant and @@ -1273,6 +1283,18 @@ cluster_injection_should_skip(const char *name) } +/* Read one point's armed state without dispatching it or consuming SKIP_N. */ +bool +cluster_injection_is_armed(const char *name) +{ + ClusterInjectPoint *p; + + cluster_injection_initialise(); + p = cluster_injection_lookup(name); + return p != NULL && pg_atomic_read_u32(&p->armed_type) != (uint32)CLUSTER_FAULT_NONE; +} + + /* * cluster_cr_injection_armed -- spec-3.9 D7 CR-specific armed-state peek. * diff --git a/src/backend/cluster/cluster_lms.c b/src/backend/cluster/cluster_lms.c index bebf5a5566..a42628e9e1 100644 --- a/src/backend/cluster/cluster_lms.c +++ b/src/backend/cluster/cluster_lms.c @@ -61,8 +61,9 @@ #include "cluster/cluster_cr_server.h" /* spec-6.12b CR work slots */ #include "cluster/cluster_conf.h" #include "cluster/cluster_cssd.h" -#include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ -#include "cluster/cluster_gcs_block.h" /* PGRAC: spec-7.2 D4 plane probe + pi drain */ +#include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ +#include "cluster/cluster_gcs_block.h" /* PGRAC: spec-7.2 D4 plane probe + pi drain */ +#include "cluster/cluster_inject.h" #include "cluster/cluster_write_fence.h" /* PGRAC: spec-7.2 D5 fence linkage */ #include "cluster/cluster_ges.h" #include "cluster/cluster_ges_dedup.h" @@ -211,6 +212,7 @@ cluster_lms_shmem_init(void) pg_atomic_init_u64(&cluster_lms_state->worker_inline_serve_count[w], 0); pg_atomic_init_u64(&cluster_lms_state->worker_outbound_not_admitted_count[w], 0); pg_atomic_init_u64(&cluster_lms_state->worker_outbound_requeue_drop_count[w], 0); + pg_atomic_init_u64(&cluster_lms_state->worker_outbound_cap_guard_drop_count[w], 0); for (b = 0; b < CLUSTER_LMS_SERVE_HIST_BUCKETS; b++) pg_atomic_init_u64(&cluster_lms_state->worker_serve_hist[w][b], 0); } @@ -699,6 +701,48 @@ lms_sigusr1_handler(SIGNAL_ARGS) SetLatch(MyLatch); } +/* Report when this DATA worker has applied the process-local finish-Flush + * injection state. The transition log is the reload acknowledgement used by + * t/400; without it, a fixed sleep can race a worker that has not processed + * SIGHUP yet and turn a fault-injection leg into a false negative. */ +static void +lms_note_pcm_x_finish_flush_injection_reload(int worker_id) +{ + static bool was_armed = false; + bool armed; + + armed = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); + if (!armed && !was_armed) + return; + ereport(LOG, + (errmsg_internal("cluster_lms: DATA worker=%d applied PCM-X finish Flush " + "injection config: pid=%d armed=%s value=\"%s\"", + worker_id, (int)MyProcPid, armed ? "true" : "false", + cluster_injection_points != NULL ? cluster_injection_points : ""))); + was_armed = armed; +} + + +void +cluster_lms_note_pcm_x_image_ready_boundary(uint8 msg_type, const char *boundary, int result, + int runtime_state, bool fence_enforcing, + bool fence_allowed, uint32 dest_node_id, + uint64 request_id, uint64 ticket_id, + uint64 grant_generation, uint64 image_id) +{ + if (boundary == NULL || !cluster_injection_is_armed("cluster-pcm-x-retain-flush-error")) + return; + ereport( + LOG, + (errmsg_internal("PCM-X IMAGE_READY transport boundary: %s", boundary), + errdetail("msg_type=%u result=%d runtime=%d fence_enforcing=%s fence_allowed=%s dest=%u " + "request_id=%llu ticket=%llu grant_generation=%llu image_id=%llu", + (unsigned)msg_type, result, runtime_state, fence_enforcing ? "true" : "false", + fence_allowed ? "true" : "false", dest_node_id, (unsigned long long)request_id, + (unsigned long long)ticket_id, (unsigned long long)grant_generation, + (unsigned long long)image_id))); +} + void LmsMain(void) { @@ -785,6 +829,7 @@ LmsMain(void) if (ConfigReloadPending) { ConfigReloadPending = false; ProcessConfigFile(PGC_SIGHUP); + lms_note_pcm_x_finish_flush_injection_reload(0); /* PGRAC: spec-7.3 D8 — re-apply cluster.lms_nice on reload. */ cluster_lms_apply_nice(); } @@ -956,6 +1001,7 @@ LmsWorkerMain(int worker_id) if (ConfigReloadPending) { ConfigReloadPending = false; ProcessConfigFile(PGC_SIGHUP); + lms_note_pcm_x_finish_flush_injection_reload(worker_id); /* PGRAC: spec-7.3 D8 — re-apply cluster.lms_nice on reload. */ cluster_lms_apply_nice(); } @@ -1123,6 +1169,14 @@ cluster_lms_obs_note_outbound_requeue_drop(int worker_id) pg_atomic_fetch_add_u64(&cluster_lms_state->worker_outbound_requeue_drop_count[worker_id], 1); } +void +cluster_lms_obs_note_outbound_cap_guard_drop(int worker_id) +{ + if (cluster_lms_state == NULL || worker_id < 0 || worker_id >= CLUSTER_LMS_MAX_WORKERS) + return; + pg_atomic_fetch_add_u64(&cluster_lms_state->worker_outbound_cap_guard_drop_count[worker_id], 1); +} + uint64 cluster_lms_obs_get_outbound_not_admitted(int worker_id) { @@ -1139,6 +1193,14 @@ cluster_lms_obs_get_outbound_requeue_drop(int worker_id) : lms_obs_read(cluster_lms_state->worker_outbound_requeue_drop_count, worker_id); } +uint64 +cluster_lms_obs_get_outbound_cap_guard_drop(int worker_id) +{ + return cluster_lms_state == NULL + ? 0 + : lms_obs_read(cluster_lms_state->worker_outbound_cap_guard_drop_count, worker_id); +} + uint64 cluster_lms_obs_get_dispatch_count(int worker_id) { diff --git a/src/backend/cluster/cluster_lms_outbound.c b/src/backend/cluster/cluster_lms_outbound.c index db9697fe54..a6bff9ad08 100644 --- a/src/backend/cluster/cluster_lms_outbound.c +++ b/src/backend/cluster/cluster_lms_outbound.c @@ -43,6 +43,7 @@ #include "cluster/cluster_ic_router.h" #include "cluster/cluster_lms.h" #include "cluster/cluster_shmem.h" +#include "cluster/cluster_sf_dep.h" #include "cluster/cluster_write_fence.h" #include "miscadmin.h" #include "storage/lwlock.h" @@ -54,13 +55,33 @@ #define PGRAC_LMS_OUTBOUND_CAPACITY 256 #define PGRAC_LMS_OUTBOUND_PAYLOAD_MAX 128 +typedef enum ClusterLmsOutboundKind { + CLUSTER_LMS_OUTBOUND_FRAME = 0, + CLUSTER_LMS_OUTBOUND_ZERO_BLOCK_REPLY = 1, + CLUSTER_LMS_OUTBOUND_DIRECT_ZERO_BLOCK_REPLY = 2 +} ClusterLmsOutboundKind; + typedef struct ClusterLmsOutboundSlot { uint32 dest_node_id; uint8 msg_type; + uint8 kind; uint16 payload_len; + uint32 required_capability; + uint32 connection_generation; uint8 payload[PGRAC_LMS_OUTBOUND_PAYLOAD_MAX]; } ClusterLmsOutboundSlot; +StaticAssertDecl(sizeof(ClusterLmsOutboundSlot) == 144, + "LMS outbound slot capability guard layout changed"); + +typedef struct ClusterLmsZeroBlockReplyWire { + GcsBlockReplyHeader header; + char block_data[GCS_BLOCK_DATA_SIZE]; +} ClusterLmsZeroBlockReplyWire; + +StaticAssertDecl(sizeof(ClusterLmsZeroBlockReplyWire) == GCS_BLOCK_REPLY_PAYLOAD_TOTAL_SIZE, + "staged zero-block reply must preserve the GCS reply wire size"); + typedef struct ClusterLmsOutboundState { uint32 head; /* next slot to fill */ uint32 tail; /* next slot to drain */ @@ -135,6 +156,31 @@ lms_outbound_pcm_x_grant_held(uint8 msg_type) return cluster_write_fence_enforcing() && !cluster_write_fence_allowed(); } + +static void +lms_outbound_pcm_x_image_ready_note(const ClusterLmsOutboundSlot *slot, const char *boundary, + int result) +{ + const PcmXGrantPayload *ready; + PcmXRuntimeSnapshot runtime; + bool fence_enforcing; + bool fence_allowed; + + if (slot == NULL || boundary == NULL + || (slot->msg_type != PGRAC_IC_MSG_PCM_X_IMAGE_READY + && slot->msg_type != PGRAC_IC_MSG_PCM_X_PREPARE_GRANT) + || slot->payload_len != sizeof(PcmXGrantPayload)) + return; + ready = (const PcmXGrantPayload *)slot->payload; + runtime = cluster_pcm_x_runtime_snapshot(); + fence_enforcing = cluster_write_fence_enforcing(); + fence_allowed = !fence_enforcing || cluster_write_fence_allowed(); + cluster_lms_note_pcm_x_image_ready_boundary( + slot->msg_type, boundary, result, (int)runtime.state, fence_enforcing, fence_allowed, + slot->dest_node_id, ready->ref.identity.request_id, ready->ref.handle.ticket_id, + ready->ref.grant_generation, ready->image.image_id); +} + void cluster_lms_outbound_shmem_register(void) { @@ -156,9 +202,10 @@ cluster_lms_outbound_request_lwlocks(void) * retry machinery). Publish-before-signal: the slot is visible * before the LMS wakeup fires. */ -bool -cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, - const void *payload, uint16 payload_len) +static bool +lms_outbound_enqueue_internal(int worker_id, uint8 msg_type, uint32 dest_node_id, + const void *payload, uint16 payload_len, uint32 required_capability, + uint32 connection_generation) { ClusterLmsOutboundState *ring; LWLock *lock; @@ -182,7 +229,10 @@ cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, slot = &ring->ring[ring->head]; slot->dest_node_id = dest_node_id; slot->msg_type = msg_type; + slot->kind = (uint8)CLUSTER_LMS_OUTBOUND_FRAME; slot->payload_len = payload_len; + slot->required_capability = required_capability; + slot->connection_generation = connection_generation; if (payload_len > 0) memcpy(slot->payload, payload, payload_len); ring->head = (ring->head + 1) % PGRAC_LMS_OUTBOUND_CAPACITY; @@ -193,6 +243,75 @@ cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, return true; } +bool +cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, + const void *payload, uint16 payload_len) +{ + return lms_outbound_enqueue_internal(worker_id, msg_type, dest_node_id, payload, payload_len, 0, + 0); +} + +/* Stage a wire-version-sensitive frame for one exact HELLO-authenticated + * connection. The reliable protocol leg remains outside this ring, so a + * drain-side guard failure may consume this stale copy and let the periodic + * producer reconstruct the correct wire version. */ +bool +cluster_lms_outbound_enqueue_cap_bound(int worker_id, uint8 msg_type, uint32 dest_node_id, + const void *payload, uint16 payload_len, + uint32 required_capability, uint32 connection_generation) +{ + if (required_capability == 0 || dest_node_id >= CLUSTER_MAX_NODES) + return false; + return lms_outbound_enqueue_internal(worker_id, msg_type, dest_node_id, payload, payload_len, + required_capability, connection_generation); +} + +/* + * Stage a header-only GCS denial from a CONTROL-plane producer. The DATA + * owner expands the ABI-mandated zero block immediately before transport + * admission. Keep this surface narrow: only the Shape-B pending-X denial + * may use it, and callers must choose worker[shard(tag)] so it stays on the + * same per-tag stream as the request it terminates. + */ +bool +cluster_lms_outbound_enqueue_zero_block_reply(int worker_id, uint32 dest_node_id, + const GcsBlockReplyHeader *header, bool direct_land) +{ + ClusterLmsOutboundState *ring; + LWLock *lock; + ClusterLmsOutboundSlot *slot; + + if (worker_id < 0 || worker_id >= CLUSTER_LMS_MAX_WORKERS || dest_node_id >= CLUSTER_MAX_NODES + || header == NULL || (direct_land && (int32)dest_node_id == cluster_node_id) + || header->status != (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X) + return false; + if (cluster_lms_outbound_rings == NULL || OB_LOCK(worker_id) == NULL) + return false; + + ring = OB_RING(worker_id); + lock = OB_LOCK(worker_id); + LWLockAcquire(lock, LW_EXCLUSIVE); + if (ring->count >= PGRAC_LMS_OUTBOUND_CAPACITY) { + LWLockRelease(lock); + return false; + } + slot = &ring->ring[ring->head]; + slot->dest_node_id = dest_node_id; + slot->msg_type = PGRAC_IC_MSG_GCS_BLOCK_REPLY; + slot->kind = (uint8)(direct_land ? CLUSTER_LMS_OUTBOUND_DIRECT_ZERO_BLOCK_REPLY + : CLUSTER_LMS_OUTBOUND_ZERO_BLOCK_REPLY); + slot->payload_len = sizeof(*header); + slot->required_capability = 0; + slot->connection_generation = 0; + memcpy(slot->payload, header, sizeof(*header)); + ring->head = (ring->head + 1) % PGRAC_LMS_OUTBOUND_CAPACITY; + ring->count++; + LWLockRelease(lock); + + cluster_lms_wakeup(worker_id); + return true; +} + /* * cluster_lms_outbound_drain_send — one worker drains + sends its own ring. * @@ -247,6 +366,9 @@ cluster_lms_outbound_drain_send(int worker_id) while (scanned < 64) { ClusterLmsOutboundSlot slot; + ClusterLmsZeroBlockReplyWire zero_reply; + const void *send_payload; + uint32 send_payload_len; ClusterICSendResult rc; bool got = false; @@ -261,10 +383,44 @@ cluster_lms_outbound_drain_send(int worker_id) if (!got) break; scanned++; + /* Wire-version-sensitive slots are valid only for the exact + * connection generation whose HELLO advertised the required bit. + * A drift consumes this stale ring copy without transport admission; + * no protocol ACK is generated, so the armed reliable leg retries. */ + if (slot.required_capability != 0 + && !cluster_sf_peer_capability_generation_matches( + (int32)slot.dest_node_id, slot.required_capability, slot.connection_generation)) { + lms_outbound_pcm_x_image_ready_note(&slot, "capability-guard", -1); + cluster_lms_obs_note_outbound_cap_guard_drop(worker_id); + continue; + } + send_payload = slot.payload_len > 0 ? slot.payload : NULL; + send_payload_len = slot.payload_len; + if (slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_ZERO_BLOCK_REPLY + || slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_DIRECT_ZERO_BLOCK_REPLY) { + if (slot.msg_type != PGRAC_IC_MSG_GCS_BLOCK_REPLY + || slot.payload_len != sizeof(GcsBlockReplyHeader)) { + rc = CLUSTER_IC_SEND_HARD_ERROR; + goto handle_send_result; + } + memset(&zero_reply, 0, sizeof(zero_reply)); + memcpy(&zero_reply.header, slot.payload, sizeof(zero_reply.header)); + if (zero_reply.header.status != (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X) { + rc = CLUSTER_IC_SEND_HARD_ERROR; + goto handle_send_result; + } + zero_reply.header.checksum = cluster_gcs_block_compute_checksum(zero_reply.block_data); + send_payload = &zero_reply; + send_payload_len = sizeof(zero_reply); + } else if (slot.kind != (uint8)CLUSTER_LMS_OUTBOUND_FRAME) { + rc = CLUSTER_IC_SEND_HARD_ERROR; + goto handle_send_result; + } /* A peer that refused a frame this batch keeps its later frames * queued BEHIND the refused one (per-peer order). */ if (slot.dest_node_id < CLUSTER_MAX_NODES && peer_blocked[slot.dest_node_id]) { + lms_outbound_pcm_x_image_ready_note(&slot, "peer-blocked", -2); Assert(n_retained < (int)lengthof(retained)); retained[n_retained++] = slot; continue; @@ -274,6 +430,7 @@ cluster_lms_outbound_drain_send(int worker_id) * transport admission. Retaining also blocks later same-peer frames, * preserving the DATA FIFO while unrelated peers keep flowing. */ if (lms_outbound_pcm_x_grant_held(slot.msg_type)) { + lms_outbound_pcm_x_image_ready_note(&slot, "grant-held", -3); if (slot.dest_node_id < CLUSTER_MAX_NODES) peer_blocked[slot.dest_node_id] = true; Assert(n_retained < (int)lengthof(retained)); @@ -296,21 +453,27 @@ cluster_lms_outbound_drain_send(int worker_id) * staged back onto this tag's ring, preserving the one-worker FIFO and * avoiding recursive handler execution. */ - if ((int32)slot.dest_node_id == cluster_node_id) { + if (slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_DIRECT_ZERO_BLOCK_REPLY) + rc = cluster_gcs_block_send_direct_zero_reply((int32)slot.dest_node_id, + &zero_reply.header); + else if ((int32)slot.dest_node_id == cluster_node_id) { ClusterICEnvelope env; - if (cluster_ic_envelope_build( - &env, slot.msg_type, (uint32)cluster_node_id, slot.dest_node_id, - slot.payload_len > 0 ? slot.payload : NULL, slot.payload_len) - && cluster_ic_dispatch_envelope(&env, slot.payload_len > 0 ? slot.payload : NULL, - cluster_node_id)) + if (cluster_ic_envelope_build(&env, slot.msg_type, (uint32)cluster_node_id, + slot.dest_node_id, send_payload, send_payload_len) + && cluster_ic_dispatch_envelope(&env, send_payload, cluster_node_id)) rc = CLUSTER_IC_SEND_DONE; else rc = CLUSTER_IC_SEND_HARD_ERROR; } else - rc = cluster_ic_send_envelope(slot.msg_type, (int32)slot.dest_node_id, - slot.payload_len > 0 ? slot.payload : NULL, - slot.payload_len); + rc = cluster_ic_send_envelope(slot.msg_type, (int32)slot.dest_node_id, send_payload, + send_payload_len); + + handle_send_result: + lms_outbound_pcm_x_image_ready_note(&slot, "send-result", (int)rc); + if (slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_ZERO_BLOCK_REPLY + || slot.kind == (uint8)CLUSTER_LMS_OUTBOUND_DIRECT_ZERO_BLOCK_REPLY) + cluster_gcs_block_note_send_outcome(GCS_BLOCK_SEND_FAMILY_REPLY, rc); switch (rc) { case CLUSTER_IC_SEND_DONE: case CLUSTER_IC_SEND_WOULD_BLOCK: diff --git a/src/backend/cluster/cluster_pcm_lock.c b/src/backend/cluster/cluster_pcm_lock.c index dd29642b4a..d80efcac8f 100644 --- a/src/backend/cluster/cluster_pcm_lock.c +++ b/src/backend/cluster/cluster_pcm_lock.c @@ -611,6 +611,29 @@ cluster_pcm_lock_authority_snapshot(BufferTag tag, PcmAuthoritySnapshot *out) return found && entry != NULL; } +bool +cluster_pcm_lock_authority_matches(BufferTag tag, const PcmAuthoritySnapshot *expected) +{ + struct GrdEntry *entry; + PcmAuthoritySnapshot current; + bool found; + bool matches = false; + + if (expected == NULL || ClusterPcm == NULL || cluster_pcm_htab == NULL) + return false; + + LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); + entry = (struct GrdEntry *)hash_search(cluster_pcm_htab, &tag, HASH_FIND, &found); + if (found && entry != NULL) { + LWLockAcquire(&entry->entry_lock.lock, LW_SHARED); + pcm_authority_snapshot_locked(entry, ¤t); + matches = pcm_authority_snapshot_equal(¤t, expected); + LWLockRelease(&entry->entry_lock.lock); + } + LWLockRelease(&ClusterPcm->htab_lock.lock); + return matches; +} + /* * Atomically install the queue winner as the sole X holder, but only if the * authority still exactly matches the snapshot captured when the transfer @@ -852,6 +875,8 @@ cluster_pcm_lock_clear_queue_pending_x_exact(BufferTag tag, int32 requester_node cleared = true; } LWLockRelease(&entry->entry_lock.lock); + if (cleared) + ConditionVariableBroadcast(&entry->wait_cv); } LWLockRelease(&ClusterPcm->htab_lock.lock); return cleared; @@ -862,6 +887,7 @@ cluster_pcm_lock_clear_pending_x(BufferTag tag) { struct GrdEntry *entry; bool found; + bool cleared = false; if (cluster_pcm_htab == NULL) return; @@ -870,11 +896,15 @@ cluster_pcm_lock_clear_pending_x(BufferTag tag) entry = (struct GrdEntry *)hash_search(cluster_pcm_htab, &tag, HASH_FIND, &found); if (found && entry != NULL) { LWLockAcquire(&entry->entry_lock.lock, LW_EXCLUSIVE); - if ((entry->pending_x_since_lsn & PCM_PENDING_X_QUEUE_KIND) == 0) { + if (entry->pending_x_requester_node != -1 + && (entry->pending_x_since_lsn & PCM_PENDING_X_QUEUE_KIND) == 0) { entry->pending_x_requester_node = -1; entry->pending_x_since_lsn = 0; + cleared = true; } LWLockRelease(&entry->entry_lock.lock); + if (cleared) + ConditionVariableBroadcast(&entry->wait_cv); } LWLockRelease(&ClusterPcm->htab_lock.lock); } @@ -911,6 +941,8 @@ cluster_pcm_lock_clear_pending_x_if(BufferTag tag, int32 expected_requester) cleared = true; } LWLockRelease(&entry->entry_lock.lock); + if (cleared) + ConditionVariableBroadcast(&entry->wait_cv); } LWLockRelease(&ClusterPcm->htab_lock.lock); return cleared; @@ -1072,6 +1104,8 @@ cluster_pcm_lock_clear_pending_x_for_node(int32 dead_node) LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); hash_seq_init(&scan, cluster_pcm_htab); while ((entry = (struct GrdEntry *)hash_seq_search(&scan)) != NULL) { + bool cleared_entry = false; + /* Fast SHARED read first — most entries will not match. */ if (entry->pending_x_requester_node != dead_node) continue; @@ -1084,8 +1118,11 @@ cluster_pcm_lock_clear_pending_x_for_node(int32 dead_node) entry->pending_x_requester_node = -1; entry->pending_x_since_lsn = 0; cleared++; + cleared_entry = true; } LWLockRelease(&entry->entry_lock.lock); + if (cleared_entry) + ConditionVariableBroadcast(&entry->wait_cv); } LWLockRelease(&ClusterPcm->htab_lock.lock); @@ -1410,67 +1447,77 @@ cluster_gcs_block_master_rebuild_from_redeclare(BufferTag tag, uint8 held_mode, /* - * PGRAC: spec-5.2 D11 — local-master record self as the new X holder after a - * writer-transfer (revoke). - * - * THIS node is the GCS master for the block. A remote node held it in X; we - * (the master + the writing requester) forwarded an X-transfer request, the - * holder shipped its current image AND released its own X (invalidating its - * local copy so it can never flush a stale page — Rule 8.A no-stale-flush), - * and we just installed the shipped bytes under content_lock EXCLUSIVE. Now - * make the authoritative master GRD entry reflect the new ownership: X held - * by self, no S holders, PI watermark advanced to the shipped page_lsn. + * P0-26: exact local-master X-transfer commit. * - * The old holder released BEFORE this point (its forward handler dropped its - * copy before replying X_GRANTED_FROM_HOLDER), so there is never a window with - * two X holders; there is at most a brief no-holder window, which is safe. - * Mirrors the X branch of cluster_gcs_block_master_rebuild_from_redeclare. + * The remote holder drops its X before replying, but the local master keeps + * the sampled remote-X authority until this barrier. Requiring the complete + * entry-lock snapshot prevents a late reply from overwriting a newer queue + * winner, holder session, or generation. STALE performs no mutation. */ -void -cluster_pcm_lock_master_take_x_after_transfer(BufferTag tag, XLogRecPtr page_lsn, SCN page_scn, - int32 holder_node, uint64 request_id, uint64 epoch) +PcmXTransferCommitResult +cluster_pcm_lock_master_take_x_after_transfer(BufferTag tag, const PcmAuthoritySnapshot *expected, + XLogRecPtr page_lsn, SCN page_scn, int32 holder_node, + uint32 requester_procno, uint64 request_id, + uint64 epoch) { struct GrdEntry *entry; + PcmAuthoritySnapshot current; + ClusterGrdHolderId requester; + bool found; - if (cluster_pcm_htab == NULL) - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("PCM lock manager disabled (cluster.pcm_grd_max_entries=0)"))); + if (expected == NULL || holder_node < 0 || holder_node >= 32 || request_id == 0) + return PCM_X_TRANSFER_COMMIT_BAD_STATE; + if (ClusterPcm == NULL || cluster_pcm_htab == NULL) + return PCM_X_TRANSFER_COMMIT_NOT_FOUND; - entry = pcm_get_or_create_entry(tag); - if (entry == NULL) - ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("cluster_pcm_lock_master_take_x_after_transfer: PCM GRD HTAB " - "FULL (cap=%d)", - pcm_grd_effective))); + requester.node_id = (uint32)cluster_node_id; + requester.procno = requester_procno; + requester.cluster_epoch = epoch; + requester.request_id = request_id; + + LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); + entry = (struct GrdEntry *)hash_search(cluster_pcm_htab, &tag, HASH_FIND, &found); + if (!found || entry == NULL) { + LWLockRelease(&ClusterPcm->htab_lock.lock); + return PCM_X_TRANSFER_COMMIT_NOT_FOUND; + } + + LWLockAcquire(&entry->entry_lock.lock, LW_EXCLUSIVE); + pcm_authority_snapshot_locked(entry, ¤t); + if (!pcm_authority_snapshot_equal(¤t, expected)) { + LWLockRelease(&entry->entry_lock.lock); + LWLockRelease(&ClusterPcm->htab_lock.lock); + return PCM_X_TRANSFER_COMMIT_STALE; + } + if (current.state != PCM_STATE_X || current.x_holder_node != holder_node + || current.s_holders_bitmap != 0 || current.pending_x_requester_node != -1 + || current.master_holder.node_id != (uint32)holder_node) { + LWLockRelease(&entry->entry_lock.lock); + LWLockRelease(&ClusterPcm->htab_lock.lock); + return PCM_X_TRANSFER_COMMIT_BAD_STATE; + } - pcm_entry_lock_exclusive(entry); pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_X); entry->x_holder_node = cluster_node_id; pg_atomic_write_u32(&entry->s_holders_bitmap, 0); - /* HC110: keep master_holder coherent with the X holder so subsequent - * forward/ship decisions (and spec-5.2 D11 path-B detection of - * master==holder==self) read the right holder identity. */ - pcm_master_holder_set_node(entry, cluster_node_id); - /* spec-2.41 D2 (§2.8.1) — local-page source advances BOTH watermarks - * monotonically: lsn for the redo-coverage serve-gate, scn for the - * lost-write detector. page_scn comes from the installed page's - * pd_block_scn (InvalidScn-safe). */ + entry->s_holder_refcount_local = 0; + pcm_master_holder_set_exact(entry, &requester); if ((uint64)page_lsn > entry->pi_watermark_lsn) entry->pi_watermark_lsn = (uint64)page_lsn; - /* monotone-max by local_scn (scn_time_cmp order); a raw SCN compare would be - * node_id-dominated — see cluster_scn.h + gcs_block_lost_write_verdict. */ if (SCN_VALID(page_scn)) { SCN wm_old = entry->pi_watermark_scn; - if (scn_local(page_scn) - > scn_local(entry->pi_watermark_scn)) { /* SCN_CMP_OK: scn_time_cmp via scn_local */ + if (scn_local(page_scn) > scn_local(entry->pi_watermark_scn)) { entry->pi_watermark_scn = page_scn; - /* step 1b: record the shipping holder's wire identity on advance. */ pcm_wm_prov_record(tag, wm_old, page_scn, CLUSTER_PCM_WM_SRC_TAKE_X, holder_node, request_id, epoch); } } + entry->last_transition_at = GetCurrentTimestamp(); + pg_atomic_fetch_add_u64(&entry->transition_count_local, 1); LWLockRelease(&entry->entry_lock.lock); + LWLockRelease(&ClusterPcm->htab_lock.lock); + return PCM_X_TRANSFER_COMMIT_OK; } /* @@ -2127,16 +2174,31 @@ cluster_pcm_transition_apply(struct GrdEntry *entry, PcmLockTransition trans, in pg_atomic_fetch_add_u64(&entry->transition_count_local, 1); } +/* The pending-X cookie and S-holder bitmap are both protected by entry_lock. + * This is the final admission point for every local/remote S publication: + * existing S residency may re-enter without changing authority, but a new bit + * cannot cross any live writer barrier (including the same-node case). */ +static bool +pcm_s_admission_allowed_locked(struct GrdEntry *entry, uint32 holder_bit) +{ + Assert(entry != NULL); + Assert(LWLockHeldByMeInMode(&entry->entry_lock.lock, LW_EXCLUSIVE)); + return (pg_atomic_read_u32(&entry->s_holders_bitmap) & holder_bit) != 0 + || entry->pending_x_requester_node == -1; +} + /* * Apply a GCS-requested PCM transition on the master side. * - * Unlike the public local APIs, this helper returns false on state - * incompatibility so the GCS request handler can send a DENIED reply instead - * of raising ERROR and leaking the caller's reply wait. Caller is the GCS - * request handler; sender-side code must not call this after a GRANTED reply. + * Unlike the public local APIs, the exact helper distinguishes a raced + * pending-X barrier from structural incompatibility so the block handler can + * return the retryable denial without raising ERROR or leaking the caller's + * reply wait. The bool wrapper preserves existing callers' applied/not- + * applied contract. */ -bool -cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, int holder_node_id) +PcmGcsTransitionApplyResult +cluster_pcm_lock_apply_gcs_transition_result(BufferTag tag, PcmLockTransition trans, + int holder_node_id) { struct GrdEntry *entry; PcmState cur; @@ -2145,25 +2207,29 @@ cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, in bool broadcast_needed = false; if (cluster_pcm_htab == NULL) - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; if (holder_node_id < 0 || holder_node_id >= 32) - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; if (trans < PCM_TRANS_N_TO_S || trans > PCM_TRANS_S_TO_X_CLEANOUT) - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; if (trans == PCM_TRANS_S_TO_X_CLEANOUT) - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; if (trans == PCM_TRANS_N_TO_S || trans == PCM_TRANS_N_TO_X) entry = pcm_get_or_create_entry(tag); else entry = pcm_find_entry(tag); if (entry == NULL) - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; holder_bit = pcm_holder_bit(holder_node_id); target = pcm_transition_target(trans); pcm_entry_lock_exclusive(entry); + if (trans == PCM_TRANS_N_TO_S && !pcm_s_admission_allowed_locked(entry, holder_bit)) { + LWLockRelease(&entry->entry_lock.lock); + return PCM_GCS_TRANSITION_PENDING_X; + } cur = (PcmState)pg_atomic_read_u32(&entry->master_state); /* @@ -2175,7 +2241,7 @@ cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, in if (trans == PCM_TRANS_N_TO_S && cur == PCM_STATE_S) { pg_atomic_fetch_or_u32(&entry->s_holders_bitmap, holder_bit); LWLockRelease(&entry->entry_lock.lock); - return true; + return PCM_GCS_TRANSITION_APPLIED; } /* @@ -2189,12 +2255,12 @@ cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, in || (cur == PCM_STATE_S && (pg_atomic_read_u32(&entry->s_holders_bitmap) & holder_bit) == 0))) { LWLockRelease(&entry->entry_lock.lock); - return true; + return PCM_GCS_TRANSITION_APPLIED; } if (!cluster_pcm_transition_legal(cur, target, trans)) { LWLockRelease(&entry->entry_lock.lock); - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; } switch (trans) { @@ -2205,7 +2271,7 @@ cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, in if ((pg_atomic_read_u32(&entry->s_holders_bitmap) & holder_bit) == 0 || (pg_atomic_read_u32(&entry->s_holders_bitmap) & ~holder_bit) != 0) { LWLockRelease(&entry->entry_lock.lock); - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; } break; case PCM_TRANS_X_TO_S_DOWNGRADE: @@ -2213,29 +2279,37 @@ cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, in case PCM_TRANS_X_TO_N_RELEASE: if (entry->x_holder_node != holder_node_id) { LWLockRelease(&entry->entry_lock.lock); - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; } break; case PCM_TRANS_S_TO_N_INVALIDATE: case PCM_TRANS_S_TO_N_RELEASE: if ((pg_atomic_read_u32(&entry->s_holders_bitmap) & holder_bit) == 0) { LWLockRelease(&entry->entry_lock.lock); - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; } break; case PCM_TRANS_S_TO_X_CLEANOUT: LWLockRelease(&entry->entry_lock.lock); - return false; + return PCM_GCS_TRANSITION_INCOMPATIBLE; } cluster_pcm_transition_apply(entry, trans, holder_node_id); - if ((PcmState)pg_atomic_read_u32(&entry->master_state) == PCM_STATE_N) + if ((PcmState)pg_atomic_read_u32(&entry->master_state) == PCM_STATE_N + || trans == PCM_TRANS_X_TO_S_DOWNGRADE) broadcast_needed = true; LWLockRelease(&entry->entry_lock.lock); if (broadcast_needed) ConditionVariableBroadcast(&entry->wait_cv); - return true; + return PCM_GCS_TRANSITION_APPLIED; +} + +bool +cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, int holder_node_id) +{ + return cluster_pcm_lock_apply_gcs_transition_result(tag, trans, holder_node_id) + == PCM_GCS_TRANSITION_APPLIED; } @@ -2427,8 +2501,8 @@ cluster_pcm_get_trans_s_to_x_cleanout_count(void) "activate the spec-2.30 PCM state machine."))) -void -cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) +static bool +pcm_lock_acquire_local(BufferTag tag, PcmLockMode mode, PcmAuthoritySnapshot *remote_x_out) { struct GrdEntry *entry; int holder_node; @@ -2504,21 +2578,23 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) */ for (;;) { PcmState cur; + bool pending_x_blocks_new_s = false; pcm_entry_lock_exclusive(entry); cur = (PcmState)pg_atomic_read_u32(&entry->master_state); if (mode == PCM_LOCK_MODE_S) { - if (cur == PCM_STATE_N) { + pending_x_blocks_new_s = !pcm_s_admission_allowed_locked(entry, holder_bit); + if (!pending_x_blocks_new_s && cur == PCM_STATE_N) { cluster_pcm_transition_apply(entry, PCM_TRANS_N_TO_S, holder_node); entry->s_holder_refcount_local = 1; LWLockRelease(&entry->entry_lock.lock); if (cv_prepared) ConditionVariableCancelSleep(); - return; + return true; } - if (cur == PCM_STATE_S) { + if (!pending_x_blocks_new_s && cur == PCM_STATE_S) { /* Same-node S re-acquire: bump refcount; or join from other-node-S. */ if ((pg_atomic_read_u32(&entry->s_holders_bitmap) & holder_bit) != 0) entry->s_holder_refcount_local++; @@ -2529,7 +2605,7 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) LWLockRelease(&entry->entry_lock.lock); if (cv_prepared) ConditionVariableCancelSleep(); - return; + return true; } /* cur == X → fall through to wait */ } else /* mode == PCM_LOCK_MODE_X */ @@ -2541,7 +2617,7 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) LWLockRelease(&entry->entry_lock.lock); if (cv_prepared) ConditionVariableCancelSleep(); - return; + return true; } /* * PGRAC: spec-6.12a — idempotent X re-acquire. This NODE already @@ -2557,7 +2633,7 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) LWLockRelease(&entry->entry_lock.lock); if (cv_prepared) ConditionVariableCancelSleep(); - return; + return true; } holders = pg_atomic_read_u32(&entry->s_holders_bitmap); @@ -2575,7 +2651,7 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) LWLockRelease(&entry->entry_lock.lock); if (cv_prepared) ConditionVariableCancelSleep(); - return; + return true; } /* cur == S or X → fall through to wait */ } @@ -2596,7 +2672,7 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) * conflicting holder under the already-held entry_lock (no extra * htab_lock → no lock-order inversion). */ - if (cluster_gcs_block_local_cache) { + if (cluster_gcs_block_local_cache && !pending_x_blocks_new_s) { int32 confl_x = entry->x_holder_node; uint32 confl_s = pg_atomic_read_u32(&entry->s_holders_bitmap) & ~holder_bit; bool remote_live = false; @@ -2611,6 +2687,19 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) remote_live = true; if (remote_live) { + /* P0-26: the optimistic buffer-aware precheck may race a queue + * handoff. Capture the complete remote-X authority under the + * same entry lock instead of escaping through the tag-only + * legacy terminal; the caller owns the BufferDesc transfer. */ + if (cur == PCM_STATE_X && confl_x >= 0 && confl_x != holder_node + && remote_x_out != NULL) { + pcm_authority_snapshot_locked(entry, remote_x_out); + LWLockRelease(&entry->entry_lock.lock); + if (cv_prepared) + ConditionVariableCancelSleep(); + return false; + } + /* * PGRAC: spec-6.12a — LOCAL-master S->X upgrade. When the * conflict is ONLY remote S copies (the quiescent X->S @@ -2641,7 +2730,7 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) pcm_entry_lock_exclusive(entry); entry->s_holder_refcount_local = 0; LWLockRelease(&entry->entry_lock.lock); - return; + return true; } /* Invalidate did not complete — fail closed, retryable * (Rule 8.A: never write past an unconfirmed invalidate). */ @@ -2701,6 +2790,14 @@ cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) } } +void +cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode) +{ + /* A tag-only caller has no BufferDesc and therefore retains the + * historical fail-closed behavior for a remote-X conflict. */ + (void)pcm_lock_acquire_local(tag, mode, NULL); +} + /* * PGRAC: spec-5.2a D2 — clean-page X-transfer arm (backend-local, one-shot). @@ -2747,12 +2844,17 @@ cluster_pcm_clean_page_xfer_consume(void) * (HC84 PageSetLSN + memcpy under content_lock EXCLUSIVE). */ bool -cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) +cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode, bool *out_retry_denied) { BufferTag tag; int master_node; bool clean_eligible; + Assert(out_retry_denied != NULL); + if (out_retry_denied == NULL) + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cluster_pcm_lock_acquire_buffer: NULL retry result"))); + *out_retry_denied = false; if (buf == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("cluster_pcm_lock_acquire_buffer: NULL BufferDesc"))); @@ -2857,8 +2959,8 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) * spec-5.2a D2/D3: carry clean-page eligibility (X only) so the remote * master takes the dedicated clean-page X-transfer path rather than the * conservative HG7 fail-closed DENY. */ - return cluster_gcs_send_block_request_and_wait(buf, trans, master_node, - clean_eligible && mode == PCM_LOCK_MODE_X); + return cluster_gcs_send_block_request_and_wait( + buf, trans, master_node, clean_eligible && mode == PCM_LOCK_MODE_X, out_retry_denied); } /* @@ -2877,11 +2979,13 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) * (durable S; caller mirrors pcm_state = S). */ if (mode == PCM_LOCK_MODE_S) { - PcmLockMode master_state = cluster_pcm_lock_query(tag); - int32 holder = cluster_pcm_master_holder_node_by_tag(tag); + PcmAuthoritySnapshot authority; + bool have_authority = cluster_pcm_lock_authority_snapshot(tag, &authority); - if (master_state == PCM_LOCK_MODE_X && holder >= 0 && holder != cluster_node_id) - return cluster_gcs_local_master_read_image_and_wait(buf, holder); + if (have_authority && authority.state == PCM_STATE_X && authority.x_holder_node >= 0 + && authority.x_holder_node != cluster_node_id && authority.s_holders_bitmap == 0 + && authority.master_holder.node_id == (uint32)authority.x_holder_node) + return cluster_gcs_local_master_read_image_and_wait(buf, &authority, out_retry_denied); } else /* mode == PCM_LOCK_MODE_X */ { /* @@ -2895,11 +2999,15 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) * The heap AM then sees the remote row lock and enters the cross-node TX * completion wait (spec-5.2 D4/D5). */ - PcmLockMode master_state = cluster_pcm_lock_query(tag); - int32 holder = cluster_pcm_master_holder_node_by_tag(tag); + PcmAuthoritySnapshot authority; + bool have_authority = cluster_pcm_lock_authority_snapshot(tag, &authority); + PcmLockMode master_state = have_authority ? (PcmLockMode)authority.state : PCM_LOCK_MODE_N; - if (master_state == PCM_LOCK_MODE_X && holder >= 0 && holder != cluster_node_id) - return cluster_gcs_local_master_x_transfer_and_wait(buf, holder, clean_eligible); + if (have_authority && authority.state == PCM_STATE_X && authority.x_holder_node >= 0 + && authority.x_holder_node != cluster_node_id && authority.s_holders_bitmap == 0 + && authority.master_holder.node_id == (uint32)authority.x_holder_node) + return cluster_gcs_local_master_x_transfer_and_wait(buf, &authority, clean_eligible, + out_retry_denied); /* * PGRAC: spec-4.6a BUG-C2 follow-through for shared_catalog DDL after @@ -2916,9 +3024,16 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) if ((cluster_pcm_lock_query_s_holders_bitmap(tag) & self_bit) == 0) { struct GrdEntry *entry; + PcmAuthoritySnapshot raced_remote_x; bool upgraded = false; - cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + /* The S bootstrap is still BufferDesc-aware: if a queue handoff + * replaces the sampled S authority with remote X before this entry + * lock, preserve that exact authority and use the normal X-transfer + * path instead of the tag-only legacy terminal. */ + if (!pcm_lock_acquire_local(tag, PCM_LOCK_MODE_S, &raced_remote_x)) + return cluster_gcs_local_master_x_transfer_and_wait( + buf, &raced_remote_x, clean_eligible, out_retry_denied); /* Amendment v1.2 (R5): the upgrade waits on remote ACKs with * CHECK_FOR_INTERRUPTS in the loop, so a cancel can THROW out * of it — release the temporary S claim on that path too, not @@ -2966,7 +3081,17 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) } } - cluster_pcm_lock_acquire(tag, mode); + { + PcmAuthoritySnapshot raced_remote_x; + + if (!pcm_lock_acquire_local(tag, mode, &raced_remote_x)) { + if (mode == PCM_LOCK_MODE_S) + return cluster_gcs_local_master_read_image_and_wait(buf, &raced_remote_x, + out_retry_denied); + return cluster_gcs_local_master_x_transfer_and_wait(buf, &raced_remote_x, + clean_eligible, out_retry_denied); + } + } /* * PGRAC: GCS-race round-4c FUNC-1 (local-master flavour) — the tag-only * grant ships no image, so the buffer keeps this backend's pre-read diff --git a/src/backend/cluster/cluster_pcm_own.c b/src/backend/cluster/cluster_pcm_own.c index 348a780299..22777b7cd1 100644 --- a/src/backend/cluster/cluster_pcm_own.c +++ b/src/backend/cluster/cluster_pcm_own.c @@ -55,6 +55,7 @@ cluster_pcm_own_shmem_init(void) for (i = 0; i < NBuffers; i++) { pg_atomic_init_u64(&ClusterPcmOwnArray[i].generation, 0); pg_atomic_init_u64(&ClusterPcmOwnArray[i].reservation_token, 0); + pg_atomic_init_u64(&ClusterPcmOwnArray[i].writer_activation_token, 0); pg_atomic_init_u32(&ClusterPcmOwnArray[i].flags, 0); ClusterPcmOwnArray[i]._pad = 0; } @@ -88,6 +89,7 @@ cluster_pcm_own_reservation_begin_exact(int buf_id, uint64 expected_generation, ClusterPcmOwnResult live_result; uint64 generation; uint64 old_token; + uint64 writer_activation_token; uint32 flags; if (out_token == NULL) @@ -100,12 +102,23 @@ cluster_pcm_own_reservation_begin_exact(int buf_id, uint64 expected_generation, generation = pg_atomic_read_u64(&entry->generation); old_token = pg_atomic_read_u64(&entry->reservation_token); + writer_activation_token = pg_atomic_read_u64(&entry->writer_activation_token); flags = pg_atomic_read_u32(&entry->flags); if (generation != expected_generation) return CLUSTER_PCM_OWN_STALE; + if (writer_activation_token != 0 && writer_activation_token != old_token) + return CLUSTER_PCM_OWN_CORRUPT; live_result = cluster_pcm_own_classify_live_flags(flags, old_token); - if (live_result != CLUSTER_PCM_OWN_OK) + if (live_result != CLUSTER_PCM_OWN_OK) { + if (writer_activation_token != 0) + return CLUSTER_PCM_OWN_CORRUPT; return live_result; + } + if (writer_activation_token != 0) { + if (reservation_flag == PCM_OWN_FLAG_REVOKING) + return CLUSTER_PCM_OWN_BUSY; + return CLUSTER_PCM_OWN_CORRUPT; + } if (generation == UINT64_MAX || old_token == UINT64_MAX) return CLUSTER_PCM_OWN_EXHAUSTED; @@ -129,6 +142,8 @@ cluster_pcm_own_reservation_abort_exact(int buf_id, uint64 expected_generation, return CLUSTER_PCM_OWN_INVALID; if (!cluster_pcm_own_entry_for_buf(buf_id, &entry)) return CLUSTER_PCM_OWN_NOT_READY; + if (pg_atomic_read_u64(&entry->writer_activation_token) != 0) + return CLUSTER_PCM_OWN_CORRUPT; if (pg_atomic_read_u64(&entry->generation) != expected_generation || pg_atomic_read_u64(&entry->reservation_token) != reservation_token || pg_atomic_read_u32(&entry->flags) != reservation_flag) @@ -155,6 +170,8 @@ cluster_pcm_own_reservation_handoff_exact(int buf_id, uint64 expected_generation return CLUSTER_PCM_OWN_NOT_READY; if (pg_atomic_read_u64(&entry->generation) != expected_generation) return CLUSTER_PCM_OWN_STALE; + if (pg_atomic_read_u64(&entry->writer_activation_token) != 0) + return CLUSTER_PCM_OWN_CORRUPT; live_token = pg_atomic_read_u64(&entry->reservation_token); flags = pg_atomic_read_u32(&entry->flags); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -205,6 +222,8 @@ cluster_pcm_own_grant_commit_exact(int buf_id, uint64 expected_generation, uint6 return CLUSTER_PCM_OWN_STALE; live_token = pg_atomic_read_u64(&entry->reservation_token); flags = pg_atomic_read_u32(&entry->flags); + if (pg_atomic_read_u64(&entry->writer_activation_token) != 0) + return CLUSTER_PCM_OWN_CORRUPT; live_result = cluster_pcm_own_classify_live_flags(flags, live_token); if (live_result == CLUSTER_PCM_OWN_CORRUPT) return live_result; @@ -224,6 +243,73 @@ cluster_pcm_own_grant_commit_exact(int buf_id, uint64 expected_generation, uint6 return CLUSTER_PCM_OWN_OK; } +ClusterPcmOwnResult +cluster_pcm_own_writer_grant_commit_exact(int buf_id, uint64 expected_generation, + uint64 reservation_token, + uint64 *out_committed_generation) +{ + ClusterPcmOwnEntry *entry; + ClusterPcmOwnResult live_result; + uint64 generation; + uint64 live_token; + uint32 flags; + + if (out_committed_generation == NULL || reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + *out_committed_generation = 0; + if (!cluster_pcm_own_entry_for_buf(buf_id, &entry)) + return CLUSTER_PCM_OWN_NOT_READY; + generation = pg_atomic_read_u64(&entry->generation); + live_token = pg_atomic_read_u64(&entry->reservation_token); + flags = pg_atomic_read_u32(&entry->flags); + if (pg_atomic_read_u64(&entry->writer_activation_token) != 0) + return CLUSTER_PCM_OWN_CORRUPT; + if (generation != expected_generation) + return CLUSTER_PCM_OWN_STALE; + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + return live_result; + if (flags == 0) + return CLUSTER_PCM_OWN_STALE; + if (flags != PCM_OWN_FLAG_GRANT_PENDING) + return CLUSTER_PCM_OWN_BUSY; + if (live_token != reservation_token) + return CLUSTER_PCM_OWN_STALE; + if (generation == UINT64_MAX) + return CLUSTER_PCM_OWN_EXHAUSTED; + /* Caller holds the matching BufferDesc header lock. The active flag is + * cleared and the fence published before that lock can be released, so a + * REVOKING begin can never pass between the two stores. */ + generation++; + pg_atomic_write_u64(&entry->generation, generation); + pg_atomic_write_u32(&entry->flags, 0); + pg_atomic_write_u64(&entry->writer_activation_token, reservation_token); + *out_committed_generation = generation; + return CLUSTER_PCM_OWN_OK; +} + +ClusterPcmOwnResult +cluster_pcm_own_writer_activation_clear_exact(int buf_id, uint64 expected_generation, + uint64 reservation_token) +{ + ClusterPcmOwnEntry *entry; + uint64 live_activation; + + if (reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + if (!cluster_pcm_own_entry_for_buf(buf_id, &entry)) + return CLUSTER_PCM_OWN_NOT_READY; + if (pg_atomic_read_u64(&entry->generation) != expected_generation + || pg_atomic_read_u64(&entry->reservation_token) != reservation_token + || pg_atomic_read_u32(&entry->flags) != 0) + return CLUSTER_PCM_OWN_STALE; + live_activation = pg_atomic_read_u64(&entry->writer_activation_token); + if (live_activation == 0 || live_activation != reservation_token) + return CLUSTER_PCM_OWN_STALE; + pg_atomic_write_u64(&entry->writer_activation_token, 0); + return CLUSTER_PCM_OWN_OK; +} + ClusterPcmOwnResult cluster_pcm_own_revoke_commit_exact(int buf_id, uint64 expected_generation, uint64 reservation_token, uint64 *out_committed_generation) @@ -243,6 +329,8 @@ cluster_pcm_own_revoke_commit_exact(int buf_id, uint64 expected_generation, generation = pg_atomic_read_u64(&entry->generation); live_token = pg_atomic_read_u64(&entry->reservation_token); flags = pg_atomic_read_u32(&entry->flags); + if (pg_atomic_read_u64(&entry->writer_activation_token) != 0) + return CLUSTER_PCM_OWN_CORRUPT; if (generation != expected_generation) return CLUSTER_PCM_OWN_STALE; live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -284,6 +372,8 @@ cluster_pcm_own_revoke_retain_commit_exact(int buf_id, uint64 expected_generatio generation = pg_atomic_read_u64(&entry->generation); live_token = pg_atomic_read_u64(&entry->reservation_token); flags = pg_atomic_read_u32(&entry->flags); + if (pg_atomic_read_u64(&entry->writer_activation_token) != 0) + return CLUSTER_PCM_OWN_CORRUPT; if (generation != expected_generation) return CLUSTER_PCM_OWN_STALE; live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -321,6 +411,8 @@ cluster_pcm_own_revoke_retain_release_exact(int buf_id, uint64 committed_generat return CLUSTER_PCM_OWN_NOT_READY; if (pg_atomic_read_u64(&entry->generation) != committed_generation) return CLUSTER_PCM_OWN_STALE; + if (pg_atomic_read_u64(&entry->writer_activation_token) != 0) + return CLUSTER_PCM_OWN_CORRUPT; live_token = pg_atomic_read_u64(&entry->reservation_token); flags = pg_atomic_read_u32(&entry->flags); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); @@ -352,10 +444,14 @@ cluster_pcm_own_gen_bump_checked(int buf_id, uint64 *out_generation) flags = pg_atomic_read_u32(&entry->flags); if (out_generation != NULL) *out_generation = generation; - if (generation == UINT64_MAX || flags != 0) + if (generation == UINT64_MAX || flags != 0 + || pg_atomic_read_u64(&entry->writer_activation_token) != 0) return false; generation++; pg_atomic_write_u64(&entry->generation, generation); + /* Descriptor/tag reuse and every ordinary transition start with no + * activation lifecycle. A nonzero fence was rejected above. */ + pg_atomic_write_u64(&entry->writer_activation_token, 0); if (out_generation != NULL) *out_generation = generation; return true; diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c index 192c71249e..712f81e855 100644 --- a/src/backend/cluster/cluster_pcm_x_convert.c +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -104,6 +104,10 @@ static bool pcm_x_stats_depth_increment(PcmXShmemHeader *header); static bool pcm_x_stats_depth_decrement(PcmXShmemHeader *header); static bool pcm_x_master_terminal_leg_is_clear(const PcmXReliableLegState *leg); static bool pcm_x_master_blocker_stage_metadata_valid(const PcmXMasterTicketSlot *ticket); +static bool pcm_x_transfer_leg_exact(const PcmXReliableLegState *leg, uint16 opcode, + int32 responder_node, uint64 responder_session); +static bool pcm_x_transfer_peer_exact(const PcmXMasterTicketSlot *ticket, int32 responder_node, + uint64 responder_session); static void pcm_x_master_blocker_snapshot_clear(PcmXMasterBlockerSnapshot *snapshot); static bool pcm_x_ticket_ref_is_zero(const PcmXTicketRef *ref); static bool pcm_x_local_terminal_ref_valid(const PcmXTicketRef *ref); @@ -561,7 +565,15 @@ typedef enum PcmXDirectoryMatch { } PcmXDirectoryMatch; -static void pcm_x_runtime_fail_closed(void); +static void pcm_x_runtime_fail_closed_impl(const char *site_file, int site_line); +static PcmXSlotHeader *pcm_x_domain_slot(PcmXAllocatorKind kind, PcmXSlotRef ref, + const BufferTag *expected_tag, uint32 allowed_state_mask); +static void pcm_x_local_gate_acquire_guarded(LWLock *lock, LWLockMode mode, + PcmXLocalTagSlot *tag_slot); + +/* Capture each internal fail-closed arm (file:line) while keeping the many + * zero-argument call sites in this file textually unchanged. */ +#define pcm_x_runtime_fail_closed() pcm_x_runtime_fail_closed_impl(__FILE__, __LINE__) /* Required runtime precondition: never nest allocator under a tag domain. */ @@ -830,6 +842,7 @@ pcm_x_init_stats(PcmXStats *stats) pg_atomic_init_u64(&stats->own_abort_count, 0); pg_atomic_init_u64(&stats->own_busy_count, 0); pg_atomic_init_u64(&stats->own_corrupt_count, 0); + pg_atomic_init_u64(&stats->barrier_unwind_count, 0); } @@ -932,6 +945,7 @@ cluster_pcm_x_stats_snapshot(PcmXStatsSnapshot *snapshot_out) snapshot_out->own_abort_count = pg_atomic_read_u64(&header->stats.own_abort_count); snapshot_out->own_busy_count = pg_atomic_read_u64(&header->stats.own_busy_count); snapshot_out->own_corrupt_count = pg_atomic_read_u64(&header->stats.own_corrupt_count); + snapshot_out->barrier_unwind_count = pg_atomic_read_u64(&header->stats.barrier_unwind_count); snapshot_out->active_tags = (uint64)header->allocator[PCM_X_ALLOC_MASTER_TAG].used; snapshot_out->live_tickets = (uint64)header->allocator[PCM_X_ALLOC_MASTER_TICKET].used; snapshot_out->local_retire_gate = (uint64)pg_atomic_read_u32(&header->local_retire_gate); @@ -972,6 +986,35 @@ cluster_pcm_x_stats_note_wait(void) } +bool +cluster_pcm_x_requester_wait_once(PcmXPreSleepRevalidateCallback revalidate, PcmXWaitCallback wait, + void *callback_arg) +{ + if (revalidate == NULL || wait == NULL || !revalidate(callback_arg)) + return false; + cluster_pcm_x_stats_note_wait(); + wait(callback_arg); + return true; +} + + +PcmXQueueResult +cluster_pcm_x_requester_wait_once_result(PcmXPreSleepResultCallback revalidate, + PcmXWaitCallback wait, void *callback_arg) +{ + PcmXQueueResult result; + + if (revalidate == NULL || wait == NULL) + return PCM_X_QUEUE_INVALID; + result = revalidate(callback_arg); + if (result != PCM_X_QUEUE_OK) + return result; + cluster_pcm_x_stats_note_wait(); + wait(callback_arg); + return PCM_X_QUEUE_OK; +} + + void cluster_pcm_x_stats_note_queue_result(PcmXQueueResult result) { @@ -1045,8 +1088,20 @@ cluster_pcm_x_stats_note_own_corrupt(void) } +/* t/400 L3 item 3: a BARRIER_CLOSED refusal consumed by a barrier-aware + * LockBuffer caller (unwound in place of a client ERROR). */ +void +cluster_pcm_x_stats_note_barrier_unwind(void) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (header != NULL) + pcm_x_stats_increment(&header->stats.barrier_unwind_count); +} + + static void -pcm_x_runtime_fail_closed(void) +pcm_x_runtime_fail_closed_impl(const char *site_file, int site_line) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; bool transitioned; @@ -1056,15 +1111,290 @@ pcm_x_runtime_fail_closed(void) if (!transitioned) transitioned = cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_SHUTTING_DOWN, PCM_X_RUNTIME_RECOVERY_BLOCKED); - if (transitioned && header != NULL) + if (transitioned && header != NULL) { + const char *base; + + /* Record the fusing arm. Single writer: only the CAS winner above + * reaches this block, and a second fuse requires a full reactivation + * in between, so writes are serialized. */ + base = strrchr(site_file, '/'); + base = (base != NULL) ? base + 1 : site_file; + snprintf(header->fail_closed_site, sizeof(header->fail_closed_site), "%s:%d", base, + site_line); pcm_x_stats_increment(&header->stats.recovery_blocked_count); + + /* At most one log line per ACTIVE generation (single CAS winner), so + * this cannot flood. Skip inside critical sections; the shmem site + * above stays observable through pg_cluster_state either way. */ + if (CritSectionCount == 0) + ereport(LOG, + (errmsg("cluster PCM-X runtime fail-closed (recovery blocked) at %s:%d", base, + site_line), + errhint("Writer conversions on this node stay fail-closed until the PCM-X " + "runtime is reformed."))); + } } void -cluster_pcm_x_runtime_fail_closed(void) +cluster_pcm_x_runtime_fail_closed_at(const char *site_file, int site_line) { - pcm_x_runtime_fail_closed(); + pcm_x_runtime_fail_closed_impl(site_file, site_line); +} + + +bool +cluster_pcm_x_runtime_fail_closed_site(char *buf, Size buflen) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (buf == NULL || buflen == 0) + return false; + buf[0] = '\0'; + if (header == NULL || header->fail_closed_site[0] == '\0') + return false; + strlcpy(buf, header->fail_closed_site, Min(buflen, sizeof(header->fail_closed_site))); + return true; +} + + +/* Map an FIFO index to a printable value: PCM_X_INVALID_SLOT_INDEX -> -1. */ +static long +pcm_x_debug_index(Size index) +{ + return index == PCM_X_INVALID_SLOT_INDEX ? -1L : (long)index; +} + + +/* Diagnostic iterator over live master tag slots. Coherence contract: the + * candidate is identified by a racy pre-read, then revalidated and copied + * under the matching master partition shared lock (the same lock every + * in-place tag/ticket field mutation holds), so the printed combination is a + * state that actually existed. Diagnostic surface only — never used for + * protocol decisions. */ +bool +cluster_pcm_x_master_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, Size buflen) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXAllocatorView view; + PcmXMasterTagSlot copy; + Size i; + + if (header == NULL || cursor_io == NULL || index_out == NULL || buf == NULL || buflen == 0) + return false; + if (!pcm_x_allocator_entry_unlocked(header) + || !pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TAG, &view)) + return false; + for (i = *cursor_io; i < view.capacity; i++) { + PcmXMasterTagSlot *raw = (PcmXMasterTagSlot *)pcm_x_allocator_slot(&view, i); + PcmXMasterTagSlot *locked; + PcmXSlotRef tag_ref; + BufferTag tag; + uint32 partition; + + if (raw == NULL || pcm_x_slot_state_read(&raw->slot) != PCM_X_TAG_LIVE + || !pcm_x_slot_generation_read(&raw->slot, &tag_ref.slot_generation)) + continue; + tag_ref.slot_index = i; + tag = raw->tag; + pg_read_barrier(); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + locked = (PcmXMasterTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TAG, tag_ref, &tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (locked == NULL) { + LWLockRelease(&header->master_locks[partition].lock); + continue; + } + memcpy(©, locked, sizeof(copy)); + LWLockRelease(&header->master_locks[partition].lock); + snprintf(buf, buflen, + "rel=%u blk=%u head=%ld tail=%ld active=%ld queued=0x%x qseq=" UINT64_FORMAT + " outstanding=%zu", + copy.tag.relNumber, copy.tag.blockNum, pcm_x_debug_index(copy.head_index), + pcm_x_debug_index(copy.tail_index), pcm_x_debug_index(copy.active_index), + pg_atomic_read_u32(©.queued_node_bitmap), copy.queue_state_sequence, + copy.outstanding_ticket_count); + *index_out = i; + *cursor_io = i + 1; + return true; + } + *cursor_io = view.capacity; + return false; +} + + +/* Diagnostic iterator over occupied master ticket slots; same partition-lock + * coherence contract as the tag iterator. */ +bool +cluster_pcm_x_master_ticket_debug_next(Size *cursor_io, Size *index_out, char *buf, Size buflen) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXAllocatorView view; + PcmXMasterTicketSlot copy; + Size i; + + if (header == NULL || cursor_io == NULL || index_out == NULL || buf == NULL || buflen == 0) + return false; + if (!pcm_x_allocator_entry_unlocked(header) + || !pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TICKET, &view)) + return false; + for (i = *cursor_io; i < view.capacity; i++) { + PcmXMasterTicketSlot *raw = (PcmXMasterTicketSlot *)pcm_x_allocator_slot(&view, i); + PcmXMasterTicketSlot *locked; + PcmXSlotRef ticket_ref; + BufferTag tag; + uint32 partition; + uint32 state; + + if (raw == NULL) + continue; + state = pcm_x_slot_state_read(&raw->slot); + if (state == PCM_XT_FREE || state == PCM_XT_RESERVED_NONVISIBLE) + continue; + if (!pcm_x_slot_generation_read(&raw->slot, &ticket_ref.slot_generation)) + continue; + ticket_ref.slot_index = i; + tag = raw->ref.identity.tag; + pg_read_barrier(); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + locked = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &tag, PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (locked == NULL) { + LWLockRelease(&header->master_locks[partition].lock); + continue; + } + memcpy(©, locked, sizeof(copy)); + LWLockRelease(&header->master_locks[partition].lock); + snprintf(buf, buflen, + "state=%u flags=0x%x node=%d ticket=" UINT64_FORMAT " grant=" UINT64_FORMAT + " tomb=0x%llx involved=0x%x drained=0x%x retire_acked=0x%x pending_s=0x%x" + " acked_s=0x%x leg_op=%u leg_phase=%u leg_flags=0x%x leg_retry=%u leg_node=%d" + " leg_deadline=" UINT64_FORMAT, + pcm_x_slot_state_read(©.slot), pcm_x_slot_flags_read(©.slot), + copy.ref.identity.node_id, copy.ref.handle.ticket_id, copy.ref.grant_generation, + (unsigned long long)copy.reliable.response_tombstone_mask, + copy.involved_nodes_bitmap, copy.drained_nodes_bitmap, + copy.retire_acked_nodes_bitmap, copy.pending_s_holders_bitmap, + copy.acked_s_holders_bitmap, copy.reliable.pending_opcode, copy.reliable.phase, + copy.reliable.flags, copy.reliable.retry_count, + copy.reliable.expected_responder_node, copy.reliable.retry_deadline_ms); + *index_out = i; + *cursor_io = i + 1; + return true; + } + *cursor_io = view.capacity; + return false; +} + + +/* Diagnostic iterator over live local tag slots; partition-lock coherent copy + * like the master iterators, but under the local lock partition. */ +bool +cluster_pcm_x_local_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, Size buflen) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXAllocatorView view; + PcmXLocalTagSlot copy; + Size i; + + if (header == NULL || cursor_io == NULL || index_out == NULL || buf == NULL || buflen == 0) + return false; + if (!pcm_x_allocator_entry_unlocked(header) + || !pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_TAG, &view)) + return false; + for (i = *cursor_io; i < view.capacity; i++) { + PcmXLocalTagSlot *raw = (PcmXLocalTagSlot *)pcm_x_allocator_slot(&view, i); + PcmXLocalTagSlot *locked; + PcmXSlotRef tag_ref; + BufferTag tag; + uint32 partition; + + if (raw == NULL || pcm_x_slot_state_read(&raw->slot) != PCM_X_TAG_LIVE + || !pcm_x_slot_generation_read(&raw->slot, &tag_ref.slot_generation)) + continue; + tag_ref.slot_index = i; + tag = raw->tag; + pg_read_barrier(); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&tag)); + pcm_x_local_gate_acquire_guarded(&header->local_locks[partition].lock, LW_SHARED, NULL); + locked = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, tag_ref, &tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (locked == NULL) { + LWLockRelease(&header->local_locks[partition].lock); + continue; + } + memcpy(©, locked, sizeof(copy)); + LWLockRelease(&header->local_locks[partition].lock); + snprintf(buf, buflen, + "rel=%u fork=%d blk=%u flags=0x%x round=%u master=%d ref_node=%d " + "ref_ticket=" UINT64_FORMAT " ref_grant=" UINT64_FORMAT + " holder_ticket=" UINT64_FORMAT " blocker_ticket=" UINT64_FORMAT + " blocker_gen=" UINT64_FORMAT + " blocker_op=%u blocker_phase=%u blocker_last=%u blocker_last_node=%u" + " blocker_tomb=0x%llx blocker_count=%u blocker_head=%ld blocker_next=%u" + " blocker_crc=%u drain_gen=" UINT64_FORMAT " holder_drain_gen=" UINT64_FORMAT + " members=%zu head=%ld leader=%ld active_writer=%ld", + copy.tag.relNumber, (int)copy.tag.forkNum, copy.tag.blockNum, + pcm_x_slot_flags_read(©.slot), copy.local_round, copy.master_node, + copy.ref.identity.node_id, copy.ref.handle.ticket_id, copy.ref.grant_generation, + copy.holder_ref.handle.ticket_id, copy.blocker_snapshot_ref.handle.ticket_id, + copy.blocker_set_generation, copy.blocker_snapshot_reliable.pending_opcode, + copy.blocker_snapshot_reliable.phase, + copy.blocker_snapshot_reliable.last_response_opcode, + copy.blocker_snapshot_reliable.last_responder_node, + (unsigned long long)copy.blocker_snapshot_reliable.response_tombstone_mask, + copy.blocker_snapshot_count, pcm_x_debug_index(copy.blocker_snapshot_head_index), + copy.blocker_snapshot_next_chunk, copy.blocker_snapshot_crc32c, + copy.terminal_drain_generation, copy.holder_terminal_drain_generation, + copy.membership_count, pcm_x_debug_index(copy.head_index), + pcm_x_debug_index(copy.leader_index), pcm_x_debug_index(copy.active_writer_index)); + *index_out = i; + *cursor_io = i + 1; + return true; + } + *cursor_io = view.capacity; + return false; +} + + +void +cluster_pcm_x_terminal_note(uint32 op, uint32 result, uint64 ticket_id) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (header == NULL) + return; + if (header->terminal_note_op == op && header->terminal_note_result == result + && header->terminal_note_ticket == ticket_id) + header->terminal_note_count++; + else { + header->terminal_note_op = op; + header->terminal_note_result = result; + header->terminal_note_ticket = ticket_id; + header->terminal_note_count = 1; + } +} + + +bool +cluster_pcm_x_terminal_note_read(uint32 *op_out, uint32 *result_out, uint64 *ticket_out, + uint32 *count_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (header == NULL || header->terminal_note_count == 0) + return false; + if (op_out != NULL) + *op_out = header->terminal_note_op; + if (result_out != NULL) + *result_out = header->terminal_note_result; + if (ticket_out != NULL) + *ticket_out = header->terminal_note_ticket; + if (count_out != NULL) + *count_out = header->terminal_note_count; + return true; } @@ -2291,6 +2621,7 @@ cluster_pcm_x_runtime_snapshot(void) PcmXRuntimeSnapshot snapshot = { 0 }; uint32 gate1; uint32 gate2; + uint32 rebase_wire_active; uint32 state1; uint64 session; @@ -2308,6 +2639,7 @@ cluster_pcm_x_runtime_snapshot(void) pg_read_barrier(); session = ClusterPcmXConvertShmem->master_session_incarnation; + rebase_wire_active = ClusterPcmXConvertShmem->rebase_wire_active; pg_read_barrier(); gate2 = pg_atomic_read_u32(&ClusterPcmXConvertShmem->runtime_gate); if (gate1 != gate2 || session == 0) @@ -2315,10 +2647,20 @@ cluster_pcm_x_runtime_snapshot(void) snapshot.state = PCM_X_RUNTIME_ACTIVE; snapshot.master_session_incarnation = session; + snapshot.rebase_wire_active = rebase_wire_active != 0; return snapshot; } +void +cluster_pcm_x_runtime_set_rebase_wire_active(bool active) +{ + if (ClusterPcmXConvertShmem == NULL) + return; + ClusterPcmXConvertShmem->rebase_wire_active = active ? 1 : 0; +} + + /* * Revalidate one fail-stop acting token at the final owner lock. * @@ -2705,6 +3047,17 @@ pcm_x_local_handle_clear(PcmXLocalHandle *handle) } +/* The exact commit proof runs against the effective grant base: the one-shot + * pre-reservation rebase when an interleaved revoke consumed the enqueue-time + * base on this node, the immutable identity base otherwise. */ +static inline uint64 +pcm_x_local_effective_grant_base(const PcmXLocalTagSlot *tag_slot) +{ + return tag_slot->grant_base_own_generation != 0 ? tag_slot->grant_base_own_generation + : tag_slot->ref.identity.base_own_generation; +} + + static void pcm_x_local_follower_wfg_snapshot_clear(PcmXLocalFollowerWfgSnapshot *snapshot) { @@ -3346,6 +3699,7 @@ pcm_x_master_admit_begin_impl(const PcmXEnqueuePayload *request, PcmXMasterAdmis ticket->blocker_stage_source_node = -1; ticket->blocker_set_source_node = -1; ticket->involved_nodes_bitmap = node_bit; + ticket->grant_base_own_generation = 0; if (new_tag) { directory_result = pcm_x_directory_insert_locked( @@ -4493,6 +4847,31 @@ pcm_x_master_drive_capture_locked(PcmXRuntimeSnapshot runtime, PcmXMasterTagSlot } +/* Revalidate a snapshot-token locator after crossing allocator -> master + * lock domains. The exact ticket may have completed, retired, or been + * reclaimed in that gap. Those monotone successors consume the old drive + * token; they are not evidence that the live queue is corrupt. */ +static PcmXQueueResult +pcm_x_master_drive_expected_capture_locked(PcmXRuntimeSnapshot runtime, PcmXMasterTagSlot *tag_slot, + PcmXSlotRef tag_ref, PcmXMasterTicketSlot *ticket, + PcmXSlotRef ticket_ref, + const PcmXTicketRef *expected_ref, + PcmXMasterDriveSnapshot *snapshot) +{ + uint32 state; + + if (expected_ref == NULL || snapshot == NULL) + return PCM_X_QUEUE_CORRUPT; + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, expected_ref)) + return PCM_X_QUEUE_STALE; + state = pcm_x_slot_state_read(&ticket->slot); + if (state == PCM_XT_COMPLETE || state == PCM_XT_CANCELLED || state == PCM_XT_RETIRE_CREDIT) + return PCM_X_QUEUE_NOT_READY; + return pcm_x_master_drive_capture_locked(runtime, tag_slot, tag_ref, ticket, ticket_ref, + snapshot); +} + + static bool pcm_x_master_drive_snapshot_base_equal(const PcmXMasterDriveSnapshot *left, const PcmXMasterDriveSnapshot *right) @@ -4584,6 +4963,214 @@ cluster_pcm_x_master_drive_snapshot_exact(const BufferTag *tag, uint64 cluster_e } +PcmXQueueResult +cluster_pcm_x_master_commit_retry_exact(const PcmXMasterDriveSnapshot *expected, uint64 now_ms, + uint64 next_retry_deadline_ms, PcmXPhasePayload *commit_out, + PcmXMasterDriveSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXMasterDriveSnapshot current; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 partition; + + if (commit_out != NULL) + memset(commit_out, 0, sizeof(*commit_out)); + pcm_x_master_drive_snapshot_clear(snapshot_out); + if (header == NULL || expected == NULL || commit_out == NULL || snapshot_out == NULL + || next_retry_deadline_ms <= now_ms || expected->ticket_state != PCM_XT_ACTIVE_TRANSFER + || expected->pending_opcode != PGRAC_IC_MSG_PCM_X_COMMIT_X + || expected->phase != PGRAC_IC_MSG_PCM_X_COMMIT_X || expected->reserved != 0 + || expected->expected_responder_node != expected->ref.identity.node_id + || expected->expected_responder_session == 0 + || !pcm_x_wait_identity_valid(&expected->ref.identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(&expected->ref, &ticket_ref, &ticket); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&expected->ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TAG, tag_ref, + &expected->ref.identity.tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &expected->ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + result = pcm_x_master_drive_expected_capture_locked(runtime, tag_slot, tag_ref, ticket, + ticket_ref, &expected->ref, ¤t); + if (result != PCM_X_QUEUE_OK) + goto retry_done; + if (!pcm_x_master_drive_snapshot_equal(¤t, expected)) { + result = PCM_X_QUEUE_STALE; + goto retry_done; + } + if (!pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_COMMIT_X, + ticket->ref.identity.node_id, + ticket->reliable.expected_responder_session) + || !pcm_x_transfer_peer_exact(ticket, ticket->ref.identity.node_id, + ticket->reliable.expected_responder_session)) { + result = PCM_X_QUEUE_STALE; + goto retry_done; + } + if (ticket->reliable.retry_deadline_ms != 0 && now_ms < ticket->reliable.retry_deadline_ms) { + result = PCM_X_QUEUE_NOT_READY; + goto retry_done; + } + + if (ticket->reliable.retry_count != PG_UINT32_MAX) + ticket->reliable.retry_count++; + ticket->reliable.retry_deadline_ms = next_retry_deadline_ms; + pg_write_barrier(); + result = pcm_x_master_drive_capture_locked(runtime, tag_slot, tag_ref, ticket, ticket_ref, + snapshot_out); + if (result == PCM_X_QUEUE_OK) { + commit_out->ref = ticket->ref; + commit_out->phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; + } + +retry_done: + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_invalidate_busy_backoff_exact(const PcmXMasterDriveSnapshot *expected, + int32 busy_node, uint64 now_ms, + uint64 retry_delay_ms, + PcmXMasterDriveSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXMasterDriveSnapshot current; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint64 next_retry_deadline_ms; + uint32 busy_bit; + uint32 partition; + + pcm_x_master_drive_snapshot_clear(snapshot_out); + if (header == NULL || expected == NULL || snapshot_out == NULL || busy_node < 0 + || busy_node >= PCM_X_PROTOCOL_NODE_LIMIT || retry_delay_ms == 0 || now_ms == UINT64_MAX + || expected->ticket_state != PCM_XT_ACTIVE_TRANSFER + || expected->pending_opcode != PGRAC_IC_MSG_PCM_X_REVOKE + || expected->phase != PGRAC_IC_MSG_PCM_X_REVOKE || expected->reserved != 0 + || expected->expected_responder_node < 0 + || expected->expected_responder_node >= PCM_X_PROTOCOL_NODE_LIMIT + || expected->expected_responder_session == 0 + || !pcm_x_wait_identity_valid(&expected->ref.identity)) + return PCM_X_QUEUE_INVALID; + busy_bit = UINT32_C(1) << (uint32)busy_node; + if ((expected->pending_s_holders_bitmap & busy_bit) == 0 + || (expected->acked_s_holders_bitmap & busy_bit) != 0) + return PCM_X_QUEUE_INVALID; + next_retry_deadline_ms + = now_ms > UINT64_MAX - retry_delay_ms ? UINT64_MAX : now_ms + retry_delay_ms; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(&expected->ref, &ticket_ref, &ticket); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&expected->ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TAG, tag_ref, + &expected->ref.identity.tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &expected->ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + result = pcm_x_master_drive_expected_capture_locked(runtime, tag_slot, tag_ref, ticket, + ticket_ref, &expected->ref, ¤t); + if (result != PCM_X_QUEUE_OK) + goto busy_done; + if (!pcm_x_master_drive_snapshot_equal(¤t, expected)) { + result = PCM_X_QUEUE_STALE; + goto busy_done; + } + if (!pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_REVOKE, + ticket->reliable.expected_responder_node, + ticket->reliable.expected_responder_session) + || !pcm_x_transfer_peer_exact(ticket, ticket->reliable.expected_responder_node, + ticket->reliable.expected_responder_session)) { + result = PCM_X_QUEUE_STALE; + goto busy_done; + } + if (ticket->reliable.retry_deadline_ms != 0 && now_ms < ticket->reliable.retry_deadline_ms) { + *snapshot_out = current; + result = PCM_X_QUEUE_DUPLICATE; + goto busy_done; + } + + /* BUSY is only scheduling evidence. Do not credit the holder, clear GRD, + * change the reliable identity, or advance the retry-attempt counter. */ + ticket->reliable.retry_deadline_ms = next_retry_deadline_ms; + pg_write_barrier(); + result = pcm_x_master_drive_capture_locked(runtime, tag_slot, tag_ref, ticket, ticket_ref, + snapshot_out); + +busy_done: + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_commit_retry_drive(const PcmXMasterDriveSnapshot *expected, uint64 now_ms, + uint64 retry_delay_ms, PcmXStageFrameCallback stage_frame, + void *callback_arg, PcmXMasterDriveSnapshot *snapshot_out) +{ + PcmXPhasePayload commit; + PcmXQueueResult result; + uint64 next_retry_deadline_ms; + + pcm_x_master_drive_snapshot_clear(snapshot_out); + if (expected == NULL || stage_frame == NULL || snapshot_out == NULL || retry_delay_ms == 0 + || now_ms == UINT64_MAX) + return PCM_X_QUEUE_INVALID; + next_retry_deadline_ms + = now_ms > UINT64_MAX - retry_delay_ms ? UINT64_MAX : now_ms + retry_delay_ms; + result = cluster_pcm_x_master_commit_retry_exact(expected, now_ms, next_retry_deadline_ms, + &commit, snapshot_out); + if (result != PCM_X_QUEUE_OK) + return result; + return stage_frame(PGRAC_IC_MSG_PCM_X_COMMIT_X, commit.ref.identity.node_id, &commit, + sizeof(commit), callback_arg) + ? PCM_X_QUEUE_OK + : PCM_X_QUEUE_BUSY; +} + + PcmXQueueResult cluster_pcm_x_master_drive_bitmap_replace_exact(const PcmXMasterDriveSnapshot *expected, uint32 pending_s_holders_bitmap, @@ -4630,8 +5217,8 @@ cluster_pcm_x_master_drive_bitmap_replace_exact(const PcmXMasterDriveSnapshot *e ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, &expected->ref.identity.tag, PCM_X_MASTER_TICKET_DOMAIN_STATES); - result = pcm_x_master_drive_capture_locked(runtime, tag_slot, tag_ref, ticket, ticket_ref, - ¤t); + result = pcm_x_master_drive_expected_capture_locked(runtime, tag_slot, tag_ref, ticket, + ticket_ref, &expected->ref, ¤t); if (result != PCM_X_QUEUE_OK) goto replace_done; *snapshot_out = current; @@ -7067,9 +7654,13 @@ cluster_pcm_x_master_revoke_arm_exact(const PcmXTicketRef *ref, int32 responder_ else if (!pcm_x_transfer_leg_is_clear(&ticket->reliable)) { if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_REVOKE, responder_node, responder_session) - && ticket->image.image_id != 0 && ticket->image.source_node == (uint32)responder_node) + && ticket->image.image_id != 0 && ticket->image.source_node == (uint32)responder_node) { + /* Diagnostic: the transfer driver re-arms this exact REVOKE leg on + * every pass; count it so leg_retry exposes a live retry pump. */ + if (ticket->reliable.retry_count != PG_UINT32_MAX) + ticket->reliable.retry_count++; result = PCM_X_QUEUE_DUPLICATE; - else + } else result = PCM_X_QUEUE_BUSY; } else if (ticket->image.image_id != 0) { result = ticket->reliable.last_response_opcode == PGRAC_IC_MSG_PCM_X_IMAGE_READY @@ -7110,9 +7701,13 @@ cluster_pcm_x_master_revoke_arm_exact(const PcmXTicketRef *ref, int32 responder_ else if (!pcm_x_transfer_leg_is_clear(&ticket->reliable)) { if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_REVOKE, responder_node, responder_session) - && ticket->image.image_id != 0 && ticket->image.source_node == (uint32)responder_node) + && ticket->image.image_id != 0 && ticket->image.source_node == (uint32)responder_node) { + /* Diagnostic: the transfer driver re-arms this exact REVOKE leg on + * every pass; count it so leg_retry exposes a live retry pump. */ + if (ticket->reliable.retry_count != PG_UINT32_MAX) + ticket->reliable.retry_count++; result = PCM_X_QUEUE_DUPLICATE; - else + } else result = PCM_X_QUEUE_BUSY; } else if (ticket->image.image_id != 0) result = PCM_X_QUEUE_CORRUPT; @@ -7221,10 +7816,21 @@ cluster_pcm_x_master_image_ready_exact(const PcmXGrantPayload *image_ready, } +/* The exact commit proof runs against the effective grant base: the one-shot + * INSTALL_READY rebase when an interleaved revoke consumed the enqueue-time + * base on the requester node, the immutable identity base otherwise. */ +static inline uint64 +pcm_x_master_ticket_effective_grant_base(const PcmXMasterTicketSlot *ticket) +{ + return ticket->grant_base_own_generation != 0 ? ticket->grant_base_own_generation + : ticket->ref.identity.base_own_generation; +} + + PcmXQueueResult cluster_pcm_x_master_install_ready_exact(const PcmXInstallReadyPayload *install_ready, - int32 authenticated_node, uint64 authenticated_session, - PcmXPhasePayload *commit_out) + uint64 rebased_own_generation, int32 authenticated_node, + uint64 authenticated_session, PcmXPhasePayload *commit_out) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; PcmXRuntimeSnapshot runtime; @@ -7241,8 +7847,14 @@ cluster_pcm_x_master_install_ready_exact(const PcmXInstallReadyPayload *install_ || install_ready->phase != PGRAC_IC_MSG_PCM_X_INSTALL_READY || install_ready->flags != 0 || authenticated_node < 0 || authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT || authenticated_session == 0 || install_ready->ref.identity.node_id != authenticated_node - || !pcm_x_wait_identity_valid(&install_ready->ref.identity)) + || !pcm_x_wait_identity_valid(&install_ready->ref.identity) + || rebased_own_generation == UINT64_MAX) return PCM_X_QUEUE_INVALID; + /* A nonzero rebase must name a strictly newer effective grant base than + * the enqueue-time identity base it supersedes. */ + if (rebased_own_generation != 0 + && rebased_own_generation <= install_ready->ref.identity.base_own_generation) + return PCM_X_QUEUE_STALE; runtime = cluster_pcm_x_runtime_snapshot(); if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) return PCM_X_QUEUE_NOT_READY; @@ -7267,12 +7879,22 @@ cluster_pcm_x_master_install_ready_exact(const PcmXInstallReadyPayload *install_ result = PCM_X_QUEUE_STALE; else if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_COMMIT_X, authenticated_node, authenticated_session)) - result = PCM_X_QUEUE_DUPLICATE; - else + /* Replay after COMMIT_X armed: the published grant base is frozen; a + * different value is evidence divergence, never contention. */ + result = ticket->grant_base_own_generation == rebased_own_generation ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_CORRUPT; + else { result = pcm_x_transfer_leg_advance_locked( ticket, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, authenticated_node, authenticated_session, PGRAC_IC_MSG_PCM_X_INSTALL_READY, PGRAC_IC_MSG_PCM_X_COMMIT_X, authenticated_node, authenticated_session); + if (result == PCM_X_QUEUE_OK && rebased_own_generation != 0) { + /* One-shot publish, atomic with arming COMMIT_X under this lock: + * a committed COMMIT_X leg implies the grant base is persisted. */ + ticket->grant_base_own_generation = rebased_own_generation; + pg_write_barrier(); + } + } if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { commit_out->ref = ticket->ref; commit_out->phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; @@ -7307,10 +7929,13 @@ cluster_pcm_x_master_final_ack_prepare_exact(const PcmXFinalAckPayload *final_ac || final_ack->ref.identity.node_id != authenticated_node || !pcm_x_wait_identity_valid(&final_ack->ref.identity)) return PCM_X_QUEUE_INVALID; + /* Monotonic ingress floor only: the exact "+1" proof runs under the ticket + * lock against the effective grant base, which a published INSTALL_READY + * rebase may have moved past the enqueue-time identity base. */ if (!cluster_pcm_x_generation_next(final_ack->ref.identity.base_own_generation, &expected_generation)) return PCM_X_QUEUE_COUNTER_EXHAUSTED; - if (final_ack->committed_own_generation != expected_generation) + if (final_ack->committed_own_generation < expected_generation) return PCM_X_QUEUE_STALE; runtime = cluster_pcm_x_runtime_snapshot(); if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) @@ -7336,6 +7961,11 @@ cluster_pcm_x_master_final_ack_prepare_exact(const PcmXFinalAckPayload *final_ac else if (ticket->image.image_id != final_ack->image_id || !pcm_x_image_token_valid(&ticket->image)) result = PCM_X_QUEUE_STALE; + else if (!cluster_pcm_x_generation_next(pcm_x_master_ticket_effective_grant_base(ticket), + &expected_generation) + || final_ack->committed_own_generation != expected_generation) + /* FINAL_ACK proves exactly one ownership round under this grant. */ + result = PCM_X_QUEUE_STALE; else if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK, authenticated_node, authenticated_session)) result = pcm_x_transfer_peer_exact(ticket, authenticated_node, authenticated_session) @@ -7389,10 +8019,12 @@ cluster_pcm_x_master_final_ack_finalize_exact(const PcmXMasterFinalAckToken *tok || !pcm_x_image_token_valid(&token->image) || token->image.image_id != token->final_ack.image_id) return PCM_X_QUEUE_INVALID; + /* Monotonic floor only; the exact "+1" proof re-runs under the ticket lock + * against the effective grant base. */ if (!cluster_pcm_x_generation_next(token->final_ack.ref.identity.base_own_generation, &expected_generation)) return PCM_X_QUEUE_COUNTER_EXHAUSTED; - if (token->final_ack.committed_own_generation != expected_generation) + if (token->final_ack.committed_own_generation < expected_generation) return PCM_X_QUEUE_STALE; runtime = cluster_pcm_x_runtime_snapshot(); if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0 @@ -7423,6 +8055,10 @@ cluster_pcm_x_master_final_ack_finalize_exact(const PcmXMasterFinalAckToken *tok || !pcm_x_transfer_peer_exact(ticket, token->authenticated_node, token->authenticated_session)) result = PCM_X_QUEUE_STALE; + else if (!cluster_pcm_x_generation_next(pcm_x_master_ticket_effective_grant_base(ticket), + &expected_generation) + || token->final_ack.committed_own_generation != expected_generation) + result = PCM_X_QUEUE_STALE; else if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK, token->authenticated_node, token->authenticated_session)) { if (ticket->reliable.state_sequence == token->state_sequence) @@ -8548,6 +9184,11 @@ cluster_pcm_x_master_terminal_leg_arm_exact(const PcmXTicketRef *ref, PcmXTermin = kind == PCM_X_TERMINAL_LEG_DRAIN ? ticket->drain_generation : 0; token_out->expected_responder_node = responder_node; token_out->kind = (uint16)kind; + /* Diagnostic: an exact re-arm is the retry pump re-staging this + * leg. The count reaches the per-slot dump as leg_retry, so a + * frozen wedge separates a dead pump from a refusing responder. */ + if (ticket->reliable.retry_count != PG_UINT32_MAX) + ticket->reliable.retry_count++; result = PCM_X_QUEUE_DUPLICATE; } else result = PCM_X_QUEUE_BUSY; @@ -8814,7 +9455,18 @@ cluster_pcm_x_master_retire_ack_resolve_exact(const PcmXRetirePayload *ack, /* Return the oldest terminal ticket so the LMON cold-path retry driver cannot * starve the global contiguous retirement frontier behind a newer slot. */ PcmXQueueResult -cluster_pcm_x_master_terminal_work_next(PcmXTicketRef *ref_out) +cluster_pcm_x_master_terminal_work_next(PcmXTicketRef *ref_out) +{ + return cluster_pcm_x_master_terminal_work_next_after(0, ref_out); +} + + +/* Floor-scoped scan for the retry driver's fairness loop: the oldest ticket + * keeps frontier priority, but its stuck RETIRE preflight may be waiting for + * a younger ticket's DRAIN evidence (cross-lane holder interlock), so the + * driver walks every terminal ticket in ticket-id order per pass. */ +PcmXQueueResult +cluster_pcm_x_master_terminal_work_next_after(uint64 after_ticket_id, PcmXTicketRef *ref_out) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; PcmXRuntimeSnapshot runtime; @@ -8890,7 +9542,8 @@ cluster_pcm_x_master_terminal_work_next(PcmXTicketRef *ref_out) break; continue; } - if (ticket->ref.handle.ticket_id < best_ticket_id) { + if (ticket->ref.handle.ticket_id > after_ticket_id + && ticket->ref.handle.ticket_id < best_ticket_id) { best_ticket_id = ticket->ref.handle.ticket_id; *ref_out = ticket->ref; } @@ -9722,8 +10375,10 @@ pcm_x_local_tag_init_common(PcmXLocalTagSlot *tag_slot, const BufferTag *tag, ui tag_slot->closed_round_member_count = 0; tag_slot->terminal_drain_generation = 0; tag_slot->committed_own_generation = 0; + tag_slot->grant_base_own_generation = 0; memset(&tag_slot->holder_ref, 0, sizeof(tag_slot->holder_ref)); memset(&tag_slot->holder_image, 0, sizeof(tag_slot->holder_image)); + tag_slot->holder_required_page_scn = 0; memset(&tag_slot->holder_reliable, 0, sizeof(tag_slot->holder_reliable)); tag_slot->holder_terminal_drain_generation = 0; memset(&tag_slot->blocker_snapshot_ref, 0, sizeof(tag_slot->blocker_snapshot_ref)); @@ -9880,7 +10535,7 @@ pcm_x_local_writer_holder_authority_check(const PcmXLocalHolderKey *key, || tag_slot->ref.grant_generation == 0 || !pcm_x_wait_identity_equal(&tag_slot->ref.identity, &leader->identity) || tag_slot->committed_own_generation == 0 - || !cluster_pcm_x_generation_next(leader->identity.base_own_generation, + || !cluster_pcm_x_generation_next(pcm_x_local_effective_grant_base(tag_slot), &expected_generation) || tag_slot->committed_own_generation != expected_generation) return PCM_X_QUEUE_CORRUPT; @@ -10592,6 +11247,7 @@ pcm_x_local_holder_detach_exact(const PcmXLocalHolderHandle *handle, uint32 allo && pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref) && pcm_x_transfer_leg_is_clear(&tag_slot->reliable) && pcm_x_image_token_equal(&tag_slot->holder_image, &zero_image) + && tag_slot->holder_required_page_scn == 0 && pcm_x_transfer_leg_is_pristine(&tag_slot->holder_reliable) && tag_slot->holder_terminal_drain_generation == 0 && tag_slot->blocker_snapshot_head_index == PCM_X_INVALID_SLOT_INDEX @@ -11475,6 +12131,23 @@ pcm_x_local_nested_wait_guard_tag(const BufferTag *tag) } +/* This backend's most recent nested-guard refusal: the held content-lock tag + * that closed the guard and the refusing result. Diagnostic only. */ +static BufferTag pcm_x_nested_guard_block_tag; +static int pcm_x_nested_guard_block_result = PCM_X_QUEUE_OK; + +bool +cluster_pcm_x_nested_wait_guard_last_block(BufferTag *tag_out, int *result_out) +{ + if (pcm_x_nested_guard_block_result == PCM_X_QUEUE_OK) + return false; + if (tag_out != NULL) + *tag_out = pcm_x_nested_guard_block_tag; + if (result_out != NULL) + *result_out = pcm_x_nested_guard_block_result; + return true; +} + PcmXQueueResult cluster_pcm_x_nested_wait_guard_before_block(void) { @@ -11490,8 +12163,11 @@ cluster_pcm_x_nested_wait_guard_before_block(void) } for (i = 0; i < snapshot.count; i++) { result = pcm_x_local_nested_wait_guard_tag(&snapshot.tags[i]); - if (result != PCM_X_QUEUE_OK) + if (result != PCM_X_QUEUE_OK) { + pcm_x_nested_guard_block_tag = snapshot.tags[i]; + pcm_x_nested_guard_block_result = (int)result; return result; + } } return PCM_X_QUEUE_OK; } @@ -11822,6 +12498,27 @@ pcm_x_local_blocker_ack_tombstone_exact(const PcmXLocalTagSlot *tag_slot, } +/* A same-ticket re-PROBE can stage a later empty generation after the master + * consumed an earlier ACK. Keep this exception exact: only the authenticated + * master/session COMMIT, with a fully empty canonical payload, is eligible. */ +static bool +pcm_x_local_empty_blocker_commit_exact(const PcmXLocalTagSlot *tag_slot, + int32 authenticated_master_node, + uint64 authenticated_master_session) +{ + return tag_slot != NULL && tag_slot->blocker_set_generation != 0 + && tag_slot->blocker_set_generation != UINT64_MAX + && tag_slot->blocker_snapshot_reliable.state_sequence != 0 + && tag_slot->blocker_snapshot_reliable.response_tombstone_mask == 0 + && pcm_x_transfer_leg_exact(&tag_slot->blocker_snapshot_reliable, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, + authenticated_master_node, authenticated_master_session) + && tag_slot->blocker_snapshot_head_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->blocker_snapshot_count == 0 && tag_slot->blocker_snapshot_crc32c == 0 + && tag_slot->blocker_snapshot_next_chunk == 0; +} + + PcmXQueueResult cluster_pcm_x_local_blocker_snapshot_arm_exact( const PcmXTicketRef *ref, int32 authenticated_master_node, uint64 authenticated_master_session, @@ -12376,9 +13073,10 @@ cluster_pcm_x_local_blocker_ack_exact(const PcmXTicketRef *ref, uint64 set_gener PcmXQueueResult -cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, - int32 authenticated_master_node, - uint64 authenticated_master_session) +cluster_pcm_x_local_holder_revoke_apply_floor_exact(const PcmXRevokePayload *revoke, + uint64 required_page_scn, + int32 authenticated_master_node, + uint64 authenticated_master_session) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; PcmXRuntimeSnapshot runtime; @@ -12451,7 +13149,8 @@ cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, } if (!pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref)) { if (!pcm_x_ticket_ref_equal(&tag_slot->holder_ref, &revoke->ref) - || tag_slot->holder_image.image_id != revoke->image_id) { + || tag_slot->holder_image.image_id != revoke->image_id + || tag_slot->holder_required_page_scn != required_page_scn) { result = PCM_X_QUEUE_STALE; goto revoke_release_gate; } @@ -12469,10 +13168,9 @@ cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, goto revoke_release_gate; } /* Type 49 is the fail-closed boundary that replaces the legacy type-48 - * unconditional ACK. The same ticket locator's immutable blocker set must - * have received an exact type-48 ACK and reclaimed every blocker slot before - * holder revoke state can become visible. A premature 49 is retryable; an - * inconsistent tombstone is corruption. */ + * unconditional ACK. The same ticket locator must carry either its exact + * type-48 ACK tombstone or the narrowly authenticated empty re-PROBE replay + * below. A premature 49 is retryable; inconsistent evidence is corruption. */ if (pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref)) { result = PCM_X_QUEUE_NOT_READY; goto revoke_release_gate; @@ -12482,18 +13180,25 @@ cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, goto revoke_release_gate; } if (!pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable)) { - result = PCM_X_QUEUE_NOT_READY; - goto revoke_release_gate; - } - if (tag_slot->blocker_set_generation == 0 - || tag_slot->blocker_snapshot_reliable.state_sequence == 0 - || tag_slot->blocker_snapshot_reliable.last_response_opcode - != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK - || tag_slot->blocker_snapshot_reliable.last_responder_node - != (uint32)authenticated_master_node - || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX - || tag_slot->blocker_snapshot_count != 0 || tag_slot->blocker_snapshot_crc32c != 0 - || tag_slot->blocker_snapshot_next_chunk != 0) { + /* Authenticated exact REVOKE proves the master already crossed into + * transfer after staging this generation's ACK on the same tag FIFO. The + * ACK handler can nevertheless observe a transient local admission gate and + * leave the exact COMMIT armed. Consume only that canonical empty set (for + * the first or a replay generation), since no blocker edge can be omitted. */ + if (!pcm_x_local_empty_blocker_commit_exact(tag_slot, authenticated_master_node, + authenticated_master_session)) { + result = PCM_X_QUEUE_NOT_READY; + goto revoke_release_gate; + } + } else if (tag_slot->blocker_set_generation == 0 + || tag_slot->blocker_snapshot_reliable.state_sequence == 0 + || tag_slot->blocker_snapshot_reliable.last_response_opcode + != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK + || tag_slot->blocker_snapshot_reliable.last_responder_node + != (uint32)authenticated_master_node + || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_count != 0 || tag_slot->blocker_snapshot_crc32c != 0 + || tag_slot->blocker_snapshot_next_chunk != 0) { result = PCM_X_QUEUE_CORRUPT; fail_closed = true; goto revoke_release_gate; @@ -12507,6 +13212,7 @@ cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, tag_slot->holder_ref = revoke->ref; memset(&tag_slot->holder_image, 0, sizeof(tag_slot->holder_image)); tag_slot->holder_image.image_id = revoke->image_id; + tag_slot->holder_required_page_scn = required_page_scn; pcm_x_transfer_leg_clear(&tag_slot->holder_reliable, PGRAC_IC_MSG_PCM_X_REVOKE, authenticated_master_node); memset(&tag_slot->blocker_snapshot_ref, 0, sizeof(tag_slot->blocker_snapshot_ref)); @@ -12541,6 +13247,16 @@ cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, } +PcmXQueueResult +cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, + int32 authenticated_master_node, + uint64 authenticated_master_session) +{ + return cluster_pcm_x_local_holder_revoke_apply_floor_exact(revoke, 0, authenticated_master_node, + authenticated_master_session); +} + + PcmXQueueResult cluster_pcm_x_local_holder_progress_exact(const BufferTag *tag, PcmXLocalHolderProgress *progress_out) @@ -12587,6 +13303,7 @@ cluster_pcm_x_local_holder_progress_exact(const BufferTag *tag, } else { progress_out->ref = tag_slot->holder_ref; progress_out->image = tag_slot->holder_image; + progress_out->required_page_scn = tag_slot->holder_required_page_scn; progress_out->reliable_state_sequence = tag_slot->holder_reliable.state_sequence; progress_out->pending_opcode = tag_slot->holder_reliable.pending_opcode; progress_out->last_response_opcode = tag_slot->holder_reliable.last_response_opcode; @@ -12673,21 +13390,43 @@ cluster_pcm_x_local_queue_invalidate_authorize_exact(const BufferTag *tag, uint6 || tag_slot->blocker_snapshot_ref.identity.request_id != request_id || tag_slot->blocker_snapshot_ref.identity.node_id != requester_node) result = PCM_X_QUEUE_STALE; - else if (!pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable) - || tag_slot->blocker_set_generation == 0) + else if (tag_slot->blocker_set_generation == 0) result = PCM_X_QUEUE_NOT_READY; - else if (tag_slot->blocker_set_generation == UINT64_MAX - || tag_slot->blocker_snapshot_reliable.state_sequence == 0 - || tag_slot->blocker_snapshot_reliable.last_response_opcode - != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK - || tag_slot->blocker_snapshot_reliable.last_responder_node - != (uint32)authenticated_master_node - || tag_slot->blocker_snapshot_reliable.response_tombstone_mask != 0 - || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX - || tag_slot->blocker_snapshot_count != 0 || tag_slot->blocker_snapshot_crc32c != 0 - || tag_slot->blocker_snapshot_next_chunk != 0 - || (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 - || tag_slot->holder_terminal_drain_generation != 0) { + else if (tag_slot->blocker_set_generation == UINT64_MAX) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else if (!pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable)) { + /* An authenticated exact INVALIDATE proves the master already crossed + * into transfer. A reordered same-ticket re-PROBE may have published + * another empty blocker COMMIT after the ACK that authorized that + * transition. No graph edge can be omitted in the empty set, so let + * passive pinned S become N+PI instead of waiting forever for an ACK + * the advanced master is no longer required to replay. */ + if (!pcm_x_transfer_leg_exact(&tag_slot->blocker_snapshot_reliable, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, + authenticated_master_node, authenticated_master_session) + || tag_slot->blocker_snapshot_count != 0) + result = PCM_X_QUEUE_NOT_READY; + else if (!pcm_x_local_empty_blocker_commit_exact(tag_slot, authenticated_master_node, + authenticated_master_session) + || (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 + || tag_slot->holder_terminal_drain_generation != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else + result = PCM_X_QUEUE_OK; + } else if (tag_slot->blocker_snapshot_reliable.state_sequence == 0 + || tag_slot->blocker_snapshot_reliable.last_response_opcode + != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK + || tag_slot->blocker_snapshot_reliable.last_responder_node + != (uint32)authenticated_master_node + || tag_slot->blocker_snapshot_reliable.response_tombstone_mask != 0 + || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_count != 0 + || tag_slot->blocker_snapshot_crc32c != 0 + || tag_slot->blocker_snapshot_next_chunk != 0 + || (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 + || tag_slot->holder_terminal_drain_generation != 0) { result = PCM_X_QUEUE_CORRUPT; fail_closed = true; } else @@ -12701,10 +13440,11 @@ cluster_pcm_x_local_queue_invalidate_authorize_exact(const BufferTag *tag, uint6 PcmXQueueResult -cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_ready, - int32 authenticated_master_node, - uint64 authenticated_master_session, - PcmXGrantPayload *replay_out) +cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed(const PcmXGrantPayload *image_ready, + int32 authenticated_master_node, + uint64 authenticated_master_session, + PcmXGrantPayload *replay_out, + PcmXLocalImageReadyRefusal *refusal_out) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; PcmXRuntimeSnapshot runtime; @@ -12717,6 +13457,8 @@ cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_r bool gate_claimed = false; bool fail_closed = false; + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_NONE; if (replay_out != NULL) memset(replay_out, 0, sizeof(*replay_out)); if (header == NULL || image_ready == NULL || replay_out == NULL @@ -12725,19 +13467,31 @@ cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_r || !pcm_x_image_token_valid(&image_ready->image) || !pcm_x_image_id_master_exact(image_ready->image.image_id, authenticated_master_node) || authenticated_master_node < 0 || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT - || authenticated_master_session == 0) + || authenticated_master_session == 0) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_INVALID; return PCM_X_QUEUE_INVALID; + } runtime = cluster_pcm_x_runtime_snapshot(); - if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_RUNTIME; return PCM_X_QUEUE_NOT_READY; + } LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); directory_result = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &image_ready->ref.identity.tag, &tag_ref); LWLockRelease(&header->allocator_lock.lock); - if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_DIRECTORY; return PCM_X_QUEUE_NOT_FOUND; - if (directory_result != PCM_X_DIRECTORY_OK) + } + if (directory_result != PCM_X_DIRECTORY_OK) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_DIRECTORY; return pcm_x_queue_result_from_directory(directory_result); + } partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&image_ready->ref.identity.tag)); LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); @@ -12745,16 +13499,22 @@ cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_r &image_ready->ref.identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); if (tag_slot == NULL) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_TAG_SLOT; result = PCM_X_QUEUE_STALE; goto ready_done; } result = pcm_x_local_gate_try_acquire(tag_slot); if (result != PCM_X_QUEUE_OK) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_LOCAL_GATE; fail_closed = result == PCM_X_QUEUE_CORRUPT; goto ready_done; } gate_claimed = true; if (!pcm_x_runtime_token_exact(&runtime, 0)) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_RUNTIME_RECHECK; result = PCM_X_QUEUE_NOT_READY; goto ready_release_gate; } @@ -12762,29 +13522,41 @@ cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_r authenticated_master_session) || !pcm_x_ticket_ref_equal(&tag_slot->holder_ref, &image_ready->ref) || tag_slot->holder_image.image_id != image_ready->image.image_id) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_IDENTITY; result = PCM_X_QUEUE_STALE; goto ready_release_gate; } if (tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_ACTIVE_HOLDER; result = PCM_X_QUEUE_NOT_READY; goto ready_release_gate; } if (!pcm_x_transfer_leg_is_clear(&tag_slot->holder_reliable)) { if (!pcm_x_transfer_leg_exact(&tag_slot->holder_reliable, PGRAC_IC_MSG_PCM_X_IMAGE_READY, authenticated_master_node, authenticated_master_session)) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_RELIABLE_LEG; result = PCM_X_QUEUE_BUSY; goto ready_release_gate; } result = pcm_x_image_token_equal(&tag_slot->holder_image, &image_ready->image) ? PCM_X_QUEUE_DUPLICATE : PCM_X_QUEUE_STALE; + if (result == PCM_X_QUEUE_STALE && refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_IDENTITY; goto ready_output; } if (tag_slot->holder_reliable.last_response_opcode != PGRAC_IC_MSG_PCM_X_REVOKE) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_BAD_PHASE; result = PCM_X_QUEUE_BAD_STATE; goto ready_release_gate; } if (!cluster_pcm_x_generation_next(tag_slot->holder_reliable.state_sequence, &next_sequence)) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_COUNTER; result = PCM_X_QUEUE_COUNTER_EXHAUSTED; fail_closed = true; goto ready_release_gate; @@ -12806,6 +13578,8 @@ cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_r } ready_release_gate: if (gate_claimed && !fail_closed && !pcm_x_local_gate_release(tag_slot)) { + if (refusal_out != NULL) + *refusal_out = PCM_X_LOCAL_IMAGE_READY_REFUSAL_GATE_RELEASE; result = PCM_X_QUEUE_CORRUPT; fail_closed = true; } @@ -12817,6 +13591,17 @@ cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_r } +PcmXQueueResult +cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_ready, + int32 authenticated_master_node, + uint64 authenticated_master_session, + PcmXGrantPayload *replay_out) +{ + return cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( + image_ready, authenticated_master_node, authenticated_master_session, replay_out, NULL); +} + + PcmXQueueResult cluster_pcm_x_local_join_begin(const PcmXWaitIdentity *identity, int32 master_node, uint64 master_session_incarnation, PcmXLocalHandle *handle_out) @@ -13462,6 +14247,7 @@ cluster_pcm_x_local_progress_exact(const PcmXLocalHandle *handle, PcmXLocalProgr progress_out->phase = tag_slot->reliable.phase; progress_out->master_session_incarnation = tag_slot->master_session_incarnation; progress_out->master_node = tag_slot->master_node; + progress_out->grant_base_own_generation = tag_slot->grant_base_own_generation; result = PCM_X_QUEUE_OK; progress_done: @@ -14085,7 +14871,7 @@ cluster_pcm_x_local_leader_rekey_generation_exact(const PcmXLocalHandle *leader, } memset(&zero_image, 0, sizeof(zero_image)); if (!pcm_x_image_token_equal(&tag_slot->image, &zero_image) - || tag_slot->committed_own_generation != 0) { + || tag_slot->committed_own_generation != 0 || tag_slot->grant_base_own_generation != 0) { result = PCM_X_QUEUE_CORRUPT; fail_closed = true; goto rekey_local_done; @@ -14486,6 +15272,11 @@ cluster_pcm_x_local_writer_claim_exact(const PcmXLocalHandle *writer, claim_out->claim_generation = next_claim_generation; claim_out->local_round = member->admitted_round; claim_out->role = member->role; + /* A' rebase: every same-round claim inherits the published effective + * grant base under this partition lock; zero (no rebase) keeps the + * enqueue-time identity math. Without this a FIFO follower would verify + * the rebased X against its stale base+1 and fail closed. */ + claim_out->grant_base_own_generation = tag_slot->grant_base_own_generation; result = PCM_X_QUEUE_OK; claim_release_gate: @@ -15690,6 +16481,7 @@ cluster_pcm_x_local_prepare_grant_exact(const PcmXLocalHandle *leader, tag_slot->ref = prepare->ref; tag_slot->image = prepare->image; tag_slot->committed_own_generation = 0; + tag_slot->grant_base_own_generation = 0; tag_slot->reliable.last_responder_node = (uint32)authenticated_node; tag_slot->reliable.last_response_opcode = PGRAC_IC_MSG_PCM_X_PREPARE_GRANT; pg_write_barrier(); @@ -15743,10 +16535,12 @@ pcm_x_local_transfer_arm_exact(const PcmXLocalHandle *leader, uint16 opcode, || !pcm_x_wait_identity_valid(&leader->identity)) return PCM_X_QUEUE_INVALID; if (opcode == PGRAC_IC_MSG_PCM_X_FINAL_ACK) { + /* Monotonic floor only; the exact "+1" proof re-runs under the local + * lock against the effective grant base. */ if (!cluster_pcm_x_generation_next(leader->identity.base_own_generation, &expected_generation)) return PCM_X_QUEUE_COUNTER_EXHAUSTED; - if (committed_own_generation != expected_generation) + if (committed_own_generation < expected_generation) return PCM_X_QUEUE_STALE; } runtime = cluster_pcm_x_runtime_snapshot(); @@ -15816,6 +16610,19 @@ pcm_x_local_transfer_arm_exact(const PcmXLocalHandle *leader, uint16 opcode, result = PCM_X_QUEUE_BAD_STATE; goto arm_done; } + if (opcode == PGRAC_IC_MSG_PCM_X_FINAL_ACK) { + if (!cluster_pcm_x_generation_next(pcm_x_local_effective_grant_base(tag_slot), + &expected_generation)) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + fail_closed = true; + goto arm_done; + } + /* FINAL_ACK proves exactly one ownership round under this grant. */ + if (committed_own_generation != expected_generation) { + result = PCM_X_QUEUE_STALE; + goto arm_done; + } + } if (!cluster_pcm_x_generation_next(tag_slot->reliable.state_sequence, &next_sequence)) { result = PCM_X_QUEUE_COUNTER_EXHAUSTED; fail_closed = true; @@ -15844,6 +16651,7 @@ pcm_x_local_transfer_arm_exact(const PcmXLocalHandle *leader, uint16 opcode, install_ready_out->image_id = tag_slot->image.image_id; install_ready_out->result = PCM_X_QUEUE_OK; install_ready_out->phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; + install_ready_out->rebased_own_generation = tag_slot->grant_base_own_generation; } else { final_ack_out->ref = tag_slot->ref; final_ack_out->image_id = tag_slot->image.image_id; @@ -15860,6 +16668,80 @@ pcm_x_local_transfer_arm_exact(const PcmXLocalHandle *leader, uint16 opcode, } +/* + * One-shot pre-reservation publication of the effective grant base for a + * prepared transfer whose enqueue-time base was consumed by an interleaved + * revoke. Runs strictly between PREPARE_GRANT apply and the INSTALL_READY + * arm: the reliable leg must be clear with PREPARE_GRANT as its last + * response. A same-value replay is DUPLICATE; a different second value is + * evidence divergence and fails the runtime closed. The immutable identity + * (directory key) is never touched. + */ +PcmXQueueResult +cluster_pcm_x_local_grant_rebase_publish_exact(const PcmXLocalHandle *leader, + uint64 rebased_own_generation) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint32 partition; + bool fail_closed = false; + + if (header == NULL || leader == NULL || rebased_own_generation == 0 + || rebased_own_generation == UINT64_MAX || !pcm_x_wait_identity_valid(&leader->identity)) + return PCM_X_QUEUE_INVALID; + if (rebased_own_generation <= leader->identity.base_own_generation) + return PCM_X_QUEUE_STALE; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(leader, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + result = pcm_x_local_transfer_slots_exact(leader, tag_ref, member_ref, &tag_slot, &member); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto rebase_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto rebase_done; + } + /* Only a prepared, not-yet-armed transfer may move its grant base. */ + if (tag_slot->ref.grant_generation == 0 || !pcm_x_image_token_valid(&tag_slot->image) + || pcm_x_slot_state_read(&member->slot) != PCM_XL_REMOTE_WAIT + || !pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) + || tag_slot->reliable.last_response_opcode != PGRAC_IC_MSG_PCM_X_PREPARE_GRANT) { + result = PCM_X_QUEUE_BAD_STATE; + goto rebase_done; + } + if (tag_slot->grant_base_own_generation == rebased_own_generation) { + result = PCM_X_QUEUE_DUPLICATE; + goto rebase_done; + } + if (tag_slot->grant_base_own_generation != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rebase_done; + } + tag_slot->grant_base_own_generation = rebased_own_generation; + pg_write_barrier(); + result = PCM_X_QUEUE_OK; + +rebase_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + PcmXQueueResult cluster_pcm_x_local_install_ready_arm_exact(const PcmXLocalHandle *leader, const PcmXTicketRef *expected_ref, @@ -16280,6 +17162,7 @@ pcm_x_local_holder_lane_retire_state(const PcmXLocalTagSlot *tag_slot, uint32 fl || tag_slot->blocker_snapshot_next_chunk != 0 || !pcm_x_transfer_leg_is_pristine(&tag_slot->blocker_snapshot_reliable) || !pcm_x_image_token_equal(&tag_slot->holder_image, &zero_image) + || tag_slot->holder_required_page_scn != 0 || !pcm_x_transfer_leg_is_pristine(&tag_slot->holder_reliable) || holder_flags != PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK || tag_slot->holder_terminal_drain_generation == 0 @@ -16295,6 +17178,7 @@ pcm_x_local_holder_lane_retire_state(const PcmXLocalTagSlot *tag_slot, uint32 fl return PCM_X_QUEUE_NOT_READY; if (holder_flags != 0 || tag_slot->holder_terminal_drain_generation != 0 || !pcm_x_image_token_equal(&tag_slot->holder_image, &zero_image) + || tag_slot->holder_required_page_scn != 0 || !pcm_x_transfer_leg_is_pristine(&tag_slot->holder_reliable)) return PCM_X_QUEUE_CORRUPT; return blocker_state; @@ -16373,16 +17257,29 @@ pcm_x_local_same_ref_dual_retire_state(const PcmXLocalTagSlot *tag_slot, uint32 source_holder = !pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref); external_ref = source_holder ? &tag_slot->holder_ref : &tag_slot->blocker_snapshot_ref; memset(&zero_image, 0, sizeof(zero_image)); + /* Per-lane completeness comes first: each terminal lane must carry its + * own full DRAIN evidence before any dual/independent classification. */ if (writer_flags != PCM_X_LOCAL_TAG_F_TERMINAL_MASK || holder_flags != PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK || tag_slot->membership_count == 0 - || pcm_x_ticket_ref_is_zero(external_ref) - || !pcm_x_ticket_ref_equal(&tag_slot->ref, external_ref) - || tag_slot->terminal_drain_generation == 0 + || pcm_x_ticket_ref_is_zero(external_ref) || tag_slot->terminal_drain_generation == 0 || tag_slot->terminal_drain_generation == UINT64_MAX - || tag_slot->terminal_drain_generation != tag_slot->holder_terminal_drain_generation || !pcm_x_local_terminal_pending_clear(tag_slot) || pcm_x_local_holder_lane_retire_state(tag_slot, flags) != PCM_X_QUEUE_OK) return PCM_X_QUEUE_CORRUPT; + if (!pcm_x_ticket_ref_equal(&tag_slot->ref, external_ref)) { + /* Two complete terminals under DISTINCT master tickets are the legal + * independent form: a granted writer awaiting RETIRE plus a newer + * revoked holder lane on the same tag (the t/400 dump-0x3d shape). + * Not a dual -- each lane retires by its own watermark. A ticket-id + * collision under a different ref can only be corruption: per-session + * ticket ids are master-unique (see the cross-lane alias-class note + * at pcm_x_local_cross_lane_holder_terminal_retirable). */ + if (external_ref->handle.ticket_id != tag_slot->ref.handle.ticket_id) + return PCM_X_QUEUE_OK; + return PCM_X_QUEUE_CORRUPT; + } + if (tag_slot->terminal_drain_generation != tag_slot->holder_terminal_drain_generation) + return PCM_X_QUEUE_CORRUPT; if (external_ref->grant_generation == 0) { if (source_holder || tag_slot->ref.grant_generation != 0 || (flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0 @@ -17616,6 +18513,80 @@ pcm_x_local_empty_frozen_round_apply_locked(PcmXLocalTagSlot *tag_slot, } +/* Read-only rehearsal of the frozen-round close that must accompany the LAST + * terminal authority leaving a barrier-frozen tag. An independent dual + * terminal retires its writer lane first while the barrier stands, so by the + * holder ticket's own RETIRE the round may hold next-round followers that + * joined in between; consuming the holder lane without closing the round + * would strand them behind a barrier with no terminal authority left to drop + * it. Every fallible check runs here BEFORE the holder lane is consumed; + * the caller then applies the close infallibly afterwards. The RETIRE + * preflight runs the same rehearsal so both passes agree (the retire gate + * excludes ordinary mutators for the whole episode). + * + * after_writer_retire projects the state a single watermark covering both + * lanes will reach: the writer target is still the closed round of one and + * still owns tag_slot->ref/membership when the preflight runs, so those two + * facts are checked in their pre-retire form instead. */ +static PcmXQueueResult +pcm_x_local_independent_holder_close_plan_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + bool after_writer_retire, + PcmXLocalRoundClosePlan *plan) +{ + PcmXQueueResult result; + Size projected_membership; + Size projected_closed; + bool candidate_promote = false; + + if (tag_slot == NULL || plan == NULL) + return PCM_X_QUEUE_CORRUPT; + memset(plan, 0, sizeof(*plan)); + plan->candidate_ref.slot_index = PCM_X_INVALID_SLOT_INDEX; + if (after_writer_retire) { + if (tag_slot->closed_round_member_count != 1 || tag_slot->membership_count == 0 + || pcm_x_ticket_ref_is_zero(&tag_slot->ref)) + return PCM_X_QUEUE_CORRUPT; + projected_membership = tag_slot->membership_count - 1; + projected_closed = tag_slot->closed_round_member_count - 1; + } else { + if (tag_slot->closed_round_member_count != 0 || !pcm_x_ticket_ref_is_zero(&tag_slot->ref)) + return PCM_X_QUEUE_CORRUPT; + projected_membership = tag_slot->membership_count; + projected_closed = tag_slot->closed_round_member_count; + } + /* Unlinked CANCELLED next-round members awaiting their own detach leave + * the FIFO empty while the membership count is still nonzero. That is + * a legal transient, not corruption: wait for their backends to release + * them (retire-gated episodes never race those detaches). */ + if (tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX && projected_membership > 0) + return PCM_X_QUEUE_NOT_READY; + if (tag_slot->local_round == 0 || tag_slot->local_round == UINT32_MAX + || tag_slot->next_sequence == 0 || tag_slot->cutoff_sequence >= tag_slot->next_sequence + || projected_closed != 0 + || ((projected_membership == 0) != (tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX)) + || ((tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX)) + || tag_slot->leader_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->leader_slot_generation != 0 + || tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_writer_slot_generation != 0 + || tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX + || !pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable)) + return PCM_X_QUEUE_CORRUPT; + if (tag_slot->head_index != PCM_X_INVALID_SLOT_INDEX) { + result = pcm_x_local_follower_chain_ready(tag_slot, tag_ref, tag_slot->head_index, + PCM_X_INVALID_SLOT_INDEX, 0, &plan->candidate, + &plan->candidate_ref, &candidate_promote); + if (result != PCM_X_QUEUE_OK) + return result == PCM_X_QUEUE_BUSY ? PCM_X_QUEUE_NOT_READY : result; + if (plan->candidate == NULL || candidate_promote) + return PCM_X_QUEUE_CORRUPT; + } + plan->close_round = true; + return PCM_X_QUEUE_OK; +} + + /* Project the state immediately after RETIRE removes the final member of a * closed writer cohort. The terminal member is already unlinked by DRAIN, so * the remaining FIFO, if any, must consist entirely of next-round followers. @@ -17666,14 +18637,16 @@ pcm_x_local_final_member_round_plan_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotR /* Close the terminal leg for an S-holder node that was not selected as the - * image source. Type-48 leaves a generation-zero admission locator plus an - * exact ACK tombstone. The authenticated DRAIN carries the first final grant - * generation this node can know, so capture it exactly and make all replays + * image source. Type-48 normally leaves an exact ACK tombstone; a reordered + * same-ticket re-PROBE can instead leave a canonical empty COMMIT after the + * master advanced. Authenticated DRAIN consumes either exact form, captures + * the first final grant generation this node can know, and makes every replay * generation- and drain-exact. Caller holds the local tag lock. */ static PcmXQueueResult pcm_x_local_blocker_participant_drain_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, const PcmXDrainPollPayload *poll, - int32 authenticated_master_node) + int32 authenticated_master_node, + uint64 authenticated_master_session) { PcmXImageToken zero_image; PcmXLocalRoundClosePlan round_plan; @@ -17717,6 +18690,7 @@ pcm_x_local_blocker_participant_drain_locked(PcmXLocalTagSlot *tag_slot, PcmXSlo return PCM_X_QUEUE_NOT_READY; if (holder_flags != 0 || tag_slot->holder_terminal_drain_generation != 0 || !pcm_x_image_token_equal(&tag_slot->holder_image, &zero_image) + || tag_slot->holder_required_page_scn != 0 || !pcm_x_transfer_leg_is_pristine(&tag_slot->holder_reliable)) return PCM_X_QUEUE_CORRUPT; if (!pcm_x_local_admission_ref_valid(&tag_slot->blocker_snapshot_ref, @@ -17724,7 +18698,9 @@ pcm_x_local_blocker_participant_drain_locked(PcmXLocalTagSlot *tag_slot, PcmXSlo || !BufferTagsEqual(&tag_slot->tag, &tag_slot->blocker_snapshot_ref.identity.tag) || tag_slot->blocker_snapshot_ref.identity.cluster_epoch != tag_slot->cluster_epoch) return PCM_X_QUEUE_CORRUPT; - if (!pcm_x_local_blocker_ack_tombstone_exact(tag_slot, authenticated_master_node) + if ((!pcm_x_local_blocker_ack_tombstone_exact(tag_slot, authenticated_master_node) + && !pcm_x_local_empty_blocker_commit_exact(tag_slot, authenticated_master_node, + authenticated_master_session)) || tag_slot->blocker_snapshot_reliable.response_tombstone_mask != 0) { blocker_state = pcm_x_local_blocker_lane_retire_state(tag_slot); return blocker_state == PCM_X_QUEUE_CORRUPT ? blocker_state : PCM_X_QUEUE_NOT_READY; @@ -17806,8 +18782,8 @@ pcm_x_local_holder_drain_poll_exact(const PcmXDrainPollPayload *poll, goto holder_drain_done; } if (pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref)) { - result = pcm_x_local_blocker_participant_drain_locked(tag_slot, tag_ref, poll, - authenticated_master_node); + result = pcm_x_local_blocker_participant_drain_locked( + tag_slot, tag_ref, poll, authenticated_master_node, authenticated_master_session); fail_closed = result == PCM_X_QUEUE_CORRUPT; goto holder_drain_done; } @@ -17886,6 +18862,20 @@ PcmXQueueResult cluster_pcm_x_local_drain_poll_exact(const PcmXDrainPollPayload *poll, int32 authenticated_master_node, uint64 authenticated_master_session) +{ + return cluster_pcm_x_local_drain_poll_certificate_exact(poll, authenticated_master_node, + authenticated_master_session, NULL); +} + + +/* First-consumption variant: capture the completion certificate under the + * same tag lock that publishes TERMINAL_DRAINED, so the caller can release + * the self-source immutable record against the exact protocol ledger. */ +PcmXQueueResult +cluster_pcm_x_local_drain_poll_certificate_exact(const PcmXDrainPollPayload *poll, + int32 authenticated_master_node, + uint64 authenticated_master_session, + PcmXLocalDrainCertificate *certificate_out) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; PcmXRuntimeSnapshot runtime; @@ -17905,6 +18895,8 @@ cluster_pcm_x_local_drain_poll_exact(const PcmXDrainPollPayload *poll, bool fail_closed = false; bool promote_candidate; + if (certificate_out != NULL) + memset(certificate_out, 0, sizeof(*certificate_out)); if (header == NULL || poll == NULL || !pcm_x_local_terminal_ref_valid(&poll->ref) || poll->drain_generation == 0 || poll->drain_generation == UINT64_MAX || authenticated_master_node < 0 || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT @@ -18089,8 +19081,8 @@ cluster_pcm_x_local_drain_poll_exact(const PcmXDrainPollPayload *poll, } } else if (!pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref) && pcm_x_ticket_locator_equal(&tag_slot->blocker_snapshot_ref, &poll->ref)) { - external_result = pcm_x_local_blocker_participant_drain_locked(tag_slot, tag_ref, poll, - authenticated_master_node); + external_result = pcm_x_local_blocker_participant_drain_locked( + tag_slot, tag_ref, poll, authenticated_master_node, authenticated_master_session); if (external_result != PCM_X_QUEUE_OK && external_result != PCM_X_QUEUE_DUPLICATE) { result = external_result; fail_closed = result == PCM_X_QUEUE_CORRUPT; @@ -18100,6 +19092,18 @@ cluster_pcm_x_local_drain_poll_exact(const PcmXDrainPollPayload *poll, tag_slot->terminal_drain_generation = poll->drain_generation; pg_write_barrier(); (void)pcm_x_slot_flags_fetch_or(&tag_slot->slot, PCM_X_LOCAL_TAG_F_TERMINAL_DRAINED); + /* Completion certificate: the writer-round ledger is captured under this + * same lock, before RETIRE can clear it, so the caller can verify the + * self-source release against the exact protocol evidence instead of the + * live descriptor (which may legitimately have moved on or been evicted). */ + if (certificate_out != NULL) { + certificate_out->ref = tag_slot->ref; + certificate_out->image = tag_slot->image; + certificate_out->committed_own_generation = tag_slot->committed_own_generation; + certificate_out->master_session_incarnation = tag_slot->master_session_incarnation; + certificate_out->master_node = tag_slot->master_node; + certificate_out->valid = true; + } result = PCM_X_QUEUE_OK; drain_release_gate: @@ -18331,6 +19335,7 @@ pcm_x_local_reset_holder_only_queue(PcmXLocalTagSlot *tag_slot) tag_slot->active_writer_slot_generation = 0; tag_slot->closed_round_member_count = 0; tag_slot->committed_own_generation = 0; + tag_slot->grant_base_own_generation = 0; } @@ -18341,6 +19346,52 @@ static PcmXQueueResult pcm_x_local_ready_leader_wake_locked(PcmXLocalTagSlot *ta bool *candidate_out); +/* Shared eligibility for retiring ONLY the writer lane of an independent + * (distinct-ticket) dual terminal while the holder lane stays authoritative. + * The RETIRE preflight (candidate scan) and the detach (retain arm) MUST + * agree bit-for-bit through this one predicate: a first-pass yes with a + * second-pass no escalates to the collect's fail-closed exit. + * + * The narrow proven shape is one closed writer round of exactly the terminal + * target under a standing REVOKE_BARRIER with sane round/cutoff invariants; + * the FIFO may hold only next-round followers (they are promoted when the + * holder ticket's own later RETIRE closes the still-frozen round -- refusing + * them here would deadlock: a pre-joined follower waits for exactly this + * RETIRE to drop the barrier). A zero grant generation on the external lane + * (a cancelled blocker-participant DRAIN) is eligible too: its staged + * retirement is writer lane first, then the cancelled lane by its own + * watermark (see pcm_x_local_cross_lane_holder_terminal_retirable). + * Caller guarantees: both terminal masks complete and the same-ref-dual + * classifier returned OK with dual=false (distinct tickets). */ +static bool +pcm_x_local_independent_writer_retire_eligible(const PcmXLocalTagSlot *tag_slot, uint32 flags) +{ + const PcmXTicketRef *external_ref; + + if (tag_slot == NULL) + return false; + external_ref = pcm_x_local_external_terminal_ref(tag_slot); + return external_ref != NULL && !pcm_x_ticket_ref_is_zero(external_ref) + && external_ref->handle.ticket_id != tag_slot->ref.handle.ticket_id + && (flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 + && tag_slot->closed_round_member_count == 1 && tag_slot->membership_count >= 1 + && tag_slot->local_round != 0 && tag_slot->local_round != UINT32_MAX + && tag_slot->next_sequence != 0 && tag_slot->cutoff_sequence != 0 + && tag_slot->cutoff_sequence < tag_slot->next_sequence + && ((tag_slot->membership_count == 1) + == (tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX)) + && ((tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX) + == (tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX)) + && tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->leader_slot_generation == 0 + && tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->active_writer_slot_generation == 0 + && tag_slot->active_holder_head_index == PCM_X_INVALID_SLOT_INDEX + && pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) + && pcm_x_local_holder_lane_retire_state(tag_slot, flags) == PCM_X_QUEUE_OK; +} + + static PcmXQueueResult pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_protocol, PcmXLocalHandle *promoted_out) @@ -18365,6 +19416,7 @@ pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_pr bool detach_after_close = false; bool detach_tag = false; bool gate_claimed = false; + bool retain_independent_holder = false; bool same_ref_dual = false; bool ready_leader_candidate = false; bool target_in_closed_round; @@ -18449,6 +19501,23 @@ pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_pr goto detach_local_domain_done; } flags = pcm_x_slot_flags_read(&tag_slot->slot); + /* An exact unlinked next-round CANCELLED member detaches by releasing + * only its own membership even while terminal evidence occupies the tag: + * it was never part of the frozen round, so the dual terminals, the + * holder lane, the barrier and the master binding all stay byte-exact. + * Without this arm the cancelled membership lingers (the generic path + * refuses under terminal evidence) and wedges both the writer-lane + * eligibility and the holder RETIRE's frozen-round close. */ + if (!retire_protocol && pcm_x_slot_state_read(&member->slot) == PCM_XL_CANCELLED + && member->admitted_round == tag_slot->local_round + 1 + && (flags & (PCM_X_LOCAL_TAG_F_TERMINAL_MASK | PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK)) + != 0) { + tag_slot->membership_count--; + pg_write_barrier(); + pcm_x_slot_state_write(&member->slot, PCM_XL_DETACHING); + result = PCM_X_QUEUE_OK; + goto detach_local_domain_done; + } if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) != 0) { uint32 terminal_flags = flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK; @@ -18489,6 +19558,27 @@ pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_pr goto detach_local_domain_done; } } + /* Independent dual terminal (dump-0x3d): a complete writer terminal + * and a complete holder terminal under DISTINCT master tickets. Only + * the shared narrow eligibility (one closed writer round of exactly + * this target under the standing barrier; FIFO holds at most + * next-round followers) may retire the writer lane, keeping the + * holder lane, the barrier and the master binding byte-exact on the + * LIVE tag for the holder ticket's own later watermark. Any other + * distinct-ticket dual stays NOT_READY with zero mutation. */ + if (!same_ref_dual && !cancel_same_ref_dual + && (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) + == PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK + && (flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) == PCM_X_LOCAL_TAG_F_TERMINAL_MASK) { + retain_independent_holder + = member->admitted_round == tag_slot->local_round + && member->local_sequence <= tag_slot->cutoff_sequence + && pcm_x_local_independent_writer_retire_eligible(tag_slot, flags); + if (!retain_independent_holder) { + result = PCM_X_QUEUE_NOT_READY; + goto detach_local_domain_done; + } + } } target_in_closed_round = (flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && member->admitted_round == tag_slot->local_round @@ -18503,7 +19593,8 @@ pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_pr result = PCM_X_QUEUE_CORRUPT; goto detach_local_domain_done; } - if (retire_protocol && target_in_closed_round && tag_slot->closed_round_member_count == 1) { + if (retire_protocol && !retain_independent_holder && target_in_closed_round + && tag_slot->closed_round_member_count == 1) { result = pcm_x_local_final_member_round_plan_locked(tag_slot, tag_ref, &round_plan); if (result != PCM_X_QUEUE_OK) goto detach_local_domain_done; @@ -18521,10 +19612,14 @@ pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_pr /* Writer membership and holder transfer are independent lifetimes on * the same tag. Neither returning to holder-only form nor deleting the * tag may erase the master binding while an exact holder ticket, send - * leg, or terminal-drain outcome remains authoritative. */ - if (!pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref) - || !pcm_x_transfer_leg_is_clear(&tag_slot->holder_reliable) - || ((flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 && !cancel_same_ref_dual)) { + * leg, or terminal-drain outcome remains authoritative. The proven + * independent dual terminal is the one exception: its writer lane may + * retire alone, leaving the holder lane authoritative on the tag. */ + if (!retain_independent_holder + && (!pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref) + || !pcm_x_transfer_leg_is_clear(&tag_slot->holder_reliable) + || ((flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 + && !cancel_same_ref_dual))) { result = PCM_X_QUEUE_NOT_READY; goto detach_local_domain_done; } @@ -18537,7 +19632,7 @@ pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_pr goto detach_local_domain_done; } pcm_x_local_reset_holder_only_queue(tag_slot); - } else if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0) { + } else if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0 && !retain_independent_holder) { pg_write_barrier(); pcm_x_slot_state_write(&tag_slot->slot, PCM_X_TAG_DETACHING); detach_tag = true; @@ -18576,6 +19671,7 @@ pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_pr * and its generation rekey correctly classifies the tag as corrupt. */ memset(&tag_slot->image, 0, sizeof(tag_slot->image)); tag_slot->committed_own_generation = 0; + tag_slot->grant_base_own_generation = 0; /* DRAIN closes the old round by leaving its response opcode as a replay * tombstone. RETIRE consumes that tombstone with the membership: a * promoted successor must see a pristine application leg or the requester @@ -18787,6 +19883,62 @@ pcm_x_local_ready_leader_wake_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag } +/* + * A fully-drained cross-lane holder/blocker terminal (no writer-terminal + * companion, so it is not a same-ref dual) belongs to an already-revoked older + * cohort: its S lane has finished DRAIN and it carries a completed grant. The + * newer writer's REVOKE_BARRIER guards that writer's own revoke round, not this + * older holder lane's terminal cleanup, so the older lane may retire itself + * while every writer ref / round / barrier / FIFO field is preserved. Retiring + * it also clears HOLDER_TERMINAL_MASK, which lets the empty-frozen-round path + * finally drop the REVOKE_BARRIER. Cancel-duals (grant_generation == 0) are + * routed through the same-ref-dual path and are excluded here. + * + * The exemption is strictly cross-lane: a writer-lane ref that aliases the + * external ref (or merely collides with its master-unique ticket id) cannot + * be an older cohort and keeps the pre-exemption NOT_READY verdict. There is + * deliberately no ticket-order requirement beyond that: a cancelled + * predecessor leader's duplicate-ACK tombstone legally lingers in + * tag_slot->ref with an OLDER ticket until the successor's ENQUEUE arm + * retires it, and refusing retirement there could wedge the empty-frozen + * path permanently if the successor exits before arming. + * + * Precondition: the caller MUST have validated + * pcm_x_local_holder_lane_retire_state() == OK before consulting this helper. + * The test here does not itself re-prove the DRAIN evidence + * (holder_terminal_drain_generation, a clear reliable leg, a DRAIN_POLL last + * response); both call sites run that check immediately above the guard. + */ +static inline bool +pcm_x_local_cross_lane_holder_terminal_retirable(const PcmXLocalTagSlot *tag_slot, uint32 flags, + const PcmXTicketRef *external_ref) +{ + if (tag_slot == NULL || external_ref == NULL) + return false; + if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) != 0 + || (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) + != PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) + return false; + /* A cancelled lane (zero grant generation) that shared the tag with a + * DISTINCT-ticket writer terminal is the staged second half of an + * independent dual: same-ref cancel forms route through the + * same-ref-dual path and never reach this guard, so once the writer + * lane has fully retired (no terminal bits above, ref consumed) the + * cancelled lane retires by its own watermark exactly like a granted + * one. While any writer ref remains, keep refusing it. */ + if (external_ref->grant_generation == 0) + return pcm_x_ticket_ref_is_zero(&tag_slot->ref); + /* Per-session ticket ids are master-unique (monotonic allocator, exhaustion + * fails closed), so id equality alone is complete for the alias class. Do + * not "simplify" this to full-ref equality: that would stop refusing an id + * collision under a distinct identity and REDUCE fail-closed coverage. */ + if (!pcm_x_ticket_ref_is_zero(&tag_slot->ref) + && external_ref->handle.ticket_id == tag_slot->ref.handle.ticket_id) + return false; + return true; +} + + static PcmXQueueResult pcm_x_local_retire_candidate_at(Size slot_index, const PcmXRetirePayload *request, int32 authenticated_master_node, @@ -18873,6 +20025,39 @@ pcm_x_local_retire_candidate_at(Size slot_index, const PcmXRetirePayload *reques result = PCM_X_QUEUE_NOT_READY; goto candidate_done; } + /* An independent (distinct-ticket) dual terminal may offer its writer + * lane only through the shared eligibility predicate; see the retain + * arm in pcm_x_local_detach_terminal_common. Refusing here keeps the + * whole RETIRE preflight retryable instead of tripping the + * second-pass fail-closed exit on an unproven wider shape. When the + * same watermark also covers the holder lane, rehearse the frozen + * round close it will need in its projected post-writer-retire form + * -- every fallible check must precede the second pass's first + * mutation. */ + if ((flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 && !same_ref_dual + && !cancel_same_ref_dual) { + if (!pcm_x_local_independent_writer_retire_eligible(tag_slot, flags)) { + result = PCM_X_QUEUE_NOT_READY; + goto candidate_done; + } + external_ref = pcm_x_local_external_terminal_ref(tag_slot); + if (external_ref != NULL + && external_ref->handle.ticket_id <= request->retire_through_ticket_id) { + PcmXLocalRoundClosePlan close_plan; + + result = pcm_x_local_independent_holder_close_plan_locked(tag_slot, tag_ref, true, + &close_plan); + if (result != PCM_X_QUEUE_OK) + goto candidate_done; + /* The close this episode will apply promotes that follower; + * project its wake now or the promotion degrades to the + * waiters' poll interval. */ + if (close_plan.candidate != NULL) { + *holder_wake_out = close_plan.candidate->identity; + *holder_wake_candidate_out = true; + } + } + } if (tag_slot->ref.handle.ticket_id == request->retire_through_ticket_id) *contains_watermark_out = true; if (tag_slot->ref.handle.ticket_id <= request->retire_through_ticket_id) { @@ -18896,10 +20081,38 @@ pcm_x_local_retire_candidate_at(Size slot_index, const PcmXRetirePayload *reques if (external_ref->handle.ticket_id == request->retire_through_ticket_id) *contains_watermark_out = true; if (external_ref->handle.ticket_id <= request->retire_through_ticket_id) { - if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual) { + /* A selected writer candidate on the same slot retires first; the + * next scan iteration re-evaluates this holder lane against the + * post-writer-retire flags (its close was already rehearsed in + * the writer arm above when the watermark covers both lanes). */ + if (!same_ref_dual && *candidate_out) + goto candidate_done; + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual + && !pcm_x_local_cross_lane_holder_terminal_retirable(tag_slot, flags, + external_ref)) { result = PCM_X_QUEUE_NOT_READY; goto candidate_done; } + /* When this holder RETIRE will be the last terminal authority on + * a barrier-frozen tag, rehearse the round close its detach must + * apply; the second pass repeats the same read-only rehearsal + * before consuming the lane. */ + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual + && !cancel_same_ref_dual && (flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) == 0 + && tag_slot->closed_round_member_count == 0 + && pcm_x_ticket_ref_is_zero(&tag_slot->ref)) { + PcmXLocalRoundClosePlan close_plan; + + result = pcm_x_local_independent_holder_close_plan_locked(tag_slot, tag_ref, false, + &close_plan); + if (result != PCM_X_QUEUE_OK) + goto candidate_done; + /* Project the promotion this close will apply. */ + if (close_plan.candidate != NULL) { + *holder_wake_out = close_plan.candidate->identity; + *holder_wake_candidate_out = true; + } + } /* A generation-zero dual terminal must validate and consume the * CANCELLED membership before its blocker evidence disappears. Keep * the writer candidate selected above; detach_terminal_common closes @@ -18916,7 +20129,7 @@ pcm_x_local_retire_candidate_at(Size slot_index, const PcmXRetirePayload *reques *candidate_out = true; } } - if (result == PCM_X_QUEUE_OK && *holder_candidate_out) + if (result == PCM_X_QUEUE_OK && *holder_candidate_out && !*holder_wake_candidate_out) result = pcm_x_local_ready_leader_wake_locked(tag_slot, tag_ref, holder_wake_out, NULL, holder_wake_candidate_out); @@ -18933,6 +20146,7 @@ pcm_x_local_holder_detach_terminal_exact(const PcmXTicketRef *ref, int32 authent bool *ready_leader_candidate_out) { PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXLocalRoundClosePlan round_plan; PcmXLocalTagSlot *tag_slot; const PcmXTicketRef *external_ref; PcmXSlotRef tag_ref; @@ -18955,6 +20169,7 @@ pcm_x_local_holder_detach_terminal_exact(const PcmXTicketRef *ref, int32 authent memset(ready_leader_out, 0, sizeof(*ready_leader_out)); *ready_leader_candidate_out = false; memset(&ready_leader, 0, sizeof(ready_leader)); + memset(&round_plan, 0, sizeof(round_plan)); pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_SHARED, NULL); directory_result @@ -19005,7 +20220,8 @@ pcm_x_local_holder_detach_terminal_exact(const PcmXTicketRef *ref, int32 authent goto holder_detach_release_gate; goto holder_detach_domain_done; } - if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual) { + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual + && !pcm_x_local_cross_lane_holder_terminal_retirable(tag_slot, flags, external_ref)) { result = PCM_X_QUEUE_NOT_READY; goto holder_detach_release_gate; } @@ -19015,6 +20231,26 @@ pcm_x_local_holder_detach_terminal_exact(const PcmXTicketRef *ref, int32 authent fail_closed = result == PCM_X_QUEUE_CORRUPT; goto holder_detach_domain_done; } + /* When this holder lane is the LAST terminal authority on a + * barrier-frozen tag (an independent dual whose writer lane already + * retired without closing the round), rehearse the frozen-round close + * BEFORE any evidence is consumed: next-round followers may have joined + * between the two watermarks, and consuming the holder lane without a + * validated close would strand them behind a barrier nothing can drop. + * Every fallible check lives in the rehearsal; the apply below is + * infallible. */ + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual + && (flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) == 0 + && tag_slot->closed_round_member_count == 0 && pcm_x_ticket_ref_is_zero(&tag_slot->ref)) { + result = pcm_x_local_independent_holder_close_plan_locked(tag_slot, tag_ref, false, + &round_plan); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + if (!fail_closed) + goto holder_detach_release_gate; + goto holder_detach_domain_done; + } + } if (blocker_terminal) { memset(&tag_slot->blocker_snapshot_ref, 0, sizeof(tag_slot->blocker_snapshot_ref)); tag_slot->blocker_set_generation = 0; @@ -19023,10 +20259,19 @@ pcm_x_local_holder_detach_terminal_exact(const PcmXTicketRef *ref, int32 authent } else { memset(&tag_slot->holder_ref, 0, sizeof(tag_slot->holder_ref)); memset(&tag_slot->holder_image, 0, sizeof(tag_slot->holder_image)); + tag_slot->holder_required_page_scn = 0; memset(&tag_slot->holder_reliable, 0, sizeof(tag_slot->holder_reliable)); } tag_slot->holder_terminal_drain_generation = 0; (void)pcm_x_slot_flags_fetch_and(&tag_slot->slot, ~PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + pcm_x_local_empty_frozen_round_apply_locked(tag_slot, &round_plan); + /* Report the promoted next-round leader through the same wake output the + * preflight projected; without it the promotion is only discovered by + * the waiter's poll interval. */ + if (round_plan.close_round && round_plan.candidate != NULL) { + ready_leader = round_plan.candidate->identity; + ready_leader_candidate = true; + } flags = pcm_x_slot_flags_read(&tag_slot->slot); detach_tag = tag_slot->membership_count == 0 && tag_slot->closed_round_member_count == 0 @@ -19211,6 +20456,14 @@ pcm_x_local_final_member_round_preflight_exact(const PcmXLocalHandle *handle, result = PCM_X_QUEUE_OK; goto preflight_done; } + /* An independent (distinct-ticket) dual terminal retires its writer lane + * WITHOUT closing the round -- the holder lane keeps the fence and its + * own later RETIRE closes and promotes. Projecting the final-member + * promotion here would demand a wake the detach never produces. */ + if ((flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 && !same_ref_dual) { + result = PCM_X_QUEUE_OK; + goto preflight_done; + } result = pcm_x_local_final_member_round_plan_locked(tag_slot, tag_ref, &plan); if (result == PCM_X_QUEUE_OK && plan.candidate != NULL) { if (*projected_wake_candidate_out) { @@ -19266,6 +20519,18 @@ pcm_x_local_wake_actual_match(const PcmXLocalWakeBatch *wake_batch, } +/* One-shot evidence ahead of the retire-collect fuse: every corruption arm + * funnels to the same fail-closed line, so name the arm and its uncoerced + * detail here. For scan arms a/b are the slot index and the raw result; + * for the wake-count arm they are the projected and actual wake counts. */ +static void +pcm_x_local_retire_collect_note(const char *site, uint64 a, uint64 b) +{ + ereport(LOG, (errmsg("PCM-X local retire collect fails closed at %s (a=%llu, b=%llu)", site, + (unsigned long long)a, (unsigned long long)b))); +} + + PcmXQueueResult cluster_pcm_x_local_retire_up_to_collect_exact(const PcmXRetirePayload *request, int32 authenticated_master_node, @@ -19357,6 +20622,7 @@ cluster_pcm_x_local_retire_up_to_collect_exact(const PcmXRetirePayload *request, return result; } if (!pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_TAG, &view)) { + pcm_x_local_retire_collect_note("allocator-view", 0, 0); result = PCM_X_QUEUE_CORRUPT; fail_closed = true; goto retire_failed; @@ -19407,6 +20673,7 @@ cluster_pcm_x_local_retire_up_to_collect_exact(const PcmXRetirePayload *request, &writer_candidate_ref, &writer_candidate, &holder_candidate, &candidate, &contains_watermark, &holder_wake, &holder_wake_candidate); if (result != PCM_X_QUEUE_OK) { + pcm_x_local_retire_collect_note("second-pass-candidate", (uint64)i, (uint64)result); fail_closed = true; result = PCM_X_QUEUE_CORRUPT; goto retire_failed; @@ -19430,6 +20697,9 @@ cluster_pcm_x_local_retire_up_to_collect_exact(const PcmXRetirePayload *request, } } if (result != PCM_X_QUEUE_OK) { + pcm_x_local_retire_collect_note(holder_candidate ? "holder-detach" + : "writer-detach", + (uint64)i, (uint64)result); fail_closed = true; result = PCM_X_QUEUE_CORRUPT; goto retire_failed; @@ -19438,6 +20708,7 @@ cluster_pcm_x_local_retire_up_to_collect_exact(const PcmXRetirePayload *request, result = pcm_x_local_wake_actual_match(wake_batch, &writer_wake, projected_wake_count, &actual_wake_count); if (result != PCM_X_QUEUE_OK) { + pcm_x_local_retire_collect_note("wake-match", (uint64)i, (uint64)result); fail_closed = true; result = PCM_X_QUEUE_CORRUPT; goto retire_failed; @@ -19446,6 +20717,8 @@ cluster_pcm_x_local_retire_up_to_collect_exact(const PcmXRetirePayload *request, } while (candidate); } if (wake_batch != NULL && actual_wake_count != projected_wake_count) { + pcm_x_local_retire_collect_note("wake-count", (uint64)projected_wake_count, + (uint64)actual_wake_count); result = PCM_X_QUEUE_CORRUPT; fail_closed = true; goto retire_failed; @@ -19483,6 +20756,7 @@ cluster_pcm_x_local_retire_up_to_collect_exact(const PcmXRetirePayload *request, /* No tag was mutated, so a retryable/stale preflight can release the claim. * Structural corruption retains both exact episode witnesses. */ if (result == PCM_X_QUEUE_CORRUPT) { + pcm_x_local_retire_collect_note("preflight", 0, (uint64)result); fail_closed = true; goto retire_failed; } diff --git a/src/backend/cluster/cluster_pcm_x_image_fetch.c b/src/backend/cluster/cluster_pcm_x_image_fetch.c index a73f60b6c0..53ce06d958 100644 --- a/src/backend/cluster/cluster_pcm_x_image_fetch.c +++ b/src/backend/cluster/cluster_pcm_x_image_fetch.c @@ -86,43 +86,90 @@ cluster_pcm_x_image_fetch_build_request(const PcmXLocalProgress *progress, int32 bool -cluster_pcm_x_image_fetch_request_exact(const ClusterICEnvelope *env, - const GcsBlockRequestPayload *request, - const PcmXLocalHolderProgress *holder, int32 holder_node, - int32 current_master_node, uint64 current_epoch) +cluster_pcm_x_image_fetch_request_exact_diagnosed(const ClusterICEnvelope *env, + const GcsBlockRequestPayload *request, + const PcmXLocalHolderProgress *holder, + int32 holder_node, int32 current_master_node, + uint64 current_epoch, + PcmXImageFetchRequestRefusal *refusal_out) { static const uint8 zero_reserved[sizeof(request->reserved_0)] = { 0 }; int32 decoded_backend_id; int32 decoded_master_node; int32 decoded_requester_node; - return env != NULL && request != NULL && holder != NULL && holder_node >= 0 - && holder_node < PCM_X_PROTOCOL_NODE_LIMIT && current_master_node >= 0 - && current_master_node < PCM_X_PROTOCOL_NODE_LIMIT - && env->source_node_id == (uint32)request->sender_node - && env->dest_node_id == (uint32)holder_node && env->payload_length == sizeof(*request) - && request->sender_node >= 0 && request->sender_node < PCM_X_PROTOCOL_NODE_LIMIT - && request->requester_backend_id > 0 && request->epoch == current_epoch - && request->transition_id == (uint8)PCM_TRANS_N_TO_S - && memcmp(request->reserved_0, zero_reserved, sizeof(zero_reserved)) == 0 - && holder->master_node == current_master_node && holder->master_session_incarnation != 0 - && holder->ref.grant_generation != 0 && holder->ref.handle.ticket_id != 0 - && holder->ref.handle.queue_generation != 0 - && holder->ref.identity.cluster_epoch == current_epoch - && holder->ref.identity.wait_seq != 0 - && BufferTagsEqual(&holder->ref.identity.tag, &request->tag) - && holder->ref.identity.node_id == request->sender_node - && holder->pending_opcode == PGRAC_IC_MSG_PCM_X_IMAGE_READY - && holder->phase == PGRAC_IC_MSG_PCM_X_IMAGE_READY && holder->flags == 0 - && pcm_x_image_fetch_token_valid(&holder->image, current_master_node) - && holder->image.image_id == request->request_id - && holder->image.source_node == (uint32)holder_node - && cluster_pcm_x_image_id_decode(request->request_id, &decoded_master_node, NULL) - && decoded_master_node == current_master_node - && cluster_gcs_requester_id_decode(holder->ref.identity.request_id, - &decoded_requester_node, &decoded_backend_id, NULL) - && decoded_requester_node == request->sender_node - && decoded_backend_id == request->requester_backend_id; + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_NONE; + if (env == NULL || request == NULL || holder == NULL || holder_node < 0 + || holder_node >= PCM_X_PROTOCOL_NODE_LIMIT || current_master_node < 0 + || current_master_node >= PCM_X_PROTOCOL_NODE_LIMIT) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_ARGUMENT; + return false; + } + if (env->source_node_id != (uint32)request->sender_node + || env->dest_node_id != (uint32)holder_node || env->payload_length != sizeof(*request)) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_ENVELOPE; + return false; + } + if (request->sender_node < 0 || request->sender_node >= PCM_X_PROTOCOL_NODE_LIMIT + || request->requester_backend_id <= 0 || request->epoch != current_epoch + || request->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(request->reserved_0, zero_reserved, sizeof(zero_reserved)) != 0) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_REQUEST; + return false; + } + if (holder->master_node != current_master_node || holder->master_session_incarnation == 0) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_MASTER; + return false; + } + if (holder->ref.grant_generation == 0 || holder->ref.handle.ticket_id == 0 + || holder->ref.handle.queue_generation == 0 + || holder->ref.identity.cluster_epoch != current_epoch || holder->ref.identity.wait_seq == 0 + || !BufferTagsEqual(&holder->ref.identity.tag, &request->tag) + || holder->ref.identity.node_id != request->sender_node) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_REF; + return false; + } + if (holder->pending_opcode != PGRAC_IC_MSG_PCM_X_IMAGE_READY + || holder->phase != PGRAC_IC_MSG_PCM_X_IMAGE_READY || holder->flags != 0) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_LEG; + return false; + } + if (!pcm_x_image_fetch_token_valid(&holder->image, current_master_node) + || holder->image.image_id != request->request_id + || holder->image.source_node != (uint32)holder_node + || !cluster_pcm_x_image_id_decode(request->request_id, &decoded_master_node, NULL) + || decoded_master_node != current_master_node) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_IMAGE; + return false; + } + if (!cluster_gcs_requester_id_decode(holder->ref.identity.request_id, &decoded_requester_node, + &decoded_backend_id, NULL) + || decoded_requester_node != request->sender_node + || decoded_backend_id != request->requester_backend_id) { + if (refusal_out != NULL) + *refusal_out = PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_REQUESTER_ID; + return false; + } + return true; +} + + +bool +cluster_pcm_x_image_fetch_request_exact(const ClusterICEnvelope *env, + const GcsBlockRequestPayload *request, + const PcmXLocalHolderProgress *holder, int32 holder_node, + int32 current_master_node, uint64 current_epoch) +{ + return cluster_pcm_x_image_fetch_request_exact_diagnosed( + env, request, holder, holder_node, current_master_node, current_epoch, NULL); } diff --git a/src/backend/cluster/cluster_sf_dep.c b/src/backend/cluster/cluster_sf_dep.c index f52774731c..f1193388f3 100644 --- a/src/backend/cluster/cluster_sf_dep.c +++ b/src/backend/cluster/cluster_sf_dep.c @@ -371,6 +371,109 @@ cluster_sf_peer_supports_pcm_x_convert(int32 peer_id) return (capabilities & PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1) != 0; } +/* A' rebase: the V2 INSTALL_READY frame is legal on this connection only when + * its verified HELLO advertised the rebase protocol capability. */ +bool +cluster_sf_peer_supports_pcm_x_rebase(int32 peer_id) +{ + uint32 capabilities; + + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return false; + + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + capabilities = cluster_sf_peer_cap_bits(&ClusterSfDep->peer_capabilities[peer_id]); + LWLockRelease(&ClusterSfDep->lock); + return (capabilities & PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1) != 0; +} + +bool +cluster_sf_peer_supports_pcm_x_source_floor(int32 peer_id) +{ + uint32 capabilities; + + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return false; + + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + capabilities = cluster_sf_peer_cap_bits(&ClusterSfDep->peer_capabilities[peer_id]); + LWLockRelease(&ClusterSfDep->lock); + return (capabilities & PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1) != 0; +} + +/* One lock-coherent sample for choosing the type-49 wire version. CONVERT + * authenticates the PCM-X family; SOURCE_FLOOR and its connection generation + * are sampled from that same record so callers never pair bits across a + * reconnect. */ +bool +cluster_sf_peer_pcm_x_source_floor_sample(int32 peer_id, bool *source_floor_out, + uint32 *generation_out) +{ + bool supported; + + if (source_floor_out != NULL) + *source_floor_out = false; + if (generation_out != NULL) + *generation_out = 0; + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return false; + + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + supported = cluster_sf_peer_cap_family_sample( + &ClusterSfDep->peer_capabilities[peer_id], PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1, source_floor_out, generation_out); + LWLockRelease(&ClusterSfDep->lock); + return supported; +} + +/* Drain-side exact fence for a capability-bound LMS slot. */ +bool +cluster_sf_peer_capability_generation_matches(int32 peer_id, uint32 required_capabilities, + uint32 expected_generation) +{ + bool matches; + + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES + || required_capabilities == 0) + return false; + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + matches = cluster_sf_peer_cap_generation_matches_exact( + &ClusterSfDep->peer_capabilities[peer_id], required_capabilities, expected_generation); + LWLockRelease(&ClusterSfDep->lock); + return matches; +} + +/* + * cluster_sf_peer_pcm_x_capability_sample + * + * review P0-2: one lock-coherent sample of the PCM-X capability family on + * the peer's CURRENT connection. True iff the verified HELLO advertised + * PCM_X_CONVERT; the REBASE bit and the capability-record generation come + * from the SAME locked read. The formation collector samples this on both + * sides of its authority pass: a reconnect between the samples changes the + * generation and rejects the tick, so a stale connection's REBASE bit can + * never be bound to a fresh session/binding. + */ +bool +cluster_sf_peer_pcm_x_capability_sample(int32 peer_id, bool *rebase_out, uint32 *generation_out) +{ + bool supported; + + if (rebase_out != NULL) + *rebase_out = false; + if (generation_out != NULL) + *generation_out = 0; + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return false; + + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + supported = cluster_sf_peer_cap_family_sample( + &ClusterSfDep->peer_capabilities[peer_id], PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, rebase_out, generation_out); + LWLockRelease(&ClusterSfDep->lock); + return supported; +} + /* Return one lock-coherent snapshot of the capability-record generation whose * verified HELLO advertised PCM-X. Tier1 owns this record on CONTROL and can * sample it around another authority read to reject reconnect ABA; RDMA keeps diff --git a/src/backend/cluster/cluster_tt_local.c b/src/backend/cluster/cluster_tt_local.c index c3fbd7324c..b44282ecb8 100644 --- a/src/backend/cluster/cluster_tt_local.c +++ b/src/backend/cluster/cluster_tt_local.c @@ -57,6 +57,7 @@ #include "cluster/cluster_undo_record_api.h" /* spec-3.12 D2b cluster_undo_tt_rollover_locked */ #include "cluster/cluster_tt_status.h" #include "cluster/cluster_tt_status_hint.h" /* spec-3.2 D4 wire emit append */ +#include "cluster/cluster_uba.h" /* P0-33 exact data-ref alias */ #include "cluster/storage/cluster_undo_alloc.h" /* cluster_undo_active_segment_for_node_or_create */ #ifdef USE_PGRAC_CLUSTER @@ -152,6 +153,12 @@ typedef struct ClusterTTLocalBinding { * segment away (cluster_tt_slot_get_wrap would then ERROR on the stale id). */ uint16 wrap; uint32 cluster_epoch; /* snapshot at bind time */ + /* Real undo records use an independent record cursor. Its segment may + * differ from segment_id after rollover, while the page ref carries that + * record segment as its exact-key segment. */ + uint16 *active_alias_segments; + uint16 active_alias_count; + uint16 active_alias_capacity; } ClusterTTLocalBinding; static ClusterTTLocalBinding *cluster_tt_local_bindings = NULL; @@ -468,40 +475,78 @@ cluster_tt_local_finish_bindings(bool committed, SCN commit_scn) cluster_tt_local_reset_binding(); } -/* - * build_local_key -- compose ClusterTTStatusKey for `xid`. - * - * Returns true with the key populated when an xact-local binding - * exists for `xid` (F11 real allocator path); returns false otherwise - * and leaves `*out` zeroed. spec-3.4b F7: production code paths MUST - * NOT take a provisional id when the binding is absent -- this would - * create a real/provisional key collision risk. Callers should - * silently skip install when build_local_key returns false (the - * matching DML never stamped an ITL slot, so no overlay entry is - * needed). - */ +/* Compose either the canonical key or one page-ref segment alias. */ static bool -build_local_key(TransactionId xid, ClusterTTStatusKey *out) +build_binding_key(const ClusterTTLocalBinding *binding, uint16 segment_id, ClusterTTStatusKey *out) { - uint32 seg; - uint16 off; - uint32 tt_id; - uint32 epoch; - memset(out, 0, sizeof(*out)); - - if (!cluster_tt_local_peek_binding(xid, &seg, &off, &tt_id, &epoch, NULL)) + if (binding == NULL || !TransactionIdIsValid(binding->top_xid) || segment_id == 0) return false; out->origin_node_id = (uint16)cluster_node_id; - out->undo_segment_id = (uint16)seg; - out->tt_slot_id = tt_id; - out->cluster_epoch = epoch; - out->local_xid = xid; - /* _reserved + _reserved2 already zero from memset. */ + out->undo_segment_id = segment_id; + out->tt_slot_id = cluster_tt_slot_offset_to_id(binding->slot_offset); + out->cluster_epoch = binding->cluster_epoch; + out->local_xid = binding->top_xid; return true; } +static bool +build_local_key(TransactionId xid, ClusterTTStatusKey *out) +{ + int idx = cluster_tt_local_find_binding(xid); + + if (idx < 0) { + memset(out, 0, sizeof(*out)); + return false; + } + return build_binding_key(&cluster_tt_local_bindings[idx], + (uint16)cluster_tt_local_bindings[idx].segment_id, out); +} + +/* Install and emit one already-minted exact key. */ +static void +install_key(const ClusterTTStatusKey *key, ClusterTTStatus status, SCN commit_scn) +{ + bool installed = cluster_tt_status_install_local(key, status, commit_scn); + + if (!installed) + return; + +#ifdef USE_ASSERT_CHECKING + { + ClusterTTStatusResult res; + bool hit = false; + bool epoch_stable = ((uint32)cluster_epoch_get_current() == key->cluster_epoch); + + if (epoch_stable) { + hit = cluster_tt_status_lookup_exact(key, &res); + epoch_stable = ((uint32)cluster_epoch_get_current() == key->cluster_epoch); + } + if (epoch_stable) + Assert(hit && res.authoritative && res.status == status); + if (hit && res.authoritative && res.status == status) + cluster_tt_status_bump_self_consumer_hit(); + } +#endif + + cluster_tt_status_hint_emit(key, status, commit_scn); +} + +static void +install_binding_aliases(const ClusterTTLocalBinding *binding, ClusterTTStatus status, + SCN commit_scn) +{ + uint16 i; + + for (i = 0; i < binding->active_alias_count; i++) { + ClusterTTStatusKey key; + + if (build_binding_key(binding, binding->active_alias_segments[i], &key)) + install_key(&key, status, commit_scn); + } +} + /* * install_status -- common path for commit / abort install + N7 * self-consumer lookup. @@ -515,7 +560,8 @@ static void install_status(TransactionId xid, ClusterTTStatus status, SCN commit_scn) { ClusterTTStatusKey key; - bool installed; + const ClusterTTLocalBinding *binding; + int idx; if (!cluster_enabled || cluster_node_id < 0) return; @@ -528,8 +574,10 @@ install_status(TransactionId xid, ClusterTTStatus status, SCN commit_scn) * (DDL-only or read-only); no overlay entry is needed and the * provisional fallback path is forbidden in production. */ - if (!build_local_key(xid, &key)) + idx = cluster_tt_local_find_binding(xid); + if (idx < 0 || !build_local_key(xid, &key)) return; + binding = &cluster_tt_local_bindings[idx]; /* * spec-3.3 D6 (L181 chain step 2): install in-memory overlay entry @@ -537,56 +585,8 @@ install_status(TransactionId xid, ClusterTTStatus status, SCN commit_scn) * carries commit_scn to (a) self-consumer N7 lookup and (b) the wire * emit path which will ship it to peers in V2 hints (D8). */ - installed = cluster_tt_status_install_local(&key, status, commit_scn); - if (!installed) - return; - - /* - * spec-3.1 v0.4 N7 self-consumer: re-lookup to prove the just-installed key - * is reachable + bump self_consumer_hit_count (D9 T8 + D10 L2). - * - * P0 perf hardening: this is now a DEBUG-ONLY invariant check. In every - * commit it cost an extra HTAB lookup + LW_SHARED on the hot path; the - * production install path does not need to re-read its own write. Assert - * builds still fail fast if the install cannot read its own key, and the - * self_consumer_hit_count counter is now a debug-build signal. - */ -#ifdef USE_ASSERT_CHECKING - { - ClusterTTStatusResult res; - bool hit = false; - bool epoch_stable = ((uint32)cluster_epoch_get_current() == key.cluster_epoch); - - /* - * lookup_exact() is the production reader API, so it intentionally fences - * against the current cluster epoch. A transaction can abort while a - * clean-leave/reconfig advances the epoch; the local old-epoch status - * install is still valid evidence for that transaction, but public lookup - * must reject it. Keep this debug wiring check only when the epoch stayed - * stable across the install/lookup window. - */ - if (epoch_stable) { - hit = cluster_tt_status_lookup_exact(&key, &res); - epoch_stable = ((uint32)cluster_epoch_get_current() == key.cluster_epoch); - } - - if (epoch_stable) - Assert(hit && res.authoritative && res.status == status); - if (hit && res.authoritative && res.status == status) - cluster_tt_status_bump_self_consumer_hit(); - } -#endif - - /* - * spec-3.2 D4 + spec-3.3 D6/D8 (L181 chain step 4): cross-node TT - * status hint wire emit. Uses the EXACT key just minted + installed - * (HC184 — no raw-xid rebuild); commit_scn flows through so peers - * see the real value via V2 wire. Fire-and-forget; emit_mode = - * disabled is a no-op inside the function. commit/abort 对称 - * (install_status is called from both record_commit + record_abort; - * abort path passes InvalidScn). - */ - cluster_tt_status_hint_emit(&key, status, commit_scn); + install_key(&key, status, commit_scn); + install_binding_aliases(binding, status, commit_scn); } /* ------------------------------------------------------------ */ @@ -700,6 +700,65 @@ cluster_tt_local_record_active(TransactionId xid) * running and may stamp additional lock-only or data ITL slots. */ } +void +cluster_tt_local_record_data_active(TransactionId xid, UBA uba) +{ + ClusterTTLocalBinding *binding; + uint32 record_segment; + uint32 block_no; + uint16 slot_offset; + uint16 row_offset; + int idx; + + if (!cluster_enabled || cluster_node_id < 0 || !TransactionIdIsNormal(xid)) + return; + idx = cluster_tt_local_find_binding(xid); + if (idx < 0) + return; + binding = &cluster_tt_local_bindings[idx]; + + /* The writer made this UBA from the same binding before entering CRIT. + * If that invariant is ever broken, publish no guessed alias. */ + if (!uba_decode(uba, &record_segment, &block_no, &slot_offset, &row_offset) + || uba_origin_node_id(uba) != (NodeId)cluster_node_id + || slot_offset != binding->slot_offset) { + Assert(false); + return; + } + (void)block_no; + (void)row_offset; + + if (record_segment != binding->segment_id) { + uint16 i; + + for (i = 0; i < binding->active_alias_count; i++) + if (binding->active_alias_segments[i] == (uint16)record_segment) + break; + + if (i == binding->active_alias_count) { + MemoryContext oldcxt; + + oldcxt = MemoryContextSwitchTo(TopTransactionContext); + if (binding->active_alias_segments == NULL) { + binding->active_alias_capacity = 4; + binding->active_alias_segments + = (uint16 *)palloc(sizeof(uint16) * binding->active_alias_capacity); + } else if (binding->active_alias_count == binding->active_alias_capacity) { + binding->active_alias_capacity *= 2; + binding->active_alias_segments + = (uint16 *)repalloc(binding->active_alias_segments, + sizeof(uint16) * binding->active_alias_capacity); + } + MemoryContextSwitchTo(oldcxt); + binding->active_alias_segments[binding->active_alias_count++] = (uint16)record_segment; + } + } + + /* install_status publishes canonical + every registered alias. Commit and + * abort use the same path, closing each alias terminally. */ + install_status(xid, CLUSTER_TT_STATUS_IN_PROGRESS, InvalidScn); +} + uint32 cluster_tt_local_slot_seq_peek(void) { @@ -743,6 +802,13 @@ cluster_tt_local_record_active(TransactionId xid) (void)xid; } +void +cluster_tt_local_record_data_active(TransactionId xid, UBA uba) +{ + (void)xid; + (void)uba; +} + uint32 cluster_tt_local_slot_seq_peek(void) { diff --git a/src/backend/cluster/cluster_visibility_verdict.c b/src/backend/cluster/cluster_visibility_verdict.c index ee3d6119d9..e6617390d1 100644 --- a/src/backend/cluster/cluster_visibility_verdict.c +++ b/src/backend/cluster/cluster_visibility_verdict.c @@ -30,9 +30,34 @@ #ifdef USE_PGRAC_CLUSTER +#include "access/htup_details.h" + #include "cluster/cluster_tt_status.h" #include "cluster/cluster_visibility_resolve.h" +/* + * P0-27: VACUUM freeze is an authoritative xmin-committed proof. Only the + * exact FROZEN bit pair bypasses old xmin identity resolution; a lone native + * COMMITTED or INVALID hint still follows cluster evidence and fails closed + * when that evidence is stale or ambiguous. + */ +bool +cluster_vis_xmin_needs_resolution(uint16 infomask) +{ + return (infomask & HEAP_XMIN_FROZEN) != HEAP_XMIN_FROZEN; +} + +/* + * P0-28: local GlobalVis/CLOG state cannot prove that no peer still names a + * shared-storage TID. Reclamation is legal only after a cluster-wide + * non-removable horizon is available. + */ +bool +cluster_vis_prune_must_defer(bool storage_mode, bool cluster_horizon_available) +{ + return storage_mode && !cluster_horizon_available; +} + /* ============================================================ * spec-3.14 §2.2 OBS truth tables (pure verdict functions). * diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index b06e340ac6..9c87ed2604 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -107,8 +107,7 @@ extern bool ignore_checksum_failure; #define BufferGetLSN(bufHdr) (PageGetLSN(BufHdrGetBlock(bufHdr))) /* Note: this macro only works on local buffers, not shared ones! */ -#define LocalBufHdrGetBlock(bufHdr) \ - LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)] +#define LocalBufHdrGetBlock(bufHdr) LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)] /* Bits in SyncOneBuffer's return value */ #define BUF_WRITTEN 0x01 @@ -116,32 +115,14 @@ extern bool ignore_checksum_failure; #define RELS_BSEARCH_THRESHOLD 20 -#ifdef USE_PGRAC_CLUSTER -/* - * spec-6.14 D4: single PCM-tracking criterion by relfilenumber. Under - * cluster.shared_catalog the catalog lives in the single shared tree, so its - * pages need the same N/S/X + CR + PI lost-write coherency as user pages - * (routing a catalog page into the shared tree without PCM tracking would be a - * double-write hazard -- pairs with the D3 smgr flip, INV-14-1). All shared - * buffers under shared_catalog=on hold permanent (non-temp) relations: temp - * relations use local buffers and never reach these paths, and unlogged - * permanent relations are rejected at DDL time (Q12). Off mode keeps the - * historic FirstNormalObjectId user-only boundary. - * - * The eviction/release hooks (HC112) MUST use this SAME criterion as the - * acquire path, otherwise a tracked catalog page's PCM lock would leak on - * eviction (spec-6.14 R6 -- do not let a gate drift). - */ -static inline bool -cluster_bufmgr_reln_pcm_tracked(RelFileNumber relnum) -{ - return cluster_shared_catalog || relnum >= (RelFileNumber) FirstNormalObjectId; -} +static void FlushBuffer(BufferDesc *buf, SMgrRelation reln, + IOObject io_object, IOContext io_context); +#ifdef USE_PGRAC_CLUSTER static inline bool cluster_bufmgr_should_pcm_track(BufferDesc *buf) { - return cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(&buf->tag)); + return cluster_pcm_x_buffer_tag_tracked(&buf->tag, cluster_shared_catalog); } /* Defined with the GCS copy/drop substrate below. The queue ownership @@ -150,6 +131,15 @@ cluster_bufmgr_should_pcm_track(BufferDesc *buf) static XLogRecPtr cluster_gcs_clamp_ship_flush_lsn(XLogRecPtr page_lsn); static void cluster_bufmgr_pin_for_gcs_locked(BufferDesc *buf, uint32 buf_state); static void cluster_bufmgr_unpin_for_gcs(BufferDesc *buf); +static ClusterPcmOwnResult cluster_bufmgr_pcm_own_finish_x_to_s_downgrade( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, + ClusterPcmOwnSnapshot *out_shared); + +/* Process-local dynamic scope for the exact retain-finish FlushBuffer call. */ +static bool cluster_pcm_x_finish_retain_flush_active = false; +static bool cluster_pcm_x_finish_retain_flush_io_active = false; +static bool cluster_pcm_x_finish_retain_flush_error_context_pushed = false; +static ErrorContextCallback *cluster_pcm_x_finish_retain_flush_error_context_previous = NULL; static ClusterPcmOwnResult cluster_pcm_own_bump_failure(BufferDesc *buf, uint64 generation, uint32 *out_flags) @@ -225,6 +215,14 @@ cluster_pcm_own_transition(BufferDesc *buf, uint8 new_pcm_state, uint32 set_flag uint32 buf_state; uint32 observed_flags = 0; uint64 new_generation; + /* Evidence for an S/X mirror transition over invalid page bytes: sampled + * under the header lock, logged after. The reservation finish itself + * fails closed on this shape; this covers the direct-transition mirror. */ + bool fx_log = false; + BufferTag fx_tag = {0}; + uint8 fx_old_state = 0; + uint64 fx_token = 0; + uint32 fx_refcount = 0; buf_state = LockBufHdr(buf); bump_result = cluster_pcm_own_bump_locked(buf, set_flags, clear_flags, @@ -234,8 +232,24 @@ cluster_pcm_own_transition(BufferDesc *buf, uint8 new_pcm_state, uint32 set_flag cluster_pcm_own_report_bump_failure(buf, bump_result, new_generation, observed_flags, "ownership transition"); } + if ((new_pcm_state == (uint8) PCM_STATE_S || new_pcm_state == (uint8) PCM_STATE_X) + && (buf_state & BM_VALID) == 0) + { + fx_log = true; + fx_tag = buf->tag; + fx_old_state = buf->pcm_state; + fx_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + fx_refcount = BUF_STATE_GET_REFCOUNT(buf_state); + } buf->pcm_state = new_pcm_state; UnlockBufHdr(buf, buf_state); + if (fx_log) + elog(LOG, + "cluster PCM own S/X over !BM_VALID: site=own-transition buffer=%d rel=%u fork=%d " + "blk=%u target=%u refcount=%u old_state=%u gen=%llu token=%llu", + buf->buf_id, fx_tag.relNumber, (int)fx_tag.forkNum, fx_tag.blockNum, + (unsigned)new_pcm_state, fx_refcount, (unsigned)fx_old_state, + (unsigned long long)new_generation, (unsigned long long)fx_token); } /* @@ -272,10 +286,34 @@ cluster_pcm_own_snapshot_locked(BufferDesc *buf, ClusterPcmOwnSnapshot *out) out->tag = buf->tag; out->generation = cluster_pcm_own_gen_get(buf->buf_id); out->reservation_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + out->writer_activation_token + = cluster_pcm_own_writer_activation_token_get(buf->buf_id); out->flags = cluster_pcm_own_flags_get(buf->buf_id); out->pcm_state = buf->pcm_state; } +/* Diagnostic only: callers capture under the BufferDesc header lock and emit + * after releasing it. Keeping one format for commit, exact clear, and the + * unguarded legacy N boundaries makes a stale activation fence traceable to + * its last exact tag/descriptor tuple without changing ownership semantics. */ +static void +cluster_pcm_own_activation_diag_emit(const char *site, int buffer_id, + const ClusterPcmOwnSnapshot *snapshot, + ClusterPcmOwnResult result) +{ + if (site == NULL || snapshot == NULL) + return; + elog(LOG, + "cluster PCM writer activation diagnostic: site=%s buffer=%d rel=%u fork=%d blk=%u " + "generation=%llu reservation_token=%llu writer_activation_token=%llu flags=0x%x " + "pcm_state=%u result=%d", + site, buffer_id, snapshot->tag.relNumber, (int)snapshot->tag.forkNum, + snapshot->tag.blockNum, (unsigned long long)snapshot->generation, + (unsigned long long)snapshot->reservation_token, + (unsigned long long)snapshot->writer_activation_token, snapshot->flags, + (unsigned int)snapshot->pcm_state, (int)result); +} + /* A normal D-h1 PI is !BM_VALID. PCM-X retained-image transfer deliberately * uses the otherwise-unoccupied PI+BM_VALID shape while N+REVOKING keeps the * exact descriptor pinned to the image record. Write/reuse gates recognize @@ -493,7 +531,7 @@ cluster_bufmgr_pcm_own_finish_s_release_to_n( * ordinary invalidate path nevertheless cannot unmap a pinned descriptor, * which makes every hot-block writer wait on its own pin. Bind tag->buffer * under the mapping lock, add one raw pin, then serialize all byte users with - * content EXCLUSIVE. Two exact tuple checks around WAL flush close retag, + * content EXCLUSIVE. Two exact tuple checks around the data flush close retag, * generation, reservation, and page-LSN ABA windows. Success keeps the clean * bytes BM_VALID as an N mirror; every later LockBuffer must reacquire PCM * before consuming them. VM/FSM never enter this shape because their @@ -520,7 +558,7 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, uint32 flags; uint64 token; int buf_id; - bool dirty = false; + bool needs_flush = false; volatile bool content_locked = false; if (out_page_lsn != NULL) @@ -581,27 +619,34 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, return result; content_lock = BufferDescriptorGetContentLock(buf); + if (!LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) { + cluster_bufmgr_unpin_for_gcs(buf); + return CLUSTER_PCM_OWN_BUSY; + } + content_locked = true; PG_TRY(); { - LWLockAcquire(content_lock, LW_EXCLUSIVE); - content_locked = true; buf_state = LockBufHdr(buf); if (!cluster_pcm_own_snapshot_matches_locked(buf, &expected_s) || (buf_state & BM_VALID) == 0 - || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state) - || (buf_state & BM_IO_IN_PROGRESS) != 0) + || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) result = CLUSTER_PCM_OWN_STALE; + else if ((buf_state & BM_IO_ERROR) != 0) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + result = CLUSTER_PCM_OWN_BUSY; else { page = (Page) BufHdrGetBlock(buf); page_lsn = PageGetLSN(page); page_scn = (uint64) ((PageHeader) page)->pd_block_scn; - dirty = (buf_state & BM_DIRTY) != 0; + needs_flush = (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | + BM_CHECKPOINT_NEEDED)) != 0; } UnlockBufHdr(buf, buf_state); - if (result == CLUSTER_PCM_OWN_OK && dirty && !XLogRecPtrIsInvalid(page_lsn)) - XLogFlush(cluster_gcs_clamp_ship_flush_lsn(page_lsn)); + if (result == CLUSTER_PCM_OWN_OK && needs_flush) + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); if (result == CLUSTER_PCM_OWN_OK) { @@ -610,9 +655,13 @@ cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, if (!cluster_pcm_own_snapshot_matches_locked(buf, &expected_s) || (buf_state & BM_VALID) == 0 || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state) - || (buf_state & BM_IO_IN_PROGRESS) != 0 || PageGetLSN(page) != page_lsn) result = CLUSTER_PCM_OWN_STALE; + else if ((buf_state & BM_IO_ERROR) != 0) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0 + || (buf_state & BM_IO_IN_PROGRESS) != 0) + result = CLUSTER_PCM_OWN_BUSY; else { result = cluster_pcm_own_bump_locked(buf, 0, 0, NULL, NULL); @@ -688,6 +737,30 @@ cluster_bufmgr_pcm_own_publish_installed_x_image( return result; } +/* Republish a kept-pinned retained PI mirror as the CURRENT image. The + * caller stands at a byte-currency proof point of the still-open legacy + * grant (a shipped-image install under content EXCLUSIVE, a storage refresh, + * or an SCN PASS proof); only the exact republish shape is flipped, under + * header authority, and every other descriptor state is left untouched so + * the grant-finish valid-image gate keeps failing closed on unproven + * bytes. */ +void +cluster_bufmgr_pcm_own_republish_grant_pending_image(BufferDesc *buf) +{ + uint32 buf_state; + + if (buf == NULL || ClusterPcmOwnArray == NULL) + return; + buf_state = LockBufHdr(buf); + if (cluster_pcm_x_grant_pending_republish_shape(buf->pcm_state, + cluster_pcm_own_flags_get(buf->buf_id), + cluster_pcm_own_reservation_token_get(buf->buf_id), + (buf_state & BM_VALID) != 0, + buf->buffer_type)) + buf->buffer_type = (uint8) BUF_TYPE_CURRENT; + UnlockBufHdr(buf, buf_state); +} + static ClusterPcmOwnResult cluster_pcm_own_begin_grant_reservation(BufferDesc *buf, ClusterPcmOwnSnapshot *out_base, uint64 *out_token) @@ -707,6 +780,10 @@ cluster_pcm_own_begin_grant_reservation(BufferDesc *buf, ClusterPcmOwnSnapshot * result = cluster_pcm_own_reservation_begin_exact(buf->buf_id, out_base->generation, PCM_OWN_FLAG_GRANT_PENDING, out_token); UnlockBufHdr(buf, buf_state); + /* Exact begin allocated a fresh token under header authority; a replayed + * or refused begin reports BUSY/STALE and never reaches here. */ + if (result == CLUSTER_PCM_OWN_OK) + cluster_pcm_x_stats_note_own_begin(); return result; } @@ -732,6 +809,8 @@ cluster_bufmgr_pcm_own_begin_x_reservation(BufferDesc *buf, const ClusterPcmOwnS result = cluster_pcm_own_reservation_begin_exact(buf->buf_id, expected->generation, PCM_OWN_FLAG_GRANT_PENDING, out_token); UnlockBufHdr(buf, buf_state); + if (result == CLUSTER_PCM_OWN_OK) + cluster_pcm_x_stats_note_own_begin(); return result; } @@ -754,6 +833,7 @@ cluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation( bool source_is_n; bool source_is_s; bool source_is_x; + bool handoff_transitioned = false; if (out_token != NULL) *out_token = 0; @@ -801,11 +881,21 @@ cluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation( else if (flags != PCM_OWN_FLAG_REVOKING) result = CLUSTER_PCM_OWN_BUSY; else + { result = cluster_pcm_own_revoke_to_grant_handoff_exact( buf->buf_id, expected_revoking->generation, expected_revoking->reservation_token); + handoff_transitioned = result == CLUSTER_PCM_OWN_OK; + } UnlockBufHdr(buf, buf_state); if (result == CLUSTER_PCM_OWN_OK) + { + /* Count only the REVOKING->GRANT_PENDING linearization; the exact + * duplicate PREPARE that already observes GRANT_PENDING is an + * idempotent replay and must not advance the begin counter. */ + if (handoff_transitioned) + cluster_pcm_x_stats_note_own_begin(); *out_token = expected_revoking->reservation_token; + } return result; } @@ -819,15 +909,26 @@ cluster_bufmgr_pcm_own_handoff_s_revoke_to_x_reservation( out_token); } +/* + * PGRAC: live_out, when non-NULL, receives the descriptor tuple sampled + * under the header lock at classification time, so a failing finish can + * report exactly which term diverged from the reservation base. The + * post-rollback descriptor is useless for that: the rollback itself + * mutates the flags and possibly the generation. + */ static ClusterPcmOwnResult cluster_pcm_own_finish_grant_reservation(BufferDesc *buf, const ClusterPcmOwnSnapshot *base, uint64 reservation_token, uint8 new_pcm_state, - uint64 *out_committed_generation) + bool arm_writer_activation, + uint64 *out_committed_generation, + ClusterPcmOwnSnapshot *live_out) { + ClusterPcmOwnSnapshot activation_diag; ClusterPcmOwnSnapshot live; ClusterPcmXGrantReservationKind kind; ClusterPcmOwnResult result; uint32 buf_state; + bool activation_diag_valid = false; if (buf == NULL || base == NULL || out_committed_generation == NULL) return CLUSTER_PCM_OWN_INVALID; @@ -839,6 +940,8 @@ cluster_pcm_own_finish_grant_reservation(BufferDesc *buf, const ClusterPcmOwnSna buf_state = LockBufHdr(buf); cluster_pcm_own_snapshot_locked(buf, &live); + if (live_out != NULL) + *live_out = live; kind = cluster_pcm_x_grant_reservation_kind(&live, base, reservation_token); if (kind == CLUSTER_PCM_X_GRANT_RESERVATION_INVALID) { ClusterPcmOwnResult live_shape @@ -849,16 +952,45 @@ cluster_pcm_own_finish_grant_reservation(BufferDesc *buf, const ClusterPcmOwnSna && (new_pcm_state != (uint8)PCM_STATE_X || !LWLockHeldByMe(BufferDescriptorGetContentLock(buf)))) result = CLUSTER_PCM_OWN_INVALID; + else if (buf->buffer_type == (uint8) BUF_TYPE_PI + || (buf_state & (BM_VALID | BM_IO_IN_PROGRESS)) == 0) + /* Never commit a durable S/X mirror over a page image that is not + * current: a frozen PI mirror or a dropped image (!BM_VALID) carries + * no bytes this grant may serve, so a commit here would silently + * cover stale or absent data (Rule 8.A). A grant that installs the + * shipped image sets BM_VALID + CURRENT before this finish; the + * !BM_VALID revoke-handoff precedent likewise fails closed. The one + * legal !BM_VALID commit shape is the direct-init window (EXTEND / + * READ_MISS): its gate runs between StartBufferIO and + * TerminateBufferIO(BM_VALID), so the descriptor still owns + * BM_IO_IN_PROGRESS and the zeroed page is published only after the + * caller initializes it under content EXCLUSIVE. */ + result = CLUSTER_PCM_OWN_CORRUPT; else { - result = cluster_pcm_own_grant_commit_exact(buf->buf_id, base->generation, - reservation_token, out_committed_generation); + if (new_pcm_state == (uint8)PCM_STATE_X && arm_writer_activation) + result = cluster_pcm_own_writer_grant_commit_exact( + buf->buf_id, base->generation, reservation_token, out_committed_generation); + else + result = cluster_pcm_own_grant_commit_exact(buf->buf_id, base->generation, + reservation_token, out_committed_generation); if (result == CLUSTER_PCM_OWN_OK) { buf->buffer_type = new_pcm_state == (uint8)PCM_STATE_S ? (uint8)BUF_TYPE_SCUR : (uint8)BUF_TYPE_XCUR; buf->pcm_state = new_pcm_state; + if (new_pcm_state == (uint8)PCM_STATE_X && arm_writer_activation) { + cluster_pcm_own_snapshot_locked(buf, &activation_diag); + activation_diag_valid = true; + } } } UnlockBufHdr(buf, buf_state); + if (activation_diag_valid) + cluster_pcm_own_activation_diag_emit("writer-commit", buf->buf_id, + &activation_diag, result); + /* Every X ownership commit funnels through this exact commit; the S-grant + * finish and every STALE/INVALID/CORRUPT refusal stay uncounted. */ + if (result == CLUSTER_PCM_OWN_OK && new_pcm_state == (uint8)PCM_STATE_X) + cluster_pcm_x_stats_note_own_commit(); return result; } @@ -871,7 +1003,9 @@ cluster_bufmgr_pcm_own_finish_x_commit(BufferDesc *buf, const ClusterPcmOwnSnaps && expected->pcm_state != (uint8)PCM_STATE_X)) return CLUSTER_PCM_OWN_STALE; return cluster_pcm_own_finish_grant_reservation(buf, expected, reservation_token, - (uint8)PCM_STATE_X, out_committed_generation); + (uint8)PCM_STATE_X, true, + out_committed_generation, + NULL); } static ClusterPcmOwnResult @@ -1050,11 +1184,13 @@ cluster_pcm_own_finish_grant_or_rollback(BufferDesc *buf, const ClusterPcmOwnSna PcmLockMode acquired_mode, uint64 *out_committed_generation) { + ClusterPcmOwnSnapshot live; ClusterPcmOwnResult finish_result; ClusterPcmOwnResult rollback_result; + memset(&live, 0, sizeof(live)); finish_result = cluster_pcm_own_finish_grant_reservation( - buf, base, reservation_token, new_pcm_state, out_committed_generation); + buf, base, reservation_token, new_pcm_state, false, out_committed_generation, &live); if (finish_result == CLUSTER_PCM_OWN_OK) return; @@ -1095,8 +1231,17 @@ cluster_pcm_own_finish_grant_or_rollback(BufferDesc *buf, const ClusterPcmOwnSna (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not finish exact cluster PCM grant reservation: result=%d", (int)finish_result), - errdetail("master grant was rolled back for buffer %d token=%llu", - buf != NULL ? buf->buf_id : -1, (unsigned long long)reservation_token))); + errdetail("master grant was rolled back for buffer %d token=%llu target=%u; " + "live state=%u gen=%llu token=%llu flags=%u vs " + "base state=%u gen=%llu token=%llu flags=%u", + buf != NULL ? buf->buf_id : -1, (unsigned long long)reservation_token, + (unsigned int)new_pcm_state, + (unsigned int)live.pcm_state, (unsigned long long)live.generation, + (unsigned long long)live.reservation_token, (unsigned int)live.flags, + base != NULL ? (unsigned int)base->pcm_state : 0, + (unsigned long long)(base != NULL ? base->generation : 0), + (unsigned long long)(base != NULL ? base->reservation_token : 0), + base != NULL ? (unsigned int)base->flags : 0))); } ClusterPcmOwnResult @@ -1187,6 +1332,9 @@ typedef struct ClusterPcmXWriterLedgerEntry { PcmXLocalWriterClaim claim; ClusterPcmOwnSnapshot granted; bool claim_handed_off; + bool claim_cleanup_complete; + bool grant_snapshot_exact; + bool activation_fence_armed; } ClusterPcmXWriterLedgerEntry; static ClusterPcmXWriterLedgerEntry cluster_bufmgr_pcm_x_writer_ledger[LWLOCK_MAX_HELD_BY_PROC]; @@ -1209,7 +1357,7 @@ static void cluster_bufmgr_pcm_direct_init_snapshot_locked(BufferDesc *buf, uint static void cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, - uint32 wait_index) + uint32 wait_index, bool *barrier_refused) { long delay_ms; PcmXQueueResult guard_result; @@ -1230,8 +1378,23 @@ cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, */ guard_result = cluster_pcm_x_nested_wait_guard_before_block(); if (guard_result != PCM_X_QUEUE_OK) + { + /* + * PGRAC (t/400 L3 item 3, holder lane): a barrier-aware caller owns + * the unwind for the frozen-barrier refusal — it releases its own + * outer lock and resolves the conversion unlocked. Every other + * guard verdict, and every non-aware caller, keeps the historical + * fail-closed report. + */ + if (barrier_refused != NULL && guard_result == PCM_X_QUEUE_BARRIER_CLOSED) + { + *barrier_refused = true; + cluster_pcm_x_stats_note_barrier_unwind(); + return; + } cluster_bufmgr_pcm_x_holder_report_failure( guard_result, GetBufferDescriptor(buffer_id), "retry wait nested guard"); + } delay_ms = cluster_pcm_x_holder_retry_delay_ms(wait_index); CHECK_FOR_INTERRUPTS(); (void) WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, delay_ms, @@ -1239,6 +1402,179 @@ cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, CHECK_FOR_INTERRUPTS(); } +/* + * PGRAC (t/400 L3 review R2 P0-2): a legacy grant reservation that observes + * a live GRANT_PENDING/REVOKING lifecycle sits in a normal ms-scale revoke or + * grant window — for a reader (or a rare un-valid-page writer) that is a + * transient to wait out off-lock, not a client ERROR. The retry is only a + * fresh begin against the re-sampled complete ownership tuple; another + * lifecycle's token/flags are never touched. No content lock is held at + * either call site, and a backend holding OTHER frozen-tag content locks + * must not sleep (nested-guard discipline). There is deliberately no retry + * budget: ordinary ownership competition is cancelable waiting, never a + * client-visible exhaustion error. + */ +static ClusterPcmOwnResult +cluster_bufmgr_pcm_begin_grant_reservation_wait(BufferDesc *buf, + ClusterPcmOwnSnapshot *base_out, + uint64 *token_out) +{ + ClusterPcmOwnResult result; + uint32 waits = 0; + + for (;;) + { + result = cluster_pcm_own_begin_grant_reservation(buf, base_out, token_out); + if (result != CLUSTER_PCM_OWN_BUSY) + return result; + if (cluster_pcm_x_nested_wait_guard_before_block() != PCM_X_QUEUE_OK) + return result; + cluster_pcm_x_stats_note_own_busy(); + CHECK_FOR_INTERRUPTS(); + (void) WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + cluster_pcm_x_holder_retry_delay_ms(waits), + WAIT_EVENT_PCM_BLOCK_CONVERT_WAIT); + CHECK_FOR_INTERRUPTS(); + if (waits < PG_UINT32_MAX) + waits++; + } +} + +/* + * PGRAC (t/400 S_NEW forensics): a legacy grant reservation that opens from + * a non-N base is the protocol-violating shape behind the deterministic + * S_NEW finish refusals — an X convert must ride the convert queue, and an + * S refresh over a held S must be served by the local cover gate. Log the + * complete begin-time context (buffer identity, validity bit, the exact + * ownership tuple the reservation snapshotted, and the acquire mode) so the + * producing entry path can be identified from a live run. + */ +static void +cluster_bufmgr_pcm_legacy_begin_probe(BufferDesc *buf, PcmLockMode pcm_mode, + const ClusterPcmOwnSnapshot *base, + const char *site) +{ + uint32 buf_state; + + if (base->pcm_state == (uint8) PCM_STATE_N) + return; + buf_state = pg_atomic_read_u32(&buf->state); + elog(LOG, + "cluster PCM legacy reservation opened from a non-N base: site=%s mode=%d buffer=%d " + "rel=%u fork=%d blk=%u valid=%d base_state=%u base_gen=%llu base_token=%llu " + "base_flags=0x%x pcm_state_now=%u", + site, (int)pcm_mode, buf->buf_id, buf->tag.relNumber, (int)buf->tag.forkNum, + buf->tag.blockNum, (buf_state & BM_VALID) != 0 ? 1 : 0, base->pcm_state, + (unsigned long long)base->generation, (unsigned long long)base->reservation_token, + base->flags, buf->pcm_state); +} + +typedef enum ClusterBufmgrPcmRetryRearmResult +{ + CLUSTER_BUFMGR_PCM_RETRY_REARMED = 0, + CLUSTER_BUFMGR_PCM_RETRY_COVERED, + CLUSTER_BUFMGR_PCM_RETRY_BARRIER_REFUSED +} ClusterBufmgrPcmRetryRearmResult; + +/* A queue INVALIDATE that meets mirror-N + GRANT_PENDING returns BUSY. Its + * master retries no sooner than max(starvation backoff, LMON tick), while the + * DENIED_PENDING_X requester owns this function's wait. Keep the exact old + * reservation absent for two such intervals before publishing the successor: + * one interval lets the master stage its retry and the second lets the DATA + * worker consume it. Later denials widen that receive window, capped at the + * configured GUC maximum (60s base, 120s quiesce) rather than overflowing. + */ +static long +cluster_bufmgr_pcm_pending_x_retry_delay_ms(uint32 wait_index) +{ + uint64 retry_delay_ms; + uint32 shift = Min(wait_index, UINT32_C(16)); + + retry_delay_ms = (uint64)Max(Max(cluster_gcs_block_starvation_backoff_ms, + cluster_lmon_main_loop_interval), + 1); + if (retry_delay_ms >= UINT64_C(60000) + || retry_delay_ms > (UINT64_C(60000) >> shift)) + retry_delay_ms = UINT64_C(60000); + else + retry_delay_ms <<= shift; + return (long)(retry_delay_ms * 2); +} + +/* Consume one authoritative DENIED_PENDING_X. The exact old reservation is + * aborted before sleeping; STALE is an ABA-safe zero-mutation result and is + * handled by re-sampling the complete ownership tuple. A successful rearm + * always publishes a fresh token, while the subsequent GCS call allocates a + * fresh request_id. */ +static ClusterBufmgrPcmRetryRearmResult +cluster_bufmgr_pcm_retry_denied_rearm(BufferDesc *buf, PcmLockMode pcm_mode, + ClusterPcmOwnSnapshot *base, + uint64 *reservation_token, uint32 wait_index, + bool *barrier_refused, uint64 *covered_generation) +{ + ClusterPcmOwnResult own_result; + PcmXQueueResult guard_result; + uint8 current_state; + uint32 current_flags; + long backoff_ms; + + if (buf == NULL || base == NULL || reservation_token == NULL + || covered_generation == NULL) + { + cluster_pcm_own_report_bump_failure(buf, CLUSTER_PCM_OWN_INVALID, 0, 0, + "pending-X retry arguments"); + return CLUSTER_BUFMGR_PCM_RETRY_BARRIER_REFUSED; + } + own_result = cluster_pcm_own_abort_grant_reservation(buf, base, *reservation_token); + if (own_result != CLUSTER_PCM_OWN_OK && own_result != CLUSTER_PCM_OWN_STALE) + cluster_pcm_own_report_bump_failure(buf, own_result, base->generation, base->flags, + "pending-X exact abort"); + + /* A stale exact abort never touches the successor. If that successor is + * already a stable covering grant, rejoin the normal post-content-lock + * generation check instead of opening another reservation over it. */ + cluster_pcm_own_read(buf, ¤t_state, covered_generation, ¤t_flags); + elog(LOG, + "cluster PCM pending-X exact abort observation: buffer=%d rel=%u fork=%d blk=%u " + "mode=%d wait_index=%u base_state=%u base_generation=%llu token=%llu " + "abort_result=%d live_state=%u live_generation=%llu live_token=%llu live_flags=0x%x", + buf->buf_id, buf->tag.relNumber, (int)buf->tag.forkNum, buf->tag.blockNum, + (int)pcm_mode, wait_index, base->pcm_state, + (unsigned long long)base->generation, (unsigned long long)*reservation_token, + (int)own_result, current_state, (unsigned long long)*covered_generation, + (unsigned long long)cluster_pcm_own_reservation_token_get(buf->buf_id), current_flags); + if (current_flags == 0 && cluster_gcs_block_local_cache + && cluster_pcm_mode_covers((PcmLockMode) current_state, pcm_mode)) + return CLUSTER_BUFMGR_PCM_RETRY_COVERED; + + guard_result = cluster_pcm_x_nested_wait_guard_before_block(); + if (guard_result != PCM_X_QUEUE_OK) + { + if (barrier_refused != NULL && guard_result == PCM_X_QUEUE_BARRIER_CLOSED) + { + *barrier_refused = true; + cluster_pcm_x_stats_note_barrier_unwind(); + return CLUSTER_BUFMGR_PCM_RETRY_BARRIER_REFUSED; + } + cluster_bufmgr_pcm_x_holder_report_failure(guard_result, buf, + "pending-X retry nested guard"); + } + + backoff_ms = cluster_bufmgr_pcm_pending_x_retry_delay_ms(wait_index); + CHECK_FOR_INTERRUPTS(); + (void) WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, backoff_ms, + WAIT_EVENT_GCS_BLOCK_STARVATION_RETRY); + ResetLatch(MyLatch); + CHECK_FOR_INTERRUPTS(); + + own_result = cluster_bufmgr_pcm_begin_grant_reservation_wait(buf, base, reservation_token); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_own_report_bump_failure(buf, own_result, base->generation, base->flags, + "pending-X retry reservation"); + cluster_bufmgr_pcm_legacy_begin_probe(buf, pcm_mode, base, "pending-X retry reservation"); + return CLUSTER_BUFMGR_PCM_RETRY_REARMED; +} + static ClusterPcmXHolderLedgerEntry * cluster_bufmgr_pcm_x_holder_find(BufferDesc *buf) { @@ -1359,7 +1695,9 @@ cluster_bufmgr_pcm_x_holder_drain_deferred_nowait(void) } if (LWLockHeldByMe(entry->content_lock)) { - elog(LOG, "could not drain deferred cluster PCM-X holder while content lock is held: buffer=%d", + elog(LOG, + "could not drain deferred cluster PCM-X holder while content lock is held: " + "buffer=%d", entry->buffer_id); continue; } @@ -1411,31 +1749,27 @@ cluster_bufmgr_pcm_x_holder_drain_deferred(ClusterPcmXHolderLedgerEntry *entry) if (action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT) { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); - cluster_bufmgr_pcm_x_holder_report_failure(result, - GetBufferDescriptor(entry->buffer_id), - "deferred detach"); + cluster_bufmgr_pcm_x_holder_report_failure( + result, GetBufferDescriptor(entry->buffer_id), "deferred detach"); } - cluster_bufmgr_pcm_x_holder_retry_wait( - entry->content_lock, entry->buffer_id, - wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); + cluster_bufmgr_pcm_x_holder_retry_wait(entry->content_lock, entry->buffer_id, + wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS, + NULL); wait_index++; - if (wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS == 0) - { + if (wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS == 0) { runtime = cluster_pcm_x_runtime_snapshot(); - if (runtime.state != PCM_X_RUNTIME_ACTIVE - || runtime.master_session_incarnation == 0) - { + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) { cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); - cluster_bufmgr_pcm_x_holder_report_failure( - PCM_X_QUEUE_NOT_READY, GetBufferDescriptor(entry->buffer_id), - "deferred detach runtime"); + cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_NOT_READY, + GetBufferDescriptor(entry->buffer_id), + "deferred detach runtime"); } } } } static ClusterPcmXHolderLedgerEntry * -cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf) +cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf, bool *barrier_refused) { ClusterPcmXHolderLedgerEntry *entry; ClusterPcmXWriterLedgerEntry *writer_entry; @@ -1596,7 +1930,9 @@ cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf) cluster_bufmgr_pcm_x_holder_report_failure(result, buf, "register"); cluster_bufmgr_pcm_x_holder_retry_wait( content_lock, buf->buf_id, - wait_index++ % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); + wait_index++ % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS, barrier_refused); + if (barrier_refused != NULL && *barrier_refused) + return NULL; } return entry; @@ -1676,7 +2012,7 @@ cluster_bufmgr_pcm_x_holder_unregister(ClusterPcmXHolderLedgerEntry *entry) result, GetBufferDescriptor(entry->buffer_id), "unregister"); } cluster_bufmgr_pcm_x_holder_retry_wait(entry->content_lock, entry->buffer_id, - waits_used++); + waits_used++, NULL); } } @@ -1812,18 +2148,30 @@ static void cluster_bufmgr_pcm_x_writer_report_failure(PcmXQueueResult result, BufferDesc *buf, const char *operation) { + BufferTag guard_tag; + int guard_result = 0; + char guard_detail[96]; + + /* Name the exact requester escape arm and, for a nested-guard refusal, + * the held content-lock tag that closed the guard. */ + guard_detail[0] = '\0'; + if (cluster_pcm_x_nested_wait_guard_last_block(&guard_tag, &guard_result)) + snprintf(guard_detail, sizeof(guard_detail), " guard_result=%d guard_rel=%u guard_blk=%u", + guard_result, guard_tag.relNumber, guard_tag.blockNum); if (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY || result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BARRIER_CLOSED || result == PCM_X_QUEUE_NO_CAPACITY) ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), errmsg("cluster PCM-X writer operation is not ready"), - errdetail("operation=%s buffer=%d result=%d", operation, - buf != NULL ? buf->buf_id : -1, (int)result))); + errdetail("operation=%s buffer=%d result=%d fail_line=%d%s", operation, + buf != NULL ? buf->buf_id : -1, (int)result, + cluster_gcs_pcm_x_requester_last_fail_line(), guard_detail))); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("cluster PCM-X writer operation failed"), - errdetail("operation=%s buffer=%d result=%d", operation, - buf != NULL ? buf->buf_id : -1, (int)result))); + errdetail("operation=%s buffer=%d result=%d fail_line=%d%s", operation, + buf != NULL ? buf->buf_id : -1, (int)result, + cluster_gcs_pcm_x_requester_last_fail_line(), guard_detail))); } static void @@ -1833,6 +2181,72 @@ cluster_bufmgr_pcm_x_writer_clear(ClusterPcmXWriterLedgerEntry *entry) MemSet(entry, 0, sizeof(*entry)); } +/* Clear the shared grant->content activation fence only for the exact writer + * grant captured by this process-local ledger. Activate calls this while it + * owns content EXCLUSIVE; exceptional cleanup calls it only after the queue + * claim has reached its exact terminal. */ +static ClusterPcmOwnResult +cluster_bufmgr_pcm_x_writer_activation_clear(ClusterPcmXWriterLedgerEntry *entry, + BufferDesc *buf, + ClusterPcmOwnSnapshot *live_out) +{ + ClusterPcmOwnSnapshot live; + ClusterPcmOwnResult result; + uint32 buf_state; + + if (entry == NULL || buf == NULL || !entry->activation_fence_armed) + return CLUSTER_PCM_OWN_INVALID; + buf_state = LockBufHdr(buf); + cluster_pcm_own_snapshot_locked(buf, &live); + if (live_out != NULL) + *live_out = live; + if (!cluster_pcm_x_writer_grant_snapshot_exact(&entry->claim, &entry->granted, &live) + || cluster_pcm_own_writer_activation_token_get(buf->buf_id) + != entry->granted.reservation_token) + result = CLUSTER_PCM_OWN_STALE; + else + result = cluster_pcm_own_writer_activation_clear_exact( + buf->buf_id, entry->granted.generation, entry->granted.reservation_token); + UnlockBufHdr(buf, buf_state); + cluster_pcm_own_activation_diag_emit("writer-activation-clear", buf->buf_id, &live, result); + if (result == CLUSTER_PCM_OWN_OK) + entry->activation_fence_armed = false; + return result; +} + +static bool +cluster_bufmgr_pcm_x_writer_finish_claim_cleanup(ClusterPcmXWriterLedgerEntry *entry, + BufferDesc *buf, const char *context) +{ + ClusterPcmOwnResult own_result; + + if (entry == NULL || buf == NULL || !entry->claim_cleanup_complete) + return false; + if (!entry->grant_snapshot_exact) { + entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; + cluster_pcm_x_runtime_fail_closed(); + elog(LOG, "preserving cluster PCM-X writer with unproven grant snapshot: " + "context=%s buffer=%d", context, entry->buffer_id); + return false; + } + if (entry->activation_fence_armed) { + own_result = cluster_bufmgr_pcm_x_writer_activation_clear(entry, buf, NULL); + if (own_result != CLUSTER_PCM_OWN_OK) { + entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; + cluster_pcm_x_runtime_fail_closed(); + elog(LOG, + "could not clear exact cluster PCM-X writer activation fence: " + "context=%s buffer=%d generation=%llu token=%llu result=%d", + context, entry->buffer_id, + (unsigned long long)entry->granted.generation, + (unsigned long long)entry->granted.reservation_token, (int)own_result); + return false; + } + } + cluster_bufmgr_pcm_x_writer_clear(entry); + return true; +} + static void cluster_bufmgr_pcm_x_writer_release(ClusterPcmXWriterLedgerEntry *entry); static void @@ -1853,9 +2267,15 @@ cluster_bufmgr_pcm_x_writer_drain_deferred_nowait(void) entry->buffer_id); continue; } - result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); - if (result == PCM_X_QUEUE_OK) - cluster_bufmgr_pcm_x_writer_clear(entry); + if (entry->claim_cleanup_complete) + result = PCM_X_QUEUE_OK; + else + result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); + if (result == PCM_X_QUEUE_OK) { + entry->claim_cleanup_complete = true; + (void)cluster_bufmgr_pcm_x_writer_finish_claim_cleanup(entry, + GetBufferDescriptor(entry->buffer_id), "deferred drain"); + } else if (result != PCM_X_QUEUE_GATE_RETRY && result != PCM_X_QUEUE_BUSY) { cluster_pcm_x_runtime_fail_closed(); elog(LOG, "could not drain deferred exact cluster PCM-X writer: buffer=%d result=%d", @@ -1865,7 +2285,7 @@ cluster_bufmgr_pcm_x_writer_drain_deferred_nowait(void) } static ClusterPcmXWriterLedgerEntry * -cluster_bufmgr_pcm_x_writer_prepare(BufferDesc *buf, PcmLockMode mode) +cluster_bufmgr_pcm_x_writer_prepare(BufferDesc *buf, PcmLockMode mode, bool *barrier_refused) { ClusterPcmXWriterLedgerEntry *entry; ClusterPcmDirectInitSnapshot observed; @@ -1901,6 +2321,9 @@ cluster_bufmgr_pcm_x_writer_prepare(BufferDesc *buf, PcmLockMode mode) entry->content_lock = content_lock; entry->phase = PCM_X_WRITER_LEDGER_HANDOFF; entry->claim_handed_off = false; + entry->claim_cleanup_complete = false; + entry->grant_snapshot_exact = false; + entry->activation_fence_armed = false; buf_state = LockBufHdr(buf); cluster_bufmgr_pcm_direct_init_snapshot_locked(buf, buf_state, false, &observed); @@ -1920,6 +2343,21 @@ cluster_bufmgr_pcm_x_writer_prepare(BufferDesc *buf, PcmLockMode mode) "failed claim handoff"); } cluster_bufmgr_pcm_x_writer_clear(entry); + + /* + * PGRAC (t/400 L3 item 3): a nested-guard BARRIER_CLOSED means this + * backend already holds another content lock whose tag sits under a + * frozen revoke barrier; the requester unwound cleanly (no claim, no + * wait identity). A barrier-aware caller owns the unwind: hand the + * refusal back instead of escalating to a client ERROR. bufmgr must + * never release the foreign content lock itself. + */ + if (result == PCM_X_QUEUE_BARRIER_CLOSED && barrier_refused != NULL) + { + *barrier_refused = true; + cluster_pcm_x_stats_note_barrier_unwind(); + return NULL; + } cluster_bufmgr_pcm_x_writer_report_failure(result, buf, "queue acquire"); } if (!entry->claim_handed_off) { @@ -1930,9 +2368,37 @@ cluster_bufmgr_pcm_x_writer_prepare(BufferDesc *buf, PcmLockMode mode) own_result = cluster_bufmgr_pcm_own_snapshot(buf, &granted); if (own_result != CLUSTER_PCM_OWN_OK || !cluster_pcm_x_writer_grant_snapshot_exact(&entry->claim, &granted, &granted)) { + ereport(LOG, + (errmsg("cluster PCM-X writer grant snapshot mismatch"), + errdetail("buffer=%d own_result=%d claim_base=%llu claim_grant_base=%llu " + "claim_generation=%llu claim_flags=%u writer_flags=%u " + "claim_role=%u writer_role=%u claim_round=%u writer_round=%u " + "claim_slot=%zu/%llu writer_slot=%zu/%llu live_generation=%llu " + "live_token=%llu live_flags=%u live_state=%u tag_exact=%d", + buf->buf_id, (int)own_result, + (unsigned long long)entry->claim.writer.identity.base_own_generation, + (unsigned long long)entry->claim.grant_base_own_generation, + (unsigned long long)entry->claim.claim_generation, + (unsigned int)entry->claim.flags, + (unsigned int)entry->claim.writer.flags, + (unsigned int)entry->claim.role, + (unsigned int)entry->claim.writer.role, entry->claim.local_round, + entry->claim.writer.local_round, entry->claim.active_slot.slot_index, + (unsigned long long)entry->claim.active_slot.slot_generation, + entry->claim.writer.membership_slot.slot_index, + (unsigned long long)entry->claim.writer.membership_slot.slot_generation, + (unsigned long long)granted.generation, + (unsigned long long)granted.reservation_token, granted.flags, + (unsigned int)granted.pcm_state, + BufferTagsEqual(&granted.tag, &entry->claim.writer.identity.tag) ? 1 : 0))); release_result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); - if (release_result == PCM_X_QUEUE_OK) - cluster_bufmgr_pcm_x_writer_clear(entry); + if (release_result == PCM_X_QUEUE_OK) { + /* The queue claim is terminal, but the ownership grant did not + * match it. Preserve the ledger/fence evidence; guessing a clear + * here could release another grant's activation barrier. */ + entry->claim_cleanup_complete = true; + entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; + } else { /* Keep the exact claim reachable for owner-exit/deferred retry. */ entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; @@ -1945,6 +2411,8 @@ cluster_bufmgr_pcm_x_writer_prepare(BufferDesc *buf, PcmLockMode mode) } entry->granted = granted; + entry->grant_snapshot_exact = true; + entry->activation_fence_armed = true; entry->phase = PCM_X_WRITER_LEDGER_ACQUIRING; return entry; } @@ -1963,9 +2431,30 @@ cluster_bufmgr_pcm_x_writer_activate(ClusterPcmXWriterLedgerEntry *entry) || !cluster_bufmgr_pcm_x_writer_entry_exact(entry, buf) || entry->content_lock == NULL || !LWLockHeldByMe(entry->content_lock)) cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_BAD_STATE, buf, "activate phase"); - own_result = cluster_bufmgr_pcm_own_snapshot(buf, &live); + own_result = cluster_bufmgr_pcm_x_writer_activation_clear(entry, buf, &live); if (own_result != CLUSTER_PCM_OWN_OK || !cluster_pcm_x_writer_grant_snapshot_exact(&entry->claim, &entry->granted, &live)) { + ereport(LOG, + (errmsg("cluster PCM-X writer activate ownership mismatch"), + errdetail("buffer=%d own_result=%d claim_base=%llu claim_grant_base=%llu " + "claim_generation=%llu claim_round=%u claim_role=%u claim_slot=%zu/%llu " + "granted_generation=%llu granted_token=%llu granted_flags=%u " + "granted_state=%u live_generation=%llu live_token=%llu live_flags=%u " + "live_state=%u tag_exact=%d", + buf->buf_id, (int)own_result, + (unsigned long long)entry->claim.writer.identity.base_own_generation, + (unsigned long long)entry->claim.grant_base_own_generation, + (unsigned long long)entry->claim.claim_generation, + entry->claim.local_round, (unsigned int)entry->claim.role, + entry->claim.active_slot.slot_index, + (unsigned long long)entry->claim.active_slot.slot_generation, + (unsigned long long)entry->granted.generation, + (unsigned long long)entry->granted.reservation_token, + entry->granted.flags, (unsigned int)entry->granted.pcm_state, + (unsigned long long)live.generation, + (unsigned long long)live.reservation_token, live.flags, + (unsigned int)live.pcm_state, + BufferTagsEqual(&live.tag, &entry->granted.tag) ? 1 : 0))); cluster_pcm_x_runtime_fail_closed(); cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_CORRUPT, buf, "activate ownership"); } @@ -2029,7 +2518,8 @@ cluster_bufmgr_pcm_x_writer_release(ClusterPcmXWriterLedgerEntry *entry) } cluster_bufmgr_pcm_x_holder_retry_wait(entry->content_lock, entry->buffer_id, waits_used++ - % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); + % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS, + NULL); } } @@ -2043,8 +2533,11 @@ cluster_bufmgr_pcm_x_writer_abort_acquiring(ClusterPcmXWriterLedgerEntry *entry) if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)) return; result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); - if (result == PCM_X_QUEUE_OK) - cluster_bufmgr_pcm_x_writer_clear(entry); + if (result == PCM_X_QUEUE_OK) { + entry->claim_cleanup_complete = true; + (void)cluster_bufmgr_pcm_x_writer_finish_claim_cleanup( + entry, GetBufferDescriptor(entry->buffer_id), "abort acquiring"); + } else if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BUSY) entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; else { @@ -2073,9 +2566,15 @@ cluster_bufmgr_pcm_x_writer_exception_cleanup_all(void) } if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)) continue; - result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); - if (result == PCM_X_QUEUE_OK) - cluster_bufmgr_pcm_x_writer_clear(entry); + if (entry->claim_cleanup_complete) + result = PCM_X_QUEUE_OK; + else + result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); + if (result == PCM_X_QUEUE_OK) { + entry->claim_cleanup_complete = true; + (void)cluster_bufmgr_pcm_x_writer_finish_claim_cleanup( + entry, GetBufferDescriptor(entry->buffer_id), "exception cleanup"); + } else if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BUSY) entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; else { @@ -2131,10 +2630,16 @@ cluster_bufmgr_pcm_x_writer_owner_exit_drain_once(bool runtime_active) entry->buffer_id, (int)entry->phase); continue; } - result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); + if (entry->claim_cleanup_complete) + result = PCM_X_QUEUE_OK; + else + result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); action = cluster_pcm_x_owner_exit_action(result, false, runtime_active); - if (action == CLUSTER_PCM_X_OWNER_EXIT_COMPLETE) - cluster_bufmgr_pcm_x_writer_clear(entry); + if (action == CLUSTER_PCM_X_OWNER_EXIT_COMPLETE) { + entry->claim_cleanup_complete = true; + if (!cluster_bufmgr_pcm_x_writer_finish_claim_cleanup(entry, buf, "owner exit")) + retry = runtime_active; + } else if (action == CLUSTER_PCM_X_OWNER_EXIT_RETRY) { entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; retry = true; @@ -2331,7 +2836,9 @@ cluster_bufmgr_pcm_gate_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind ki ClusterPcmOwnSnapshot pending_base; ClusterPcmOwnResult pending_result; bool grant_acquired = false; + bool retry_denied = false; uint32 buf_state; + uint32 retry_wait_index = 0; uint64 pending_token = 0; uint64 committed_generation; @@ -2359,21 +2866,40 @@ cluster_bufmgr_pcm_gate_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind ki } UnlockBufHdr(buf, buf_state); - if (pending_result != CLUSTER_PCM_OWN_OK) + if (pending_result == CLUSTER_PCM_OWN_OK) + cluster_pcm_x_stats_note_own_begin(); + else cluster_bufmgr_pcm_direct_init_report_failure(buf, pending_result, &observed, "direct-init consume"); - PG_TRY(); - { - grant_acquired = cluster_pcm_lock_acquire_buffer(buf, PCM_LOCK_MODE_X); - } - PG_CATCH(); + for (;;) { - cluster_pcm_own_abort_grant_after_error(buf, &pending_base, pending_token, + ClusterBufmgrPcmRetryRearmResult rearm_result; + uint64 covered_generation = 0; + + retry_denied = false; + PG_TRY(); + { + grant_acquired = cluster_pcm_lock_acquire_buffer(buf, PCM_LOCK_MODE_X, &retry_denied); + } + PG_CATCH(); + { + cluster_pcm_own_abort_grant_after_error(buf, &pending_base, pending_token, "direct-init acquire"); - PG_RE_THROW(); + PG_RE_THROW(); + } + PG_END_TRY(); + if (!retry_denied) + break; + rearm_result = cluster_bufmgr_pcm_retry_denied_rearm( + buf, PCM_LOCK_MODE_X, &pending_base, &pending_token, retry_wait_index, NULL, + &covered_generation); + if (retry_wait_index < PG_UINT32_MAX) + retry_wait_index++; + if (rearm_result != CLUSTER_BUFMGR_PCM_RETRY_REARMED) + cluster_bufmgr_pcm_direct_init_report_failure( + buf, CLUSTER_PCM_OWN_STALE, &observed, "direct-init pending-X rearm"); } - PG_END_TRY(); if (grant_acquired) cluster_pcm_own_finish_grant_or_rollback(buf, &pending_base, pending_token, @@ -2751,16 +3277,10 @@ ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref) * NOTE: what we check here is that *this* backend holds a pin on * the buffer. We do not care whether some other backend does. */ -#define BufferIsPinned(bufnum) \ -( \ - !BufferIsValid(bufnum) ? \ - false \ - : \ - BufferIsLocal(bufnum) ? \ - (LocalRefCount[-(bufnum) - 1] > 0) \ - : \ - (GetPrivateRefCount(bufnum) > 0) \ -) +#define BufferIsPinned(bufnum) \ + (!BufferIsValid(bufnum) ? false \ + : BufferIsLocal(bufnum) ? (LocalRefCount[-(bufnum) - 1] > 0) \ + : (GetPrivateRefCount(bufnum) > 0)) static Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, @@ -2824,8 +3344,6 @@ static BufferDesc *BufferAlloc(SMgrRelation smgr, BufferAccessStrategy strategy, bool *foundPtr, IOContext io_context); static Buffer GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context); -static void FlushBuffer(BufferDesc *buf, SMgrRelation reln, - IOObject io_object, IOContext io_context); static void FindAndDropRelationBuffers(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber nForkBlock, @@ -3993,7 +4511,7 @@ InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, } old_pcm_mode = eviction_capture.pcm_state; release_pcm_holder = cluster_pcm_is_active() - && cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(oldTag)) + && cluster_pcm_x_buffer_tag_tracked(oldTag, cluster_shared_catalog) && (old_pcm_mode == (uint8) PCM_LOCK_MODE_S || old_pcm_mode == (uint8) PCM_LOCK_MODE_X); #endif @@ -4250,7 +4768,7 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) * before BufTableDelete completes the eviction. */ if (cluster_pcm_is_active() - && cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(&tag)) + && cluster_pcm_x_buffer_tag_tracked(&tag, cluster_shared_catalog) && buf_hdr->pcm_state != (uint8) PCM_STATE_N) { PcmLockMode old_mode = (PcmLockMode) buf_hdr->pcm_state; @@ -4583,11 +5101,11 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ClusterExtendEngage engage = cluster_extend_liveness_engage(true); if (engage == CLUSTER_EXTEND_ENGAGE_FAIL_CLOSED) - ereport(ERROR, - (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), - errmsg("could not acquire the cluster relation-extend lock for \"%s\"", - RelationGetRelationName(bmr.rel)), - errdetail("The cluster coordination substrate is not ready and an alive peer could not be ruled out."))); + ereport(ERROR, (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), + errmsg("could not acquire the cluster relation-extend lock for \"%s\"", + RelationGetRelationName(bmr.rel)), + errdetail("The cluster coordination substrate is not ready and an " + "alive peer could not be ruled out."))); if (engage == CLUSTER_EXTEND_ENGAGE_COORDINATE) { @@ -4599,11 +5117,12 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, */ hwc = cluster_hw_classify_persistence(bmr.rel->rd_rel->relpersistence, true); if (hwc == CLUSTER_HW_FAIL_CLOSED) - ereport(ERROR, - (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), - errmsg("unlogged relation \"%s\" cannot be safely extended in a multi-node cluster", - RelationGetRelationName(bmr.rel)), - errhint("Unlogged relations have no WAL authority to coordinate cross-node extension."))); + ereport(ERROR, (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), + errmsg("unlogged relation \"%s\" cannot be safely extended in a " + "multi-node cluster", + RelationGetRelationName(bmr.rel)), + errhint("Unlogged relations have no WAL authority to coordinate " + "cross-node extension."))); } /* @@ -4762,11 +5281,11 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, StrategyFreeBuffer(buf_hdr); UnpinBuffer(buf_hdr); } - ereport(ERROR, - (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), - errmsg("cluster relation-extend authority unavailable for \"%s\"", - RelationGetRelationName(bmr.rel)), - errhint("The HW_ALLOC round trip to the resource master could not be proven; retry."))); + ereport(ERROR, (errcode(ERRCODE_CLUSTER_RELATION_EXTEND_UNAVAILABLE), + errmsg("cluster relation-extend authority unavailable for \"%s\"", + RelationGetRelationName(bmr.rel)), + errhint("The HW_ALLOC round trip to the resource master could not be " + "proven; retry."))); } if (hw_granted < extend_by) @@ -4906,9 +5425,10 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, ereport(ERROR, (errmsg("unexpected data beyond EOF in block %u of relation %s", existing_hdr->tag.blockNum, relpath(bmr.smgr->smgr_rlocator, fork)), - errhint("This has been seen to occur with buggy kernels; consider updating your system."))); + errhint("This has been seen to occur with buggy kernels; consider " + "updating your system."))); - /* + /* * We *must* do smgr[zero]extend before succeeding, else the page * will not be reserved by the kernel, and the next P_NEW call * will decide to return the same page. Clear the BM_VALID bit, @@ -5983,11 +6503,11 @@ BgBufferSync(WritebackContext *wb_context) PendingBgWriterStats.buf_written_clean += num_written; #ifdef BGW_DEBUG - elog(DEBUG1, "bgwriter: recent_alloc=%u smoothed=%.2f delta=%ld ahead=%d density=%.2f reusable_est=%d upcoming_est=%d scanned=%d wrote=%d reusable=%d", - recent_alloc, smoothed_alloc, strategy_delta, bufs_ahead, - smoothed_density, reusable_buffers_est, upcoming_alloc_est, - bufs_to_lap - num_to_scan, - num_written, + elog(DEBUG1, + "bgwriter: recent_alloc=%u smoothed=%.2f delta=%ld ahead=%d density=%.2f reusable_est=%d " + "upcoming_est=%d scanned=%d wrote=%d reusable=%d", + recent_alloc, smoothed_alloc, strategy_delta, bufs_ahead, smoothed_density, + reusable_buffers_est, upcoming_alloc_est, bufs_to_lap - num_to_scan, num_written, reusable_buffers - reusable_buffers_est); #endif @@ -6485,6 +7005,19 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, if (!StartBufferIO(buf, false)) return; +#ifdef USE_PGRAC_CLUSTER + /* The retain-finish caller catches and absorbs this ERROR at the DATA + * worker boundary. Remember the exact ResourceOwner-tracked BufferIO so + * that caller can abort it before releasing its raw pin. Ordinary + * FlushBuffer errors keep PostgreSQL's transaction-abort cleanup. */ + if (cluster_pcm_x_finish_retain_flush_active) + { + Assert(!cluster_pcm_x_finish_retain_flush_io_active); + Assert(!cluster_pcm_x_finish_retain_flush_error_context_pushed); + cluster_pcm_x_finish_retain_flush_io_active = true; + } +#endif + #ifdef USE_PGRAC_CLUSTER /* @@ -6495,6 +7028,8 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, */ if (cluster_smart_fusion && cluster_sf_dep_buffer_flush_blocked(buf)) { TerminateBufferIO(buf, false, 0); + if (cluster_pcm_x_finish_retain_flush_io_active) + cluster_pcm_x_finish_retain_flush_io_active = false; return; } #endif @@ -6504,6 +7039,13 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, errcallback.arg = (void *) buf; errcallback.previous = error_context_stack; error_context_stack = &errcallback; +#ifdef USE_PGRAC_CLUSTER + if (cluster_pcm_x_finish_retain_flush_io_active) + { + cluster_pcm_x_finish_retain_flush_error_context_previous = errcallback.previous; + cluster_pcm_x_finish_retain_flush_error_context_pushed = true; + } +#endif /* Find smgr relation for buffer */ if (reln == NULL) @@ -6585,6 +7127,20 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, io_start = pgstat_prepare_io_time(); +#ifdef USE_PGRAC_CLUSTER + /* This dynamic scope is entered only by + * cluster_bufmgr_pcm_own_finish_revoke_retain after exact snapshot + * revalidation under caller pin + content EXCLUSIVE. Copy/materialize + * FlushBuffer calls cannot consume this one-shot. */ + if (cluster_pcm_x_finish_retain_flush_active + && cluster_pcm_own_flags_get(buf->buf_id) == PCM_OWN_FLAG_REVOKING) { + CLUSTER_INJECTION_POINT("cluster-pcm-x-retain-flush-error"); + if (cluster_injection_should_skip("cluster-pcm-x-retain-flush-error")) + ereport(ERROR, (errcode(ERRCODE_IO_ERROR), + errmsg("injected PCM-X retained-image FlushBuffer failure"))); + } +#endif + /* * bufToWrite is either the shared buffer or a copy, as appropriate. */ @@ -6643,6 +7199,10 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, * end the BM_IO_IN_PROGRESS state. */ TerminateBufferIO(buf, true, 0); +#ifdef USE_PGRAC_CLUSTER + if (cluster_pcm_x_finish_retain_flush_io_active) + cluster_pcm_x_finish_retain_flush_io_active = false; +#endif TRACE_POSTGRESQL_BUFFER_FLUSH_DONE(BufTagGetForkNum(&buf->tag), buf->tag.blockNum, @@ -6652,6 +7212,13 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, /* Pop the error context stack */ error_context_stack = errcallback.previous; +#ifdef USE_PGRAC_CLUSTER + if (cluster_pcm_x_finish_retain_flush_error_context_pushed) + { + cluster_pcm_x_finish_retain_flush_error_context_pushed = false; + cluster_pcm_x_finish_retain_flush_error_context_previous = NULL; + } +#endif } /* @@ -7745,12 +8312,16 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) #ifdef USE_PGRAC_CLUSTER /* - * Hint changes are optional. A retained image may already have had an - * in-memory hint bit touched by its caller, but it must never regain - * dirty state and become eligible for stale output. + * Hint changes are optional. A tracked buffer without live S/X current + * authority may already have had its in-memory hint bit touched by the + * caller, but it must never gain writeback eligibility. In particular, + * DRAIN can leave a pinned retained PI as a clean N mirror; dirtying that + * stale mirror would both block the next storage refresh and make it + * eligible to overwrite newer shared-storage bytes. */ retained_state = LockBufHdr(bufHdr); - if (cluster_bufmgr_pcm_x_retained_image_locked(bufHdr, retained_state)) + if (cluster_pcm_is_active() && cluster_bufmgr_should_pcm_track(bufHdr) + && !cluster_bufmgr_pcm_current_image_locked(bufHdr, retained_state)) { UnlockBufHdr(bufHdr, retained_state); return; @@ -7915,9 +8486,16 @@ UnlockBuffers(void) * PGRAC: spec-2.31 wires the PCM state machine as the outer lock for * shared-buffer content locks. PCM acquisition happens before the local * content_lock; release happens after the local content_lock is released. + * + * PGRAC (t/400 L3 item 3): pcm_barrier_refused, when non-NULL, arms the + * barrier-aware mode used by ClusterLockBufferExclusiveBarrierAware(): a + * nested-guard BARRIER_CLOSED from the PCM-X queue acquire sets the flag and + * returns WITHOUT taking the content lock, leaving the unwind to the caller + * that owns the outer (frozen-tag) content lock. A NULL pointer keeps the + * historical behavior (the refusal escalates to a client ERROR). */ -void -LockBuffer(Buffer buffer, int mode) +static void +LockBufferInternal(Buffer buffer, int mode, bool *pcm_barrier_refused) { BufferDesc *buf; #ifdef USE_PGRAC_CLUSTER @@ -7934,6 +8512,8 @@ LockBuffer(Buffer buffer, int mode) ClusterPcmXHolderLedgerEntry *pcm_x_holder = NULL; ClusterPcmXWriterLedgerEntry *pcm_x_writer = NULL; bool pcm_x_writer_managed = false; + bool pcm_retry_denied = false; + uint32 pcm_retry_wait_index = 0; #endif Assert(BufferIsPinned(buffer)); @@ -7970,7 +8550,10 @@ LockBuffer(Buffer buffer, int mode) } if (!pcm_covered) - pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode); + pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode, + pcm_barrier_refused); + if (pcm_barrier_refused != NULL && *pcm_barrier_refused) + return; /* barrier-aware caller owns the unwind */ if (pcm_x_writer == NULL && !pcm_covered) { /* * PGRAC: spec-4.7a D2 — hold-until-revoked acquire gate. @@ -8037,12 +8620,15 @@ LockBuffer(Buffer buffer, int mode) * JOIN/WAIT and call begin_x_reservation only from * ACTIVE_TRANSFER/PREPARE. */ - pcm_pending_result = cluster_pcm_own_begin_grant_reservation(buf, &pcm_pending_base, - &pcm_pending_token); + pcm_pending_result = cluster_bufmgr_pcm_begin_grant_reservation_wait( + buf, &pcm_pending_base, &pcm_pending_token); if (pcm_pending_result != CLUSTER_PCM_OWN_OK) cluster_pcm_own_report_bump_failure( buf, pcm_pending_result, pcm_pending_base.generation, pcm_pending_base.flags, "LockBuffer master reservation"); + cluster_bufmgr_pcm_legacy_begin_probe(buf, pcm_mode, + &pcm_pending_base, + "master-reservation"); pcm_pending_set = true; /* @@ -8052,25 +8638,44 @@ LockBuffer(Buffer buffer, int mode) * left at N (the next access re-fetches — a cached copy * with no invalidation path would go stale, Rule 8.A). */ - PG_TRY(); - { - pcm_acquired = cluster_pcm_lock_acquire_buffer(buf, pcm_mode); - } - PG_CATCH(); + for (;;) { - /* - * W3 — an acquire that THREW (upgrade-invalidate - * timeout, transfer deny, ...) must not leak - * GRANT_PENDING: a stale marker parks every later - * same-tag INVALIDATE on this node (never ACKed -> remote - * upgrades wedge, liveness). The later PG_CATCH below - * only covers the content-lock window, not this acquire. - */ - cluster_pcm_own_abort_grant_after_error( - buf, &pcm_pending_base, pcm_pending_token, "LockBuffer master acquire"); - PG_RE_THROW(); + ClusterBufmgrPcmRetryRearmResult rearm_result; + + pcm_retry_denied = false; + PG_TRY(); + { + pcm_acquired = cluster_pcm_lock_acquire_buffer(buf, pcm_mode, + &pcm_retry_denied); + } + PG_CATCH(); + { + /* An acquire that throws must not leak the exact + * GRANT_PENDING marker into later INVALIDATEs. */ + cluster_pcm_own_abort_grant_after_error( + buf, &pcm_pending_base, pcm_pending_token, + "LockBuffer master acquire"); + PG_RE_THROW(); + } + PG_END_TRY(); + if (!pcm_retry_denied) + break; + + pcm_pending_set = false; + rearm_result = cluster_bufmgr_pcm_retry_denied_rearm( + buf, pcm_mode, &pcm_pending_base, &pcm_pending_token, + pcm_retry_wait_index, pcm_barrier_refused, &pcm_covered_gen); + if (pcm_retry_wait_index < PG_UINT32_MAX) + pcm_retry_wait_index++; + if (rearm_result == CLUSTER_BUFMGR_PCM_RETRY_BARRIER_REFUSED) + return; + if (rearm_result == CLUSTER_BUFMGR_PCM_RETRY_COVERED) + { + pcm_covered = true; + break; + } + pcm_pending_set = true; } - PG_END_TRY(); } } } @@ -8113,7 +8718,20 @@ LockBuffer(Buffer buffer, int mode) CLUSTER_INJECTION_POINT("cluster-pcm-writer-cached-x-stall"); PG_TRY(); { - pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf); + pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf, pcm_barrier_refused); + if (pcm_barrier_refused != NULL && *pcm_barrier_refused) + { + /* + * PGRAC (t/400 L3 item 3, holder lane): the holder gate + * refused under the frozen barrier. Roll the granted writer + * claim back exactly like the ERROR path would and take + * nothing; the barrier-aware caller owns the unwind. + */ + if (pcm_x_writer != NULL) + cluster_bufmgr_pcm_x_writer_abort_acquiring(pcm_x_writer); + } + else + { #endif if (mode == BUFFER_LOCK_SHARE) LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED); @@ -8126,14 +8744,13 @@ LockBuffer(Buffer buffer, int mode) * PGRAC ownership-generation wave (W1) — cached-cover * re-verify. The cover fast path decided we already held the mode * on a raw, unlocked pcm_state read. A BAST X->S downgrade (or - * any ownership round) can have raced this content-lock window - * and revoked the cover. Re-read the state/generation/flags - * projection of the coherent ownership tuple under the header - * spinlock: if the generation moved, the mode no longer covers, - * or a grant/revoke is in flight, the cover is STALE -- - * writing/reading a block we no longer own is the Rule 8.A - * violation. Release, do a real master re-acquire, re-take the - * content lock. Bounded: after a real acquire we hold the + * any ownership round) can have raced this content-lock window. + * Re-read the coherent ownership tuple now that content authority + * serializes page-image transitions. A stable successor S/X still + * covers an S read; an X writer stays generation-exact. A mode + * loss or live lifecycle is STALE for both. Release, do a real + * master/queue re-acquire, and re-take the content lock. Bounded: + * after a real acquire we hold the * content lock and the downgrade path serializes under it, so no * further downgrade can intervene -- at most one fallback. */ @@ -8144,30 +8761,107 @@ LockBuffer(Buffer buffer, int mode) uint32 cur_flags; cluster_pcm_own_read(buf, &cur_state, &cur_gen, &cur_flags); - if (cur_gen != pcm_covered_gen - || !cluster_pcm_mode_covers((PcmLockMode) cur_state, pcm_mode) - || (cur_flags & (PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING)) != 0) + if (!cluster_pcm_x_cached_cover_reverify_accepts( + (uint8) pcm_mode, pcm_covered_gen, cur_gen, cur_state, cur_flags)) { cluster_pcm_note_writer_cover_stale_detected(); pcm_covered = false; LWLockRelease(BufferDescriptorGetContentLock(buf)); - pcm_pending_result = cluster_pcm_own_begin_grant_reservation( - buf, &pcm_pending_base, &pcm_pending_token); - if (pcm_pending_result != CLUSTER_PCM_OWN_OK) - cluster_pcm_own_report_bump_failure( - buf, pcm_pending_result, pcm_pending_base.generation, - pcm_pending_base.flags, "LockBuffer revalidate reservation"); - pcm_pending_set = true; - pcm_acquired = cluster_pcm_lock_acquire_buffer(buf, pcm_mode); - pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf); - if (mode == BUFFER_LOCK_SHARE) - LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED); + + /* + * PGRAC (t/400 fast-fail finish family): a stale X cover + * means an ownership round moved this block while we + * raced the content-lock window (a BAST X->S downgrade + * being the production case). A writer re-acquire from + * that state is a NEW writer conversion and MUST go back + * through the convert queue's FIFO/WFG arbitration -- the + * legacy master begin from an S base would both bypass + * the queue (the original S3 unordered multi-writer + * defect) and mint the S_NEW reservation shape the finish + * classifier refuses by design. Detach the ACQUIRING + * holder prepared above first; the queue grant binds a + * fresh one. + */ + if (pcm_mode == PCM_LOCK_MODE_X) + { + if (pcm_x_holder != NULL) + { + cluster_bufmgr_pcm_x_holder_abort_acquiring(pcm_x_holder); + pcm_x_holder = NULL; + } + pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode, + pcm_barrier_refused); + } + if (pcm_barrier_refused != NULL && *pcm_barrier_refused) + { + /* + * The queue gate refused under a frozen barrier; the + * barrier-aware caller owns the unwind. No lock is + * retaken and no holder is bound (the post-PG_TRY + * refusal exit returns before any mirror update). + */ + } else - LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE); - cluster_pcm_note_writer_reverify_reacquire(); + { + if (pcm_x_writer == NULL) + { + ClusterBufmgrPcmRetryRearmResult rearm_result; + + /* + * Share-mode refresh, or the convert queue is not + * managing this relation: the legacy master + * acquire remains the ordered path. + */ + pcm_pending_result = cluster_bufmgr_pcm_begin_grant_reservation_wait( + buf, &pcm_pending_base, &pcm_pending_token); + if (pcm_pending_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_own_report_bump_failure( + buf, pcm_pending_result, pcm_pending_base.generation, + pcm_pending_base.flags, "LockBuffer revalidate reservation"); + cluster_bufmgr_pcm_legacy_begin_probe( + buf, pcm_mode, &pcm_pending_base, + "revalidate-reservation"); + pcm_pending_set = true; + for (;;) + { + pcm_retry_denied = false; + pcm_acquired = cluster_pcm_lock_acquire_buffer( + buf, pcm_mode, &pcm_retry_denied); + if (!pcm_retry_denied) + break; + pcm_pending_set = false; + rearm_result = cluster_bufmgr_pcm_retry_denied_rearm( + buf, pcm_mode, &pcm_pending_base, &pcm_pending_token, + pcm_retry_wait_index, pcm_barrier_refused, + &pcm_covered_gen); + if (pcm_retry_wait_index < PG_UINT32_MAX) + pcm_retry_wait_index++; + if (rearm_result + == CLUSTER_BUFMGR_PCM_RETRY_BARRIER_REFUSED) + break; + if (rearm_result == CLUSTER_BUFMGR_PCM_RETRY_COVERED) + { + pcm_covered = true; + break; + } + pcm_pending_set = true; + } + } + if (pcm_barrier_refused == NULL || !*pcm_barrier_refused) + { + pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf, NULL); + if (mode == BUFFER_LOCK_SHARE) + LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED); + else + LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE); + cluster_bufmgr_pcm_x_writer_activate(pcm_x_writer); + cluster_pcm_note_writer_reverify_reacquire(); + } + } } } cluster_bufmgr_pcm_x_holder_activate(pcm_x_holder); + } } PG_CATCH(); { @@ -8212,6 +8906,11 @@ LockBuffer(Buffer buffer, int mode) } PG_END_TRY(); + /* PGRAC (t/400 L3 item 3, holder lane): nothing is held and the + * writer claim was rolled back — hand the refusal to the caller. */ + if (pcm_barrier_refused != NULL && *pcm_barrier_refused) + return; + /* * PGRAC ownership-generation wave (W3) — grant-finalize window. A * real X acquire has installed the grant but pcm_state is still N @@ -8241,8 +8940,8 @@ LockBuffer(Buffer buffer, int mode) "cluster-pcm-grant-finalize-deliver-invalidate")) { if (cluster_gcs_block_test_deliver_self_invalidate(buf->tag)) - elog(WARNING, - "cluster W3 delivery shim: synthetic INVALIDATE was ACKed instead of parked (GRANT_PENDING not honored)"); + elog(WARNING, "cluster W3 delivery shim: synthetic INVALIDATE was ACKed " + "instead of parked (GRANT_PENDING not honored)"); } } @@ -8327,6 +9026,34 @@ LockBuffer(Buffer buffer, int mode) #endif } +void +LockBuffer(Buffer buffer, int mode) +{ + LockBufferInternal(buffer, mode, NULL); +} + +/* + * PGRAC (t/400 L3 item 3): EXCLUSIVE LockBuffer that reports a nested-guard + * BARRIER_CLOSED refusal instead of raising a client ERROR. + * + * Returns true with the content lock held (every non-barrier path behaves + * exactly like LockBuffer, including its error surface). Returns false with + * NOTHING held when the PCM-X queue acquire refused because this backend + * already holds another content lock whose tag sits under a frozen revoke + * barrier: sleeping here could deadlock across locks and the frozen barrier + * snapshot cannot see the nested edge. The caller — the owner of that outer + * content lock — must release its own lock(s), resolve the map-page + * conversion while holding no content lock, and re-enter a requalify point. + */ +bool +ClusterLockBufferExclusiveBarrierAware(Buffer buffer) +{ + bool barrier_refused = false; + + LockBufferInternal(buffer, BUFFER_LOCK_EXCLUSIVE, &barrier_refused); + return !barrier_refused; +} + /* VM/FSM pages read with RBM_ZERO_ON_ERROR become BM_VALID before their * PageIsNew initialization. Dedicated wrappers are the provenance: a plain * LockBuffer(X) on the same N buffer remains queue-only. */ @@ -9525,76 +10252,126 @@ cluster_bufmgr_read_storage_scn_for_gcs(BufferTag tag, SCN *out_page_scn) return true; } +const char * +cluster_bufmgr_gcs_copy_refusal_name(ClusterBufmgrGcsCopyRefusal refusal) +{ + switch (refusal) { + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE: + return "none"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INVALID_ARGUMENT: + return "invalid_argument"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT: + return "not_resident"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID: + return "current_invalid"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST: + return "buffer_content_conditional_1"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_SECOND: + return "buffer_content_conditional_2"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY: + return "ownership_revoke_busy"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_HC89_LSN_DRIFT: + return "hc89_lsn_drift"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_SMART_FUSION_UNCLASSIFIED: + return "smart_fusion_unclassified"; + case CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INJECTED_EVICT: + return "injected_evict"; + } + return "unknown"; +} + /* * Copy the 8KB block bytes for `tag` into *dst, flushing WAL up to the - * page's LSN before reading the bytes (HC82 I-WAL-before-ship). Sets + * page's LSN before reading the bytes (HC82 I-WAL-before-ship), then making + * any dirty source current in shared storage before it may be retired. Sets * *out_page_lsn to the page LSN observed at the second-stable revalidation. * - * Returns false on: - * - Buffer no longer in pool (evicted between probe and pin) - * - HC89 revalidation single-retry exhausted + * Returns false on a non-resident or non-current image, either conditional + * content-lock refusal, or HC89 revalidation single-retry exhaustion. When + * requested, *out_refusal identifies that exact terminal stage. * * Returns true with *dst populated and *out_page_lsn set otherwise. * * Concurrency: caller does NOT hold any buffer/partition locks. We take a * SHARED partition lock to look up, raw-pin the shared buffer refcount, then - * drop the partition lock and operate on the buffer's content_lock. The raw - * pin is always released before return. + * drop the partition lock and conditionally acquire the buffer's + * content_lock. GCS DATA workers must keep draining replies while a backend + * owns the content lock and waits for one of those replies; waiting here + * would deadlock that backend against its own receive worker. A busy lock is + * therefore a retryable no-image result. The raw pin is always released + * before return. */ bool -cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char *dst) +cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char *dst, + ClusterBufmgrGcsCopyRefusal *out_refusal) { - uint32 hashcode; - LWLock *partition_lock; - int buf_id; + uint32 hashcode; + LWLock *partition_lock; + int buf_id; BufferDesc *buf; - LWLock *content_lock; - XLogRecPtr first_lsn; - XLogRecPtr second_lsn; - int retries; - bool stable; - Page page; + LWLock *content_lock; + XLogRecPtr first_lsn; + XLogRecPtr second_lsn; + int retries; + bool stable; + bool needs_flush; + bool storage_current; + Page page; + volatile bool content_locked = false; + volatile bool caller_pinned = false; + + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE; Assert(dst != NULL); Assert(out_page_lsn != NULL); + if (dst == NULL || out_page_lsn == NULL) { + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INVALID_ARGUMENT; + return false; + } hashcode = BufTableHashCode(&tag); partition_lock = BufMappingPartitionLock(hashcode); LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hashcode); - if (buf_id < 0) - { + if (buf_id < 0) { LWLockRelease(partition_lock); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT; return false; } buf = GetBufferDescriptor(buf_id); /* Partition lock keeps the buffer from being recycled before we raw-pin. */ { - uint32 buf_state; + uint32 buf_state; buf_state = LockBufHdr(buf); /* Re-verify tag under header lock to defend against tag-rewrite * races between the partition-lock-protected lookup and the pin. */ - if (!BufferTagsEqual(&buf->tag, &tag)) - { + if (!BufferTagsEqual(&buf->tag, &tag)) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT; return false; } - if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) - { + if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; return false; } cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); + caller_pinned = true; } LWLockRelease(partition_lock); content_lock = BufferDescriptorGetContentLock(buf); - page = (Page) BufHdrGetBlock(buf); + page = (Page)BufHdrGetBlock(buf); /* * HC89: revalidation loop with single retry budget. retries = 0 — first @@ -9602,76 +10379,159 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char * fail-closed */ stable = false; - for (retries = 0; retries < 2; retries++) + PG_TRY(); { - /* Read page_lsn under content_lock SHARED. */ - LWLockAcquire(content_lock, LW_SHARED); - first_lsn = PageGetLSN(page); - LWLockRelease(content_lock); + for (retries = 0; retries < 2; retries++) { + /* Read page_lsn under content_lock SHARED. Never park a GCS DATA + * worker behind a backend that may itself be waiting for this worker. */ + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) { + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST; + break; + } + content_locked = true; + first_lsn = PageGetLSN(page); + LWLockRelease(content_lock); + content_locked = false; - /* + /* * HC82: flush WAL up to the page LSN before shipping the bytes. Use * XLogFlush(page_lsn) specifically — NOT XLogFlush(insert pointer) * which would be correct but over-flushes and doesn't express the * "flush this page before ship" safety contract. */ #ifdef USE_CLUSTER_UNIT - if (cluster_gcs_block_test_xlog_flush_hook != NULL) - cluster_gcs_block_test_xlog_flush_hook((uint64) first_lsn); + if (cluster_gcs_block_test_xlog_flush_hook != NULL) + cluster_gcs_block_test_xlog_flush_hook((uint64)first_lsn); #endif - if (!XLogRecPtrIsInvalid(first_lsn)) - XLogFlush(cluster_gcs_clamp_ship_flush_lsn(first_lsn)); + if (!XLogRecPtrIsInvalid(first_lsn)) + XLogFlush(cluster_gcs_clamp_ship_flush_lsn(first_lsn)); - /* + /* * Reacquire content_lock SHARED and revalidate that the page LSN has * not advanced past first_lsn AND the buffer tag still matches. * Either signals concurrent mutation that would break HC82's "ship * the bytes that I just flushed WAL for" contract. */ - LWLockAcquire(content_lock, LW_SHARED); - { - uint32 buf_state = LockBufHdr(buf); - bool current = BufferTagsEqual(&buf->tag, &tag) - && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); - - UnlockBufHdr(buf, buf_state); - if (!current) - { - LWLockRelease(content_lock); + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) { + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_SECOND; break; } - } - second_lsn = PageGetLSN(page); + content_locked = true; + { + uint32 buf_state = LockBufHdr(buf); + bool tag_matches = BufferTagsEqual(&buf->tag, &tag); + bool current = tag_matches + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); + + needs_flush + = current + && (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0; + storage_current = current && (buf_state & BM_IO_ERROR) == 0; + + UnlockBufHdr(buf, buf_state); + if (!storage_current) { + if (out_refusal != NULL) + *out_refusal = tag_matches + ? CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID + : CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT; + LWLockRelease(content_lock); + content_locked = false; + break; + } + } + second_lsn = PageGetLSN(page); #ifdef USE_CLUSTER_UNIT - /* + /* * Spec L25/L26 hook: if the test injection returns N > 0 we mimic N * consecutive LSN drift events. retries 0 + drift available means * second_lsn != first_lsn synthetically. */ - if (cluster_gcs_block_test_lsn_drift_hook != NULL) - { - int drift_remaining = cluster_gcs_block_test_lsn_drift_hook(); + if (cluster_gcs_block_test_lsn_drift_hook != NULL) { + int drift_remaining = cluster_gcs_block_test_lsn_drift_hook(); - if (drift_remaining > retries) - second_lsn = first_lsn + 1; /* synthetic mismatch */ - } + if (drift_remaining > retries) + second_lsn = first_lsn + 1; /* synthetic mismatch */ + } #endif - if (BufferTagsEqual(&buf->tag, &tag) && first_lsn == second_lsn) - { - memcpy(dst, page, BLCKSZ); - *out_page_lsn = second_lsn; + if (first_lsn == second_lsn) { + uint32 buf_state; + + /* A queue handoff may retire this descriptor immediately after + * copying it. Make the shared-storage fallback at least as current + * as the shipped image before publishing that handoff watermark. */ + if (needs_flush) + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + + buf_state = LockBufHdr(buf); + storage_current = BufferTagsEqual(&buf->tag, &tag) + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state) + && PageGetLSN(page) == second_lsn + && (buf_state + & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED + | BM_IO_ERROR | BM_IO_IN_PROGRESS)) + == 0; + UnlockBufHdr(buf, buf_state); + if (!storage_current) { + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + LWLockRelease(content_lock); + content_locked = false; + continue; + } + + memcpy(dst, page, BLCKSZ); + + /* Hint-bit dirties may occur under a shared content lock. Do not + * certify a copy if one raced the memcpy after the first clean check. */ + buf_state = LockBufHdr(buf); + storage_current = BufferTagsEqual(&buf->tag, &tag) + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state) + && PageGetLSN(page) == second_lsn + && (buf_state + & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED + | BM_IO_ERROR | BM_IO_IN_PROGRESS)) + == 0; + UnlockBufHdr(buf, buf_state); + if (!storage_current) { + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + LWLockRelease(content_lock); + content_locked = false; + continue; + } + + *out_page_lsn = second_lsn; + LWLockRelease(content_lock); + content_locked = false; + stable = true; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE; + break; + } + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_HC89_LSN_DRIFT; LWLockRelease(content_lock); - stable = true; - break; + content_locked = false; + /* fall through: retry once if budget remains */ } - LWLockRelease(content_lock); - /* fall through: retry once if budget remains */ } + PG_CATCH(); + { + if (content_locked && LWLockHeldByMe(content_lock)) + LWLockRelease(content_lock); + if (caller_pinned) + cluster_bufmgr_unpin_for_gcs(buf); + PG_RE_THROW(); + } + PG_END_TRY(); - cluster_bufmgr_unpin_for_gcs(buf); + if (caller_pinned) + cluster_bufmgr_unpin_for_gcs(buf); return stable; } @@ -9740,7 +10600,8 @@ cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page for (retries = 0; retries < 2; retries++) { - LWLockAcquire(content_lock, LW_SHARED); + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) + break; first_lsn = PageGetLSN(page); LWLockRelease(content_lock); @@ -9751,7 +10612,8 @@ cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page if (!XLogRecPtrIsInvalid(first_lsn)) XLogFlush(cluster_gcs_clamp_ship_flush_lsn(first_lsn)); - LWLockAcquire(content_lock, LW_SHARED); + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) + break; { uint32 buf_state = LockBufHdr(buf); bool current = BufferTagsEqual(&buf->tag, &tag) @@ -9891,17 +10753,36 @@ cluster_bufmgr_finish_direct_land_target_for_gcs(BufferDesc *buf, bool valid, * harmless). Runs in the LMON IC-dispatch context; the FlushBuffer * call mirrors the checkpointer contract (pin + content lock held). * ======================================================================== */ -bool -cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) +ClusterBufmgrGcsDowngradeOutcome +cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image( + BufferTag tag, XLogRecPtr *out_page_lsn, char *dst, + ClusterBufmgrGcsCopyRefusal *out_refusal) { uint32 hashcode; LWLock *partition_lock; int buf_id; BufferDesc *buf; LWLock *content_lock; + ClusterPcmOwnSnapshot current; + ClusterPcmOwnSnapshot revoking; + ClusterPcmOwnSnapshot shared; + ClusterPcmOwnResult own_result; + Page page; + XLogRecPtr first_lsn; + XLogRecPtr second_lsn; uint32 buf_state; bool dirty; + if (out_page_lsn != NULL) + *out_page_lsn = InvalidXLogRecPtr; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE; + if (out_page_lsn == NULL || dst == NULL) { + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INVALID_ARGUMENT; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + hashcode = BufTableHashCode(&tag); partition_lock = BufMappingPartitionLock(hashcode); @@ -9910,7 +10791,9 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) if (buf_id < 0) { LWLockRelease(partition_lock); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } buf = GetBufferDescriptor(buf_id); @@ -9919,18 +10802,31 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); LWLockRelease(partition_lock); content_lock = BufferDescriptorGetContentLock(buf); - LWLockAcquire(content_lock, LW_EXCLUSIVE); + /* This helper is called by the GCS DATA worker for FORWARD/BAST. A + * blocking content-lock wait can prevent the same worker from receiving + * the reply awaited by the lock owner, so refusal must be immediate. */ + if (!LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) + { + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } /* @@ -9945,34 +10841,124 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + own_result = cluster_bufmgr_pcm_own_snapshot(buf, ¤t); + if (own_result != CLUSTER_PCM_OWN_OK || !BufferTagsEqual(¤t.tag, &tag) + || current.pcm_state != (uint8)PCM_STATE_X || current.flags != 0) { + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + own_result = cluster_bufmgr_pcm_own_begin_x_revoke(buf, ¤t, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) { + if (own_result != CLUSTER_PCM_OWN_BUSY && own_result != CLUSTER_PCM_OWN_STALE) + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } buf_state = LockBufHdr(buf); dirty = (buf_state & BM_DIRTY) != 0; UnlockBufHdr(buf, buf_state); - if (dirty) - FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + PG_TRY(); + { + if (dirty) + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + } + PG_CATCH(); + { + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + PG_RE_THROW(); + } + PG_END_TRY(); + + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0 + || (buf_state & (BM_DIRTY | BM_IO_IN_PROGRESS)) != 0 + || buf->pcm_state != (uint8)PCM_STATE_X + || cluster_pcm_own_gen_get(buf->buf_id) != revoking.generation + || cluster_pcm_own_reservation_token_get(buf->buf_id) != revoking.reservation_token + || cluster_pcm_own_flags_get(buf->buf_id) != PCM_OWN_FLAG_REVOKING + || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { + UnlockBufHdr(buf, buf_state); + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + UnlockBufHdr(buf, buf_state); + page = (Page)BufHdrGetBlock(buf); + first_lsn = PageGetLSN(page); + memcpy(dst, page, BLCKSZ); + second_lsn = PageGetLSN(page); + if (first_lsn != second_lsn) { + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_HC89_LSN_DRIFT; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } if (!cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_X_TO_S_DOWNGRADE, cluster_node_id)) { /* Master refused (state moved under us) — leave local X untouched. */ + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return false; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } - /* PGRAC ownership-generation wave: the X->S downgrade is a committed - * ownership transition -- set pcm_state and bump the generation atomically - * (header spinlock) so a cached-X writer racing the content-lock window - * detects the revoke even across an X->S->X ABA. */ - cluster_pcm_own_transition(buf, (uint8) PCM_STATE_S, 0, 0); + own_result = cluster_bufmgr_pcm_own_finish_x_to_s_downgrade(buf, &revoking, &shared); + if (own_result != CLUSTER_PCM_OWN_OK || revoking.generation == UINT64_MAX + || shared.generation != revoking.generation + 1 + || shared.reservation_token != revoking.reservation_token || shared.flags != 0 + || shared.pcm_state != (uint8)PCM_STATE_S || !BufferTagsEqual(&shared.tag, &tag)) { + /* The local master transition already committed. Do not invent a + * rollback or expose an S grant from a divergent local tuple. */ + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + return CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY; + } + *out_page_lsn = second_lsn; LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return true; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED; +} + +bool +cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) +{ + PGAlignedBlock block; + XLogRecPtr page_lsn; + + return cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image( + tag, &page_lsn, block.data, NULL) + == CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED; } /* ======================================================================== @@ -10112,19 +11098,93 @@ cluster_bufmgr_unlock_resident_stamp(Buffer buffer) * requester's registration try-ACK fails closed to one-shot semantics — * no path serves a copy the master does not track (Rule 8.A). * ======================================================================== */ -bool -cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) +static ClusterPcmOwnResult +cluster_bufmgr_pcm_own_finish_x_to_s_downgrade( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, + ClusterPcmOwnSnapshot *out_shared) +{ + ClusterPcmOwnResult live_result; + ClusterPcmOwnResult result; + uint64 committed_generation = 0; + uint64 live_token; + uint32 flags; + uint32 buf_state; + + if (buf == NULL || expected_revoking == NULL || out_shared == NULL) + return CLUSTER_PCM_OWN_INVALID; + memset(out_shared, 0, sizeof(*out_shared)); + if (!LWLockHeldByMe(BufferDescriptorGetContentLock(buf)) + || expected_revoking->pcm_state != (uint8)PCM_STATE_X + || expected_revoking->flags != PCM_OWN_FLAG_REVOKING + || expected_revoking->reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, &expected_revoking->tag) + || (buf_state & BM_VALID) == 0 + || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation + || buf->pcm_state != (uint8)PCM_STATE_X) + result = CLUSTER_PCM_OWN_STALE; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + result = CLUSTER_PCM_OWN_CORRUPT; + else { + live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if (flags == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (flags != PCM_OWN_FLAG_REVOKING) + result = CLUSTER_PCM_OWN_BUSY; + else if (live_token != expected_revoking->reservation_token) + result = CLUSTER_PCM_OWN_STALE; + else { + result = cluster_pcm_own_revoke_commit_exact( + buf->buf_id, expected_revoking->generation, + expected_revoking->reservation_token, &committed_generation); + if (result == CLUSTER_PCM_OWN_OK) { + buf->buffer_type = (uint8)BUF_TYPE_SCUR; + buf->pcm_state = (uint8)PCM_STATE_S; + cluster_pcm_own_snapshot_locked(buf, out_shared); + } + } + } + UnlockBufHdr(buf, buf_state); + return result; +} + +ClusterBufmgrGcsDowngradeOutcome +cluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image( + BufferTag tag, int32 master_node, XLogRecPtr *out_page_lsn, char *dst, + ClusterBufmgrGcsCopyRefusal *out_refusal) { uint32 hashcode; LWLock *partition_lock; int buf_id; BufferDesc *buf; LWLock *content_lock; + ClusterPcmOwnSnapshot current; + ClusterPcmOwnSnapshot revoking; + ClusterPcmOwnSnapshot shared; + ClusterPcmOwnResult own_result; + Page page; + XLogRecPtr first_lsn; + XLogRecPtr second_lsn; uint32 buf_state; bool dirty; + bool notify_handed_off = false; - if (master_node < 0 || master_node == cluster_node_id) - return false; + if (out_page_lsn != NULL) + *out_page_lsn = InvalidXLogRecPtr; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE; + + if (master_node < 0 || master_node == cluster_node_id || out_page_lsn == NULL || dst == NULL) { + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INVALID_ARGUMENT; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } hashcode = BufTableHashCode(&tag); partition_lock = BufMappingPartitionLock(hashcode); @@ -10134,7 +11194,9 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) if (buf_id < 0) { LWLockRelease(partition_lock); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } buf = GetBufferDescriptor(buf_id); @@ -10143,18 +11205,31 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); LWLockRelease(partition_lock); content_lock = BufferDescriptorGetContentLock(buf); - LWLockAcquire(content_lock, LW_EXCLUSIVE); + /* Same receive-worker liveness rule as the master==holder downgrade: a + * busy local writer makes this downgrade request retryable, never a reason + * to park the DATA dispatch loop. */ + if (!LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) + { + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } /* Same re-verify as the local variant: tag, PCM X, quiescent. */ @@ -10164,15 +11239,87 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) { LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return false; + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + + own_result = cluster_bufmgr_pcm_own_snapshot(buf, ¤t); + if (own_result != CLUSTER_PCM_OWN_OK || !BufferTagsEqual(¤t.tag, &tag) + || current.pcm_state != (uint8)PCM_STATE_X || current.flags != 0) { + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + own_result = cluster_bufmgr_pcm_own_begin_x_revoke(buf, ¤t, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) { + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY; + if (own_result != CLUSTER_PCM_OWN_BUSY && own_result != CLUSTER_PCM_OWN_STALE) + cluster_pcm_x_runtime_fail_closed(); + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } buf_state = LockBufHdr(buf); dirty = (buf_state & BM_DIRTY) != 0; UnlockBufHdr(buf, buf_state); - if (dirty) - FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + PG_TRY(); + { + if (dirty) + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + } + PG_CATCH(); + { + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + PG_RE_THROW(); + } + PG_END_TRY(); + + /* Content EXCLUSIVE keeps the page bytes stable. Prove the exact + * REVOKING ownership and clean current-image shape both before and after + * the stack copy, while the notify has not yet crossed the wire boundary. */ + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0 + || (buf_state & (BM_DIRTY | BM_IO_IN_PROGRESS)) != 0 + || buf->pcm_state != (uint8)PCM_STATE_X + || cluster_pcm_own_gen_get(buf->buf_id) != revoking.generation + || cluster_pcm_own_reservation_token_get(buf->buf_id) != revoking.reservation_token + || cluster_pcm_own_flags_get(buf->buf_id) != PCM_OWN_FLAG_REVOKING + || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { + UnlockBufHdr(buf, buf_state); + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } + UnlockBufHdr(buf, buf_state); + page = (Page)BufHdrGetBlock(buf); + first_lsn = PageGetLSN(page); + memcpy(dst, page, BLCKSZ); + second_lsn = PageGetLSN(page); + if (first_lsn != second_lsn) { + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + if (out_refusal != NULL) + *out_refusal = CLUSTER_BUFMGR_GCS_COPY_REFUSAL_HC89_LSN_DRIFT; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; + } /* * PGRAC: GCS-race round-4c P1 (yield-notify liveness) — deterministic @@ -10185,24 +11332,50 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) if (cluster_injection_should_skip("cluster-gcs-block-yield-notify-drop")) { /* simulated post-handoff loss: fall through to the S flip */ + notify_handed_off = true; } - else if (!cluster_gcs_send_transition_nowait(tag, PCM_TRANS_X_TO_S_DOWNGRADE, master_node)) + else + notify_handed_off + = cluster_gcs_send_transition_nowait(tag, PCM_TRANS_X_TO_S_DOWNGRADE, master_node); + if (!notify_handed_off) { - /* Notify not handed to transport — keep local X, master unchanged. */ + own_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, &revoking); + if (own_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return false; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY; } - /* PGRAC ownership-generation wave: the X->S downgrade is a committed - * ownership transition -- set pcm_state and bump the generation atomically - * (header spinlock) so a cached-X writer racing the content-lock window - * detects the revoke even across an X->S->X ABA. */ - cluster_pcm_own_transition(buf, (uint8) PCM_STATE_S, 0, 0); - - LWLockRelease(content_lock); + own_result = cluster_bufmgr_pcm_own_finish_x_to_s_downgrade(buf, &revoking, &shared); + if (own_result != CLUSTER_PCM_OWN_OK + || revoking.generation == UINT64_MAX + || shared.generation != revoking.generation + 1 + || shared.reservation_token != revoking.reservation_token || shared.flags != 0 + || shared.pcm_state != (uint8)PCM_STATE_S || !BufferTagsEqual(&shared.tag, &tag)) { + /* Notify may already be visible to the master. Never fabricate a local + * rollback or S grant after this irreversible boundary. */ + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + return CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY; + } + *out_page_lsn = second_lsn; + + LWLockRelease(content_lock); cluster_bufmgr_unpin_for_gcs(buf); - return true; + return CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED; +} + +bool +cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) +{ + PGAlignedBlock block; + XLogRecPtr page_lsn; + + return cluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image( + tag, master_node, &page_lsn, block.data, NULL) + == CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED; } /* ======================================================================== @@ -10320,7 +11493,8 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa stable = false; for (retries = 0; retries < 2; retries++) { - LWLockAcquire(content_lock, LW_SHARED); + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) + break; { uint32 buf_state = LockBufHdr(buf); bool current = BufferTagsEqual(&buf->tag, &tag) @@ -10485,6 +11659,7 @@ ClusterBufmgrGcsDropResult cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode, XLogRecPtr *out_page_lsn, SCN *out_page_scn) { + ClusterPcmOwnSnapshot activation_diag; uint32 hashcode; LWLock *partition_lock; int buf_id; @@ -10495,6 +11670,8 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode bool was_dirty = false; uint8 saved_pcm_state; uint64 staged_gen; /* PGRAC W2: ownership gen captured at stage-N */ + bool activation_diag_valid = false; + bool invalidate_succeeded; (void) expected_mode; @@ -10561,7 +11738,10 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode LWLock *content_lock = BufferDescriptorGetContentLock(buf); Page page = (Page) BufHdrGetBlock(buf); - LWLockAcquire(content_lock, LW_SHARED); + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) { + cluster_bufmgr_unpin_for_gcs(buf); + return CLUSTER_BUFMGR_GCS_DROP_PINNED; + } page_lsn = PageGetLSN(page); page_scn = ((PageHeader) page)->pd_block_scn; LWLockRelease(content_lock); @@ -10611,10 +11791,16 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode saved_pcm_state = buf->pcm_state; staged_gen = cluster_pcm_own_gen_get(buf->buf_id); /* PGRAC W2 */ buf->pcm_state = (uint8) PCM_STATE_N; + cluster_pcm_own_snapshot_locked(buf, &activation_diag); + activation_diag_valid = activation_diag.writer_activation_token != 0; /* PGRAC: spec-6.12h D-h1 — keep a Past Image instead of dropping. */ - if (cluster_bufmgr_convert_to_pi_locked(buf, buf_state)) + if (cluster_bufmgr_convert_to_pi_locked(buf, buf_state)) { + if (activation_diag_valid) + cluster_pcm_own_activation_diag_emit("invalidate-stage-n-pi", buf->buf_id, + &activation_diag, CLUSTER_PCM_OWN_CORRUPT); return CLUSTER_BUFMGR_GCS_DROP_DROPPED; + } /* * PGRAC: GCS serve-stall round-5 (A2) — bounded drop. A pinner racing @@ -10624,7 +11810,11 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode * its true PCM state. */ cluster_bufmgr_in_gcs_drop = true; /* gates the drop-prepin inject */ - if (!InvalidateBufferTry(buf)) /* releases the header spinlock */ + invalidate_succeeded = InvalidateBufferTry(buf); /* releases the header spinlock */ + if (activation_diag_valid) + cluster_pcm_own_activation_diag_emit("invalidate-stage-n-drop", buf->buf_id, + &activation_diag, CLUSTER_PCM_OWN_CORRUPT); + if (!invalidate_succeeded) { cluster_bufmgr_in_gcs_drop = false; @@ -10968,6 +12158,7 @@ ClusterBufmgrGcsDropResult cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn, XLogRecPtr *out_page_lsn) { + ClusterPcmOwnSnapshot activation_diag; uint32 hashcode; LWLock *partition_lock; int buf_id; @@ -10977,6 +12168,8 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn bool was_dirty = false; uint8 saved_pcm_state; uint64 staged_gen; /* PGRAC W2: ownership gen captured at stage-N */ + bool activation_diag_valid = false; + bool invalidate_succeeded; if (out_page_lsn != NULL) *out_page_lsn = InvalidXLogRecPtr; @@ -11096,18 +12289,28 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn saved_pcm_state = buf->pcm_state; staged_gen = cluster_pcm_own_gen_get(buf->buf_id); /* PGRAC W2 */ buf->pcm_state = (uint8) PCM_STATE_N; + cluster_pcm_own_snapshot_locked(buf, &activation_diag); + activation_diag_valid = activation_diag.writer_activation_token != 0; /* * PGRAC: spec-6.12h D-h1 — keep a Past Image instead of dropping (the * shipped current went to the new holder; our copy becomes the PI). */ - if (cluster_bufmgr_convert_to_pi_locked(buf, buf_state)) + if (cluster_bufmgr_convert_to_pi_locked(buf, buf_state)) { + if (activation_diag_valid) + cluster_pcm_own_activation_diag_emit("drop-no-wire-stage-n-pi", buf->buf_id, + &activation_diag, CLUSTER_PCM_OWN_CORRUPT); return CLUSTER_BUFMGR_GCS_DROP_DROPPED; + } /* PGRAC: GCS serve-stall round-5 (A2) — bounded drop; restore the * residency mode on a raced pin (mirrors the invalidate wrapper). */ cluster_bufmgr_in_gcs_drop = true; /* gates the drop-prepin inject */ - if (!InvalidateBufferTry(buf)) /* releases the header spinlock */ + invalidate_succeeded = InvalidateBufferTry(buf); /* releases the header spinlock */ + if (activation_diag_valid) + cluster_pcm_own_activation_diag_emit("drop-no-wire-stage-n-drop", buf->buf_id, + &activation_diag, CLUSTER_PCM_OWN_CORRUPT); + if (!invalidate_succeeded) { cluster_bufmgr_in_gcs_drop = false; @@ -11215,6 +12418,7 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, uint64 reservation_token = 0; uint32 buf_state; volatile bool content_locked = false; + volatile bool private_pinned = false; if (out_revoking != NULL) memset(out_revoking, 0, sizeof(*out_revoking)); @@ -11254,14 +12458,33 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, * and report BUSY: one flush converges the state and the image pump * retries against a clean page. */ + content_lock = BufferDescriptorGetContentLock(buf); PinBuffer_Locked(buf); /* consumes the buffer header lock */ - LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED); - FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); - LWLockRelease(BufferDescriptorGetContentLock(buf)); - UnpinBuffer(buf); + private_pinned = true; + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) { + UnpinBuffer(buf); + return CLUSTER_PCM_OWN_BUSY; + } + content_locked = true; + PG_TRY(); + { + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + LWLockRelease(content_lock); + content_locked = false; + UnpinBuffer(buf); + private_pinned = false; + } + PG_CATCH(); + { + if (content_locked && LWLockHeldByMe(content_lock)) + LWLockRelease(content_lock); + if (private_pinned) + UnpinBuffer(buf); + PG_RE_THROW(); + } + PG_END_TRY(); return CLUSTER_PCM_OWN_BUSY; - } - else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + } else if ((buf_state & BM_IO_IN_PROGRESS) != 0) result = CLUSTER_PCM_OWN_BUSY; else { result = cluster_pcm_own_reservation_begin_exact(buf->buf_id, expected_n->generation, @@ -11284,8 +12507,7 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, result = CLUSTER_PCM_OWN_CORRUPT; } - if (result == CLUSTER_PCM_OWN_OK) { - LWLockAcquire(content_lock, LW_EXCLUSIVE); + if (result == CLUSTER_PCM_OWN_OK && LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) { content_locked = true; buf_state = LockBufHdr(buf); if (!cluster_pcm_own_snapshot_matches_locked(buf, &live) @@ -11315,7 +12537,8 @@ cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, UnlockBufHdr(buf, buf_state); LWLockRelease(content_lock); content_locked = false; - } + } else if (result == CLUSTER_PCM_OWN_OK) + result = CLUSTER_PCM_OWN_BUSY; } PG_CATCH(); { @@ -11408,6 +12631,403 @@ cluster_bufmgr_pcm_own_begin_s_revoke(BufferDesc *buf, return result; } + +typedef enum ClusterPcmOwnSSourceHardFailureReason +{ + CLUSTER_PCM_S_SOURCE_HARD_NONE = 0, + CLUSTER_PCM_S_SOURCE_HARD_INITIAL_CURRENT_IMAGE, + CLUSTER_PCM_S_SOURCE_HARD_INITIAL_IO_ERROR, + CLUSTER_PCM_S_SOURCE_HARD_BEGIN_REVOKE_CORRUPT, + CLUSTER_PCM_S_SOURCE_HARD_STORAGE_READ_ERROR, + CLUSTER_PCM_S_SOURCE_HARD_STORAGE_VERIFY, + CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_CURRENT_IMAGE, + CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_IO_ERROR, + CLUSTER_PCM_S_SOURCE_HARD_NO_COVER, + CLUSTER_PCM_S_SOURCE_HARD_ABORT_FAILURE +} ClusterPcmOwnSSourceHardFailureReason; + +typedef struct ClusterPcmOwnSSourceHardFailureObservation +{ + bool valid; + ClusterPcmOwnSSourceHardFailureReason reason; + ClusterPcmOwnSSourceHardFailureReason cause; + BufferTag tag; + int buffer_id; + uint8 pcm_state; + uint8 buffer_type; + uint32 flags; + uint32 buffer_state; + uint64 generation; + uint64 reservation_token; + uint64 required_scn; + uint64 local_scn; + uint64 storage_scn; + int result; + int abort_result; +} ClusterPcmOwnSSourceHardFailureObservation; + +static const char * +cluster_bufmgr_pcm_own_s_source_hard_failure_reason_name( + ClusterPcmOwnSSourceHardFailureReason reason) +{ + switch (reason) + { + case CLUSTER_PCM_S_SOURCE_HARD_NONE: + return "none"; + case CLUSTER_PCM_S_SOURCE_HARD_INITIAL_CURRENT_IMAGE: + return "initial-current-image"; + case CLUSTER_PCM_S_SOURCE_HARD_INITIAL_IO_ERROR: + return "initial-io-error"; + case CLUSTER_PCM_S_SOURCE_HARD_BEGIN_REVOKE_CORRUPT: + return "begin-revoke-corrupt"; + case CLUSTER_PCM_S_SOURCE_HARD_STORAGE_READ_ERROR: + return "storage-read-error"; + case CLUSTER_PCM_S_SOURCE_HARD_STORAGE_VERIFY: + return "storage-verify"; + case CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_CURRENT_IMAGE: + return "post-lock-current-image"; + case CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_IO_ERROR: + return "post-lock-io-error"; + case CLUSTER_PCM_S_SOURCE_HARD_NO_COVER: + return "no-cover"; + case CLUSTER_PCM_S_SOURCE_HARD_ABORT_FAILURE: + return "abort-failure"; + } + return "unknown"; +} + +/* One record for the first exact hard-failure shape and every later state + * change. The small per-process cache prevents a retrying LMS worker from + * flooding the server log without hiding a different ownership tuple or SCN + * comparison. This is observation only: no ownership or buffer state is + * read or changed here beyond the caller's already coherent samples. */ +static void +cluster_bufmgr_pcm_own_observe_s_source_hard_failure( + ClusterPcmOwnSSourceHardFailureReason reason, + ClusterPcmOwnSSourceHardFailureReason cause, BufferDesc *buf, + const ClusterPcmOwnSnapshot *identity, SCN required_scn, SCN local_scn, + SCN storage_scn, uint32 buffer_state, uint8 buffer_type, + ClusterPcmOwnResult result, ClusterPcmOwnResult abort_result) +{ + static ClusterPcmOwnSSourceHardFailureObservation cache[8]; + static uint32 next_cache_slot = 0; + ClusterPcmOwnSSourceHardFailureObservation obs; + int cache_idx = -1; + int i; + + if (reason == CLUSTER_PCM_S_SOURCE_HARD_NONE || buf == NULL || identity == NULL) + return; + memset(&obs, 0, sizeof(obs)); + obs.valid = true; + obs.reason = reason; + obs.cause = cause; + obs.tag = identity->tag; + obs.buffer_id = buf->buf_id; + obs.pcm_state = identity->pcm_state; + obs.buffer_type = buffer_type; + obs.flags = identity->flags; + obs.buffer_state = buffer_state; + obs.generation = identity->generation; + obs.reservation_token = identity->reservation_token; + obs.required_scn = (uint64)required_scn; + obs.local_scn = (uint64)local_scn; + obs.storage_scn = (uint64)storage_scn; + obs.result = (int)result; + obs.abort_result = (int)abort_result; + + for (i = 0; i < lengthof(cache); i++) + { + if (cache[i].valid && cache[i].reason == obs.reason + && BufferTagsEqual(&cache[i].tag, &obs.tag)) + { + cache_idx = i; + break; + } + } + if (cache_idx >= 0 && memcmp(&cache[cache_idx], &obs, sizeof(obs)) == 0) + return; + if (cache_idx < 0) + { + cache_idx = (int)(next_cache_slot % lengthof(cache)); + next_cache_slot++; + } + cache[cache_idx] = obs; + + elog(LOG, + "cluster PCM S-source hard failure observation: " + "reason=%s cause=%s result=%d abort_result=%d " + "buffer=%d spc=%u db=%u rel=%u fork=%d blk=%u " + "state=%u generation=%llu token=%llu flags=0x%x " + "required_scn=%llu local_scn=%llu storage_scn=%llu " + "buffer_state=0x%x buffer_type=%u", + cluster_bufmgr_pcm_own_s_source_hard_failure_reason_name(obs.reason), + cluster_bufmgr_pcm_own_s_source_hard_failure_reason_name(obs.cause), + obs.result, obs.abort_result, obs.buffer_id, obs.tag.spcOid, obs.tag.dbOid, + obs.tag.relNumber, (int)obs.tag.forkNum, obs.tag.blockNum, obs.pcm_state, + (unsigned long long)obs.generation, + (unsigned long long)obs.reservation_token, obs.flags, + (unsigned long long)obs.required_scn, (unsigned long long)obs.local_scn, + (unsigned long long)obs.storage_scn, obs.buffer_state, + (unsigned int)obs.buffer_type); +} + +/* Freeze an exact S source and materialize the newest safe image visible in + * either its clean resident page or shared storage. REVOKING prevents local + * data writes/reuse while the conditional content-lock hold binds the final + * comparison and optional storage refresh to the same ownership tuple. */ +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_prepare_s_source_image( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_s, SCN required_page_scn, + ClusterPcmOwnSnapshot *out_revoking, char block_data[BLCKSZ], + XLogRecPtr *out_page_lsn, uint64 *out_page_scn, + ClusterPcmOwnSourcePrepareRefusal *out_refusal) +{ + PGIOAlignedBlock scratch; + ClusterPcmOwnResult abort_result; + ClusterPcmOwnResult result; + ClusterPcmOwnSnapshot live; + SMgrRelation reln; + LWLock *content_lock; + Page local_page; + Page storage_page; + SCN local_scn; + SCN storage_scn; + uint32 buf_state; + uint32 observed_buf_state = 0; + uint8 observed_buffer_type = 0; + bool local_covers; + bool storage_covers; + volatile bool content_locked = false; + volatile bool private_pinned = false; + ClusterPcmOwnSSourceHardFailureReason hard_failure_reason = + CLUSTER_PCM_S_SOURCE_HARD_NONE; + + if (out_revoking != NULL) + memset(out_revoking, 0, sizeof(*out_revoking)); + if (block_data != NULL) + memset(block_data, 0, BLCKSZ); + if (out_page_lsn != NULL) + *out_page_lsn = InvalidXLogRecPtr; + if (out_page_scn != NULL) + *out_page_scn = 0; + if (out_refusal != NULL) + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_NONE; + if (buf == NULL || expected_s == NULL || out_revoking == NULL || block_data == NULL + || out_page_lsn == NULL || out_page_scn == NULL || out_refusal == NULL) + return CLUSTER_PCM_OWN_INVALID; + if (expected_s->pcm_state != (uint8)PCM_STATE_S || expected_s->flags != 0) + return CLUSTER_PCM_OWN_INVALID; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + + /* A read can install a commit hint after the checkpoint that made this S + * image storage-consistent. Flush that legal dirt before REVOKING and + * return BUSY after making progress; otherwise the pump would reserve, + * reject the dirt, abort, and repeat forever. This is the same WAL-first + * convergence used by requester-as-source N. */ + ReservePrivateRefCountEntry(); + ResourceOwnerEnlargeBuffers(CurrentResourceOwner); + buf_state = LockBufHdr(buf); + observed_buf_state = buf_state; + observed_buffer_type = buf->buffer_type; + if (!cluster_pcm_own_snapshot_matches_locked(buf, expected_s) || (buf_state & BM_VALID) == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + { + hard_failure_reason = CLUSTER_PCM_S_SOURCE_HARD_INITIAL_CURRENT_IMAGE; + result = CLUSTER_PCM_OWN_CORRUPT; + } + else if ((buf_state & BM_IO_ERROR) != 0) + { + hard_failure_reason = CLUSTER_PCM_S_SOURCE_HARD_INITIAL_IO_ERROR; + result = CLUSTER_PCM_OWN_CORRUPT; + } + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) + { + content_lock = BufferDescriptorGetContentLock(buf); + PinBuffer_Locked(buf); /* consumes the buffer header lock */ + private_pinned = true; + if (!LWLockConditionalAcquire(content_lock, LW_SHARED)) { + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_CONTENT_LOCK; + UnpinBuffer(buf); + return CLUSTER_PCM_OWN_BUSY; + } + content_locked = true; + PG_TRY(); + { + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_DIRTY_FLUSHED; + LWLockRelease(content_lock); + content_locked = false; + UnpinBuffer(buf); + private_pinned = false; + } + PG_CATCH(); + { + if (content_locked && LWLockHeldByMe(content_lock)) + LWLockRelease(content_lock); + if (private_pinned) + UnpinBuffer(buf); + PG_RE_THROW(); + } + PG_END_TRY(); + return CLUSTER_PCM_OWN_BUSY; + } + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) { + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_IO_IN_PROGRESS; + result = CLUSTER_PCM_OWN_BUSY; + } + else + result = CLUSTER_PCM_OWN_OK; + UnlockBufHdr(buf, buf_state); + if (result != CLUSTER_PCM_OWN_OK) { + if (result == CLUSTER_PCM_OWN_CORRUPT) + cluster_bufmgr_pcm_own_observe_s_source_hard_failure( + hard_failure_reason, CLUSTER_PCM_S_SOURCE_HARD_NONE, buf, expected_s, + required_page_scn, InvalidScn, InvalidScn, observed_buf_state, + observed_buffer_type, result, CLUSTER_PCM_OWN_INVALID); + return result; + } + + result = cluster_bufmgr_pcm_own_begin_s_revoke(buf, expected_s, &live); + if (result != CLUSTER_PCM_OWN_OK) { + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_BEGIN_REVOKE; + if (result == CLUSTER_PCM_OWN_CORRUPT) + cluster_bufmgr_pcm_own_observe_s_source_hard_failure( + CLUSTER_PCM_S_SOURCE_HARD_BEGIN_REVOKE_CORRUPT, + CLUSTER_PCM_S_SOURCE_HARD_NONE, buf, expected_s, required_page_scn, + InvalidScn, InvalidScn, observed_buf_state, observed_buffer_type, result, + CLUSTER_PCM_OWN_INVALID); + return result; + } + content_lock = BufferDescriptorGetContentLock(buf); + storage_page = (Page)scratch.data; + local_scn = InvalidScn; + storage_scn = InvalidScn; + + PG_TRY(); + { + reln = smgropen(BufTagGetRelFileLocator(&expected_s->tag), InvalidBackendId); + smgrread(reln, BufTagGetForkNum(&expected_s->tag), expected_s->tag.blockNum, + scratch.data); + if (!PageIsVerifiedExtended(storage_page, expected_s->tag.blockNum, + PIV_LOG_WARNING | PIV_REPORT_STAT)) { + hard_failure_reason = CLUSTER_PCM_S_SOURCE_HARD_STORAGE_VERIFY; + result = CLUSTER_PCM_OWN_CORRUPT; + } + if (result == CLUSTER_PCM_OWN_OK + && LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) { + content_locked = true; + buf_state = LockBufHdr(buf); + observed_buf_state = buf_state; + observed_buffer_type = buf->buffer_type; + if (!cluster_pcm_own_snapshot_matches_locked(buf, &live) + || live.flags != PCM_OWN_FLAG_REVOKING + || live.pcm_state != (uint8)PCM_STATE_S || (buf_state & BM_VALID) == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { + hard_failure_reason = CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_CURRENT_IMAGE; + result = CLUSTER_PCM_OWN_CORRUPT; + } + else if ((buf_state & BM_IO_ERROR) != 0) { + hard_failure_reason = CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_IO_ERROR; + result = CLUSTER_PCM_OWN_CORRUPT; + } + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) { + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_DIRTY_RACED; + result = CLUSTER_PCM_OWN_BUSY; + } + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) { + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_IO_IN_PROGRESS; + result = CLUSTER_PCM_OWN_BUSY; + } + else { + local_page = (Page)BufHdrGetBlock(buf); + local_scn = ((PageHeader)local_page)->pd_block_scn; + storage_scn = ((PageHeader)storage_page)->pd_block_scn; + local_covers + = !SCN_VALID(required_page_scn) + || (SCN_VALID(local_scn) + && scn_local(local_scn) >= scn_local(required_page_scn)); + storage_covers + = !SCN_VALID(required_page_scn) + || (SCN_VALID(storage_scn) + && scn_local(storage_scn) >= scn_local(required_page_scn)); + /* Every S grant is storage-consistent before publication. Once an + * exact non-source ACK has published a higher floor, neither copy + * covering it is durable lost-write evidence, not retryable lag. */ + if (!local_covers && !storage_covers) { + hard_failure_reason = CLUSTER_PCM_S_SOURCE_HARD_NO_COVER; + result = CLUSTER_PCM_OWN_CORRUPT; + } + else if (local_covers + && (!storage_covers || !SCN_VALID(storage_scn) + || (SCN_VALID(local_scn) + && scn_local(local_scn) >= scn_local(storage_scn)))) { + memcpy(block_data, local_page, BLCKSZ); + *out_page_lsn = PageGetLSN(local_page); + *out_page_scn = (uint64)local_scn; + *out_revoking = live; + } else { + memcpy((char *)BufHdrGetBlock(buf), scratch.data, BLCKSZ); + buf->buffer_type = (uint8)BUF_TYPE_CURRENT; + memcpy(block_data, scratch.data, BLCKSZ); + *out_page_lsn = PageGetLSN(storage_page); + *out_page_scn = (uint64)storage_scn; + *out_revoking = live; + } + } + UnlockBufHdr(buf, buf_state); + LWLockRelease(content_lock); + content_locked = false; + } else if (result == CLUSTER_PCM_OWN_OK) { + *out_refusal = CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_CONTENT_LOCK; + result = CLUSTER_PCM_OWN_BUSY; + } + } + PG_CATCH(); + { + if (content_locked && LWLockHeldByMe(content_lock)) + LWLockRelease(content_lock); + abort_result = cluster_bufmgr_pcm_own_abort_s_revoke(buf, &live); + if (abort_result != CLUSTER_PCM_OWN_OK) { + cluster_bufmgr_pcm_own_observe_s_source_hard_failure( + CLUSTER_PCM_S_SOURCE_HARD_ABORT_FAILURE, + CLUSTER_PCM_S_SOURCE_HARD_STORAGE_READ_ERROR, buf, &live, + required_page_scn, local_scn, storage_scn, observed_buf_state, + observed_buffer_type, CLUSTER_PCM_OWN_CORRUPT, abort_result); + elog(LOG, + "could not abort exact PCM-X S-source reservation after storage read error: " + "buffer=%d generation=%llu token=%llu result=%d; evidence retained", + buf->buf_id, (unsigned long long)live.generation, + (unsigned long long)live.reservation_token, (int)abort_result); + } + PG_RE_THROW(); + } + PG_END_TRY(); + + if (result == CLUSTER_PCM_OWN_OK) + return result; + abort_result = cluster_bufmgr_pcm_own_abort_s_revoke(buf, &live); + if (abort_result != CLUSTER_PCM_OWN_OK) { + cluster_bufmgr_pcm_own_observe_s_source_hard_failure( + CLUSTER_PCM_S_SOURCE_HARD_ABORT_FAILURE, hard_failure_reason, buf, &live, + required_page_scn, local_scn, storage_scn, observed_buf_state, + observed_buffer_type, result, abort_result); + return CLUSTER_PCM_OWN_CORRUPT; + } + if (result == CLUSTER_PCM_OWN_CORRUPT) + cluster_bufmgr_pcm_own_observe_s_source_hard_failure( + hard_failure_reason, CLUSTER_PCM_S_SOURCE_HARD_NONE, buf, &live, + required_page_scn, local_scn, storage_scn, observed_buf_state, + observed_buffer_type, result, abort_result); + memset(out_revoking, 0, sizeof(*out_revoking)); + memset(block_data, 0, BLCKSZ); + *out_page_lsn = InvalidXLogRecPtr; + *out_page_scn = 0; + return result; +} + /* Abort only the matching S-source staging reservation. */ ClusterPcmOwnResult cluster_bufmgr_pcm_own_abort_s_revoke(BufferDesc *buf, @@ -11576,7 +13196,8 @@ static ClusterPcmOwnResult cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, XLogRecPtr expected_lsn, - ClusterPcmOwnSnapshot *out_finished) + ClusterPcmOwnSnapshot *out_finished, + ClusterPcmOwnFinishRefusal *out_refusal) { ClusterPcmXRevokeFinishMode finish_mode; ClusterPcmOwnResult live_result; @@ -11587,6 +13208,7 @@ cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(BufferDesc *buf, uint64 live_token; uint32 hashcode; uint32 flags; + uint32 shared_refcount; uint32 buf_state; int mapped_buf_id; @@ -11600,33 +13222,51 @@ cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(BufferDesc *buf, } buf_state = LockBufHdr(buf); - finish_mode = cluster_pcm_x_revoke_finish_mode(&tag, BUF_STATE_GET_REFCOUNT(buf_state)); + shared_refcount = BUF_STATE_GET_REFCOUNT(buf_state); + finish_mode = cluster_pcm_x_revoke_finish_mode(&tag, shared_refcount); if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0 || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation || buf->pcm_state != expected_revoking->pcm_state) result = CLUSTER_PCM_OWN_STALE; else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) result = CLUSTER_PCM_OWN_CORRUPT; - else if (finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_BUSY) - result = CLUSTER_PCM_OWN_BUSY; - else if (finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_DROP) - result = CLUSTER_PCM_OWN_CORRUPT; - else if ((buf_state & BM_IO_IN_PROGRESS) != 0) - result = CLUSTER_PCM_OWN_BUSY; else { live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); - live_result = cluster_pcm_own_classify_live_flags(flags, live_token); - if (live_result == CLUSTER_PCM_OWN_CORRUPT) - result = live_result; - else if (flags == 0) - result = CLUSTER_PCM_OWN_STALE; - else if (flags != PCM_OWN_FLAG_REVOKING) + if (out_refusal != NULL) { + out_refusal->shared_refcount = shared_refcount; + out_refusal->bm_io_in_progress = (buf_state & BM_IO_IN_PROGRESS) != 0; + out_refusal->live_flags = flags; + out_refusal->live_token = live_token; + } + if (finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_BUSY) { + if (out_refusal != NULL) + out_refusal->reason = CLUSTER_PCM_OWN_FINISH_REFUSAL_VM_FSM_PINNED; result = CLUSTER_PCM_OWN_BUSY; - else if (live_token != expected_revoking->reservation_token) - result = CLUSTER_PCM_OWN_STALE; - else if (PageGetLSN((Page)BufHdrGetBlock(buf)) != expected_lsn) - result = CLUSTER_PCM_OWN_STALE; + } + else if (finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_DROP) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) { + if (out_refusal != NULL) + out_refusal->reason = CLUSTER_PCM_OWN_FINISH_REFUSAL_IO_IN_PROGRESS; + result = CLUSTER_PCM_OWN_BUSY; + } + else { + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if (flags == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (flags != PCM_OWN_FLAG_REVOKING) { + if (out_refusal != NULL) + out_refusal->reason = CLUSTER_PCM_OWN_FINISH_REFUSAL_LIVE_FLAGS; + result = CLUSTER_PCM_OWN_BUSY; + } + else if (live_token != expected_revoking->reservation_token) + result = CLUSTER_PCM_OWN_STALE; + else if (PageGetLSN((Page)BufHdrGetBlock(buf)) != expected_lsn) + result = CLUSTER_PCM_OWN_STALE; + } } if (result == CLUSTER_PCM_OWN_OK) @@ -11652,11 +13292,11 @@ cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(BufferDesc *buf, /* * Commit one staged S/X source as a retained image, except for VM/FSM. * - * REVOKING already binds the descriptor to its tag through D5a and the victim - * guard, so this path deliberately does not nest a mapping lock with the - * content lock. Content EXCLUSIVE is the byte/I/O linearization: a flush - * that already owns SHARE completes first; once this lock is acquired, no - * writer or output I/O can overlap the exact LSN check and transition. + * Mapping authority binds the tag to the descriptor while this DATA worker + * takes its own raw pin. The pin then closes descriptor-retag ABA after the + * mapping lock is released. Content EXCLUSIVE is the byte/I/O linearization: + * a flush that already owns SHARE completes first; once this lock is acquired, + * no writer or output I/O can overlap the exact LSN check and transition. * Passive pins remain untouched. The descriptor keeps tag, BM_VALID, * refcount, and bytes, while PI+N marks those bytes non-current and the exact * REVOKING token prevents reuse until DRAIN. @@ -11664,22 +13304,32 @@ cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(BufferDesc *buf, ClusterPcmOwnResult cluster_bufmgr_pcm_own_finish_revoke_retain( BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, - XLogRecPtr expected_lsn, ClusterPcmOwnSnapshot *out_retained) + XLogRecPtr expected_lsn, ClusterPcmOwnSnapshot *out_retained, + ClusterPcmOwnFinishRefusal *out_refusal) { ClusterPcmOwnResult live_result; ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; BufferTag tag; LWLock *content_lock; + LWLock *partition_lock; uint64 committed_generation; uint64 live_token; + uint32 hashcode; uint32 flags; uint32 buf_state; + int mapped_buf_id; bool source_is_s; bool source_is_x; + bool needs_flush = false; + bool forced_test_flush = false; + volatile bool content_locked = false; + volatile bool caller_pinned = false; ClusterPcmXRevokeFinishMode finish_mode; if (out_retained != NULL) memset(out_retained, 0, sizeof(*out_retained)); + if (out_refusal != NULL) + memset(out_refusal, 0, sizeof(*out_refusal)); if (buf == NULL || expected_revoking == NULL || out_retained == NULL) return CLUSTER_PCM_OWN_INVALID; source_is_s = expected_revoking->pcm_state == (uint8) PCM_STATE_S; @@ -11697,55 +13347,180 @@ cluster_bufmgr_pcm_own_finish_revoke_retain( return CLUSTER_PCM_OWN_INVALID; if (finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_DROP) return cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(buf, expected_revoking, - expected_lsn, out_retained); + expected_lsn, out_retained, out_refusal); + if (finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_RETAIN) + return CLUSTER_PCM_OWN_INVALID; Assert(finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_RETAIN); - content_lock = BufferDescriptorGetContentLock(buf); - LWLockAcquire(content_lock, LW_EXCLUSIVE); + /* + * Bind tag -> descriptor under mapping authority, then add this DATA + * worker's own raw pin before dropping that authority. The pin, not a + * passive backend's incidental refcount, is the FlushBuffer lifetime + * contract and closes descriptor-retag ABA through the final commit. + */ + hashcode = BufTableHashCode(&tag); + partition_lock = BufMappingPartitionLock(hashcode); + LWLockAcquire(partition_lock, LW_SHARED); + mapped_buf_id = BufTableLookup(&tag, hashcode); + if (mapped_buf_id != buf->buf_id) { + LWLockRelease(partition_lock); + return CLUSTER_PCM_OWN_STALE; + } buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) - || (buf_state & BM_VALID) == 0 - || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation - || buf->pcm_state != expected_revoking->pcm_state) + if (!cluster_pcm_own_snapshot_matches_locked(buf, expected_revoking) + || (buf_state & BM_VALID) == 0) result = CLUSTER_PCM_OWN_STALE; else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) result = CLUSTER_PCM_OWN_CORRUPT; else if ((buf_state & BM_IO_IN_PROGRESS) != 0) result = CLUSTER_PCM_OWN_BUSY; - else + else if ((buf_state & BM_IO_ERROR) != 0) + result = CLUSTER_PCM_OWN_CORRUPT; + else { + cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); + caller_pinned = true; + buf_state = 0; + } + if (buf_state != 0) + UnlockBufHdr(buf, buf_state); + LWLockRelease(partition_lock); + if (result != CLUSTER_PCM_OWN_OK) + return result; + + content_lock = BufferDescriptorGetContentLock(buf); + if (!LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) { + cluster_bufmgr_unpin_for_gcs(buf); + return CLUSTER_PCM_OWN_BUSY; + } + content_locked = true; + + PG_TRY(); { - live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); - flags = cluster_pcm_own_flags_get(buf->buf_id); - live_result = cluster_pcm_own_classify_live_flags(flags, live_token); - if (live_result == CLUSTER_PCM_OWN_CORRUPT) - result = live_result; - else if (flags == 0) + buf_state = LockBufHdr(buf); + if (!cluster_pcm_own_snapshot_matches_locked(buf, expected_revoking) + || (buf_state & BM_VALID) == 0) result = CLUSTER_PCM_OWN_STALE; - else if (flags != PCM_OWN_FLAG_REVOKING) + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) result = CLUSTER_PCM_OWN_BUSY; - else if (live_token != expected_revoking->reservation_token) - result = CLUSTER_PCM_OWN_STALE; - else if (PageGetLSN((Page) BufHdrGetBlock(buf)) != expected_lsn) + else if ((buf_state & BM_IO_ERROR) != 0) + result = CLUSTER_PCM_OWN_CORRUPT; + else if (PageGetLSN((Page)BufHdrGetBlock(buf)) != expected_lsn) result = CLUSTER_PCM_OWN_STALE; - } + else { + /* + * The copy/materialize leg normally flushes a dirty source before + * finish. Test arming dirties the unchanged exact image under the + * caller pin + content EXCLUSIVE so the finish FlushBuffer boundary + * is deterministic, without fabricating different page bytes. + */ + forced_test_flush = cluster_injection_is_armed("cluster-pcm-x-retain-flush-error"); + if (forced_test_flush) + buf_state |= BM_DIRTY | BM_JUST_DIRTIED; + needs_flush = (buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0; + } + UnlockBufHdr(buf, buf_state); - if (result == CLUSTER_PCM_OWN_OK) + if (result == CLUSTER_PCM_OWN_OK) { + /* + * A hint-bit update may legally dirty the page under the shared + * content lock after the immutable A-record copy. EXCLUSIVE freezes + * that image; the caller-owned pin covers FlushBuffer and the exact + * ownership commit that follows it. + */ + if (needs_flush && !caller_pinned) + result = CLUSTER_PCM_OWN_CORRUPT; + else if (needs_flush) { + cluster_pcm_x_finish_retain_flush_active = true; + FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); + cluster_pcm_x_finish_retain_flush_active = false; + if (forced_test_flush) + elog(LOG, + "cluster PCM-X retained-image finish FlushBuffer succeeded: " + "buffer=%d rel=%u fork=%d blk=%u", + buf->buf_id, tag.relNumber, (int)tag.forkNum, tag.blockNum); + } + + if (result == CLUSTER_PCM_OWN_OK) { + buf_state = LockBufHdr(buf); + if (!cluster_pcm_own_snapshot_matches_locked(buf, expected_revoking) + || (buf_state & BM_VALID) == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + result = CLUSTER_PCM_OWN_BUSY; + else if ((buf_state & BM_IO_ERROR) != 0) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED)) != 0) + result = CLUSTER_PCM_OWN_BUSY; + else if (PageGetLSN((Page)BufHdrGetBlock(buf)) != expected_lsn) + result = CLUSTER_PCM_OWN_STALE; + else { + live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if (flags != PCM_OWN_FLAG_REVOKING) + result = flags == 0 ? CLUSTER_PCM_OWN_STALE : CLUSTER_PCM_OWN_BUSY; + else if (live_token != expected_revoking->reservation_token) + result = CLUSTER_PCM_OWN_STALE; + } + + if (result == CLUSTER_PCM_OWN_OK) + result = cluster_pcm_own_revoke_retain_commit_exact( + buf->buf_id, expected_revoking->generation, + expected_revoking->reservation_token, &committed_generation); + if (result == CLUSTER_PCM_OWN_OK) { + buf->pcm_state = (uint8)PCM_STATE_N; + buf->buffer_type = (uint8)BUF_TYPE_PI; + buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR); + cluster_pcm_own_snapshot_locked(buf, out_retained); + } + UnlockBufHdr(buf, buf_state); + } + } + + LWLockRelease(content_lock); + content_locked = false; + } + PG_CATCH(); { - result = cluster_pcm_own_revoke_retain_commit_exact( - buf->buf_id, expected_revoking->generation, - expected_revoking->reservation_token, &committed_generation); - if (result == CLUSTER_PCM_OWN_OK) + /* elog(ERROR) resets InterruptHoldoffCount before longjmp, but a + * still-held LWLock retains its ownership entry. Recreate that lock's + * interrupt hold only on the exact release path; LWLockRelease consumes + * the replacement hold itself. */ + cluster_pcm_x_finish_retain_flush_active = false; + /* FlushBuffer normally leaves BufferIO cleanup to transaction abort. + * This DATA worker intentionally absorbs the injected/physical write + * ERROR after preserving the immutable revoke record, so restore its + * stack-local error callback and terminate its exact ResourceOwner IO + * before dropping the caller-owned pin. */ + if (cluster_pcm_x_finish_retain_flush_error_context_pushed) { - buf->pcm_state = (uint8) PCM_STATE_N; - buf->buffer_type = (uint8) BUF_TYPE_PI; - buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED | - BM_CHECKPOINT_NEEDED | BM_IO_ERROR); - cluster_pcm_own_snapshot_locked(buf, out_retained); + error_context_stack = cluster_pcm_x_finish_retain_flush_error_context_previous; + cluster_pcm_x_finish_retain_flush_error_context_pushed = false; + cluster_pcm_x_finish_retain_flush_error_context_previous = NULL; } + if (content_locked && LWLockHeldByMe(content_lock)) { + HOLD_INTERRUPTS(); + LWLockRelease(content_lock); + } + if (cluster_pcm_x_finish_retain_flush_io_active) + { + cluster_pcm_x_finish_retain_flush_io_active = false; + AbortBufferIO(BufferDescriptorGetBuffer(buf)); + } + if (caller_pinned) + cluster_bufmgr_unpin_for_gcs(buf); + PG_RE_THROW(); } + PG_END_TRY(); - UnlockBufHdr(buf, buf_state); - LWLockRelease(content_lock); + if (caller_pinned) + cluster_bufmgr_unpin_for_gcs(buf); return result; } @@ -11769,6 +13544,10 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, uint32 flags; uint32 buf_state; int buf_id; + /* Release evidence: sampled under the header lock, logged after. */ + bool fx_log = false; + bool fx_dropped = false; + uint32 fx_refcount = 0; if (tag == NULL || source_generation == UINT64_MAX) return CLUSTER_PCM_OWN_INVALID; @@ -11821,7 +13600,8 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, return result; content_lock = BufferDescriptorGetContentLock(buf); - LWLockAcquire(content_lock, LW_EXCLUSIVE); + if (!LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)) + return CLUSTER_PCM_OWN_BUSY; buf_state = LockBufHdr(buf); if (!BufferTagsEqual(&buf->tag, tag) @@ -11851,32 +13631,50 @@ cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, buf->buf_id, committed_generation, retained_token); if (result == CLUSTER_PCM_OWN_OK) { /* - * The exact token is the last stale-write fence. Release it - * before making the retained PI reloadable, while content - * EXCLUSIVE and the header lock keep page bytes invalid. - * Keep the mapping so any pre-existing pin can leave - * normally; the next ordinary read reloads the current page - * image. + * The exact token was the last stale-write fence; release it + * first, then retag under the same header-lock hold. With no + * pins the image is dropped so the next ordinary read reloads + * the current page bytes. With a pre-existing pin the frozen + * PI+BM_VALID N mirror is kept byte-exact instead: clearing + * BM_VALID under a pin holder would let a later LockBuffer + * open a legacy master reservation over an invalid base and + * stamp S/X ownership onto it (the deterministic S_NEW finish + * refusals), while reloading in place would swap the page + * image under the pin (both violate the pin contract). See + * cluster_pcm_x_retained_release_retag. */ - buf_state &= ~BM_VALID; - buf->buffer_type = (uint8)BUF_TYPE_CURRENT; + fx_dropped = cluster_pcm_x_retained_release_retag(&buf->buffer_type, + &buf_state); + fx_log = true; + fx_refcount = BUF_STATE_GET_REFCOUNT(buf_state); } } } UnlockBufHdr(buf, buf_state); LWLockRelease(content_lock); + if (fx_log) + elog(LOG, + "cluster PCM retained image released: image=%s buffer=%d rel=%u fork=%d blk=%u " + "refcount=%u gen=%llu token=%llu", + fx_dropped ? "dropped" : "kept-pinned-pi", buf->buf_id, tag->relNumber, + (int)tag->forkNum, tag->blockNum, fx_refcount, + (unsigned long long)committed_generation, (unsigned long long)retained_token); return result; } -/* Prove that a sole-requester source image was adopted by the exact S->X - * handoff. This is intentionally read-only: a delayed DRAIN may release its - * immutable A-record, but it must never mutate the current X descriptor. */ +/* Corruption-only probe under a delayed self-source DRAIN. The release + * authority is the exact protocol completion certificate captured by the + * DRAIN consumption; the live descriptor is NOT that authority, because its + * per-buf_id generation is no BufferTag lineage - the tag may legitimately + * have moved on (X->S downgrade for a reader, a later lifecycle) or been + * evicted and reloaded on another descriptor. This probe is intentionally + * read-only and only reports a structurally malformed flags/token shape. */ ClusterPcmOwnResult -cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, uint64 source_generation) +cluster_bufmgr_pcm_own_self_handoff_probe(const BufferTag *tag, + ClusterPcmOwnSelfHandoffSample *sample_out) { ClusterPcmOwnResult live_result; - ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; BufferDesc *buf; BufferTag lookup_tag; LWLock *partition_lock; @@ -11886,7 +13684,9 @@ cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, uint64 source_ uint32 buf_state; int buf_id; - if (tag == NULL || source_generation == UINT64_MAX) + if (sample_out != NULL) + memset(sample_out, 0, sizeof(*sample_out)); + if (tag == NULL) return CLUSTER_PCM_OWN_INVALID; if (ClusterPcmOwnArray == NULL) return CLUSTER_PCM_OWN_NOT_READY; @@ -11897,26 +13697,26 @@ cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, uint64 source_ buf_id = BufTableLookup(&lookup_tag, hashcode); if (buf_id < 0) { LWLockRelease(partition_lock); - return CLUSTER_PCM_OWN_STALE; + return CLUSTER_PCM_OWN_OK; } buf = GetBufferDescriptor(buf_id); buf_state = LockBufHdr(buf); live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); flags = cluster_pcm_own_flags_get(buf->buf_id); live_result = cluster_pcm_own_classify_live_flags(flags, live_token); - if (!BufferTagsEqual(&buf->tag, tag) - || cluster_pcm_own_gen_get(buf->buf_id) != source_generation + 1) - result = CLUSTER_PCM_OWN_STALE; - else if (live_result == CLUSTER_PCM_OWN_CORRUPT) - result = live_result; - else if (flags != 0) - result = CLUSTER_PCM_OWN_BUSY; - else if (live_token == 0 || (buf_state & BM_VALID) == 0 || buf->pcm_state != (uint8)PCM_STATE_X - || buf->buffer_type != (uint8)BUF_TYPE_XCUR) - result = CLUSTER_PCM_OWN_STALE; + if (sample_out != NULL) { + /* Snapshot under the header lock only; the caller logs after unlock. */ + sample_out->live_generation = cluster_pcm_own_gen_get(buf->buf_id); + sample_out->live_token = live_token; + sample_out->own_flags = flags; + sample_out->pcm_state = buf->pcm_state; + sample_out->buffer_type = buf->buffer_type; + sample_out->bm_valid = (buf_state & BM_VALID) != 0; + sample_out->buffer_found = BufferTagsEqual(&buf->tag, tag); + } UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); - return result; + return live_result == CLUSTER_PCM_OWN_CORRUPT ? CLUSTER_PCM_OWN_CORRUPT : CLUSTER_PCM_OWN_OK; } /* ======================================================================== @@ -11971,11 +13771,13 @@ cluster_bufmgr_block_is_pi(BufferTag tag) bool cluster_bufmgr_discard_pi_block(BufferTag tag) { + ClusterPcmOwnSnapshot activation_diag; uint32 hash = BufTableHashCode(&tag); LWLock *partition_lock = BufMappingPartitionLock(hash); int buf_id; BufferDesc *buf; uint32 buf_state; + bool activation_diag_valid; LWLockAcquire(partition_lock, LW_SHARED); buf_id = BufTableLookup(&tag, hash); @@ -11999,6 +13801,8 @@ cluster_bufmgr_discard_pi_block(BufferTag tag) * but clear it again under the lock so InvalidateBuffer's eviction hook * can never see a stale mode and emit a release wire from LMON. */ buf->pcm_state = (uint8) PCM_STATE_N; + cluster_pcm_own_snapshot_locked(buf, &activation_diag); + activation_diag_valid = activation_diag.writer_activation_token != 0; /* PGRAC: spec-6.12h D-h3a — hygiene: drop the shadow stamp with the PI. * Correctness never depends on this clear (the D-h3 consumer * re-validates the PI shape + tag under this same header lock, and @@ -12006,6 +13810,9 @@ cluster_bufmgr_discard_pi_block(BufferTag tag) * slot should not linger as a plausible-looking stamp. */ cluster_pi_shadow_clear(buf->buf_id); InvalidateBuffer(buf); /* releases the header spinlock */ + if (activation_diag_valid) + cluster_pcm_own_activation_diag_emit("discard-pi-stage-n", buf->buf_id, + &activation_diag, CLUSTER_PCM_OWN_CORRUPT); return true; } diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h index c103997703..a969b31555 100644 --- a/src/include/access/visibilitymap.h +++ b/src/include/access/visibilitymap.h @@ -33,6 +33,8 @@ extern bool visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags); extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk, Buffer *vmbuf); +extern bool visibilitymap_pin_recent(Relation rel, BlockNumber heapBlk, + Buffer recent_buffer, Buffer *vmbuf); extern bool visibilitymap_pin_ok(BlockNumber heapBlk, Buffer vmbuf); extern void visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf, XLogRecPtr recptr, Buffer vmBuf, TransactionId cutoff_xid, diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index 08ceb43971..12ea0e7533 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -176,7 +176,8 @@ typedef enum GcsBlockPcmXRequesterSite { GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, GCS_BLOCK_PCM_X_RETRY_SITE_IMAGE_FETCH, GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_ARM, - GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_REPLAY_ARM + GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_REPLAY_ARM, + GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT } GcsBlockPcmXRequesterSite; typedef enum GcsBlockPcmXRetryAction { @@ -407,6 +408,15 @@ cluster_gcs_pcm_x_requester_retry_action(GcsBlockPcmXRequesterSite site, PcmXQue if (result == PCM_X_QUEUE_BAD_STATE) return GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS; break; + case GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT: + /* BUSY is a transient own-slot lifecycle flag (a revoke/grant on this + * node mid-flight); wait and re-snapshot. STALE means an interleaved + * revoke consumed the queued identity's base_own_generation, which the + * grant/final-ack chain cannot absorb without a master-visible rebase; + * keep the fail-closed verdict until that amendment lands. */ + if (result == PCM_X_QUEUE_BUSY) + return GCS_BLOCK_PCM_X_RETRY_WAIT; + break; case GCS_BLOCK_PCM_X_RETRY_SITE_WFG_CLEAR: case GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_ARM: break; @@ -504,12 +514,13 @@ typedef struct GcsBlockPcmXImageIdentity { typedef struct GcsBlockPcmXImageBinding { GcsBlockPcmXImageIdentity identity; uint64 master_session; + uint64 required_page_scn; } GcsBlockPcmXImageBinding; StaticAssertDecl(sizeof(GcsBlockPcmXImageIdentity) == 128, "PCM-X image identity must fit the existing dedup 128-byte metadata cell"); -StaticAssertDecl(sizeof(GcsBlockPcmXImageBinding) == 136, - "PCM-X image binding is identity plus authenticated master session"); +StaticAssertDecl(sizeof(GcsBlockPcmXImageBinding) == 144, + "PCM-X image binding includes session and required source floor"); static inline bool GcsBlockPcmXImageIdentityEqual(const GcsBlockPcmXImageIdentity *left, @@ -523,6 +534,7 @@ GcsBlockPcmXImageBindingEqual(const GcsBlockPcmXImageBinding *left, const GcsBlockPcmXImageBinding *right) { return left != NULL && right != NULL && left->master_session == right->master_session + && left->required_page_scn == right->required_page_scn && GcsBlockPcmXImageIdentityEqual(&left->identity, &right->identity); } @@ -939,9 +951,11 @@ cluster_gcs_pcm_x_revoke_ingress_valid(const PcmXRevokePayload *request, Size pa int32 authenticated_node, uint64 current_epoch, int32 tag_master, int32 local_node) { - return request != NULL && payload_length == sizeof(*request) && authenticated_node >= 0 - && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 - && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == authenticated_node + return request != NULL + && (payload_length == sizeof(*request) || payload_length == sizeof(PcmXRevokePayloadV2)) + && authenticated_node >= 0 && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT + && local_node >= 0 && local_node < PCM_X_PROTOCOL_NODE_LIMIT + && tag_master == authenticated_node && cluster_gcs_pcm_x_transfer_ref_wire_valid(&request->ref, current_epoch) && cluster_gcs_pcm_x_image_id_master_wire_valid(request->image_id, authenticated_node); } @@ -974,16 +988,27 @@ cluster_gcs_pcm_x_prepare_grant_ingress_valid(const PcmXGrantPayload *grant, Siz && cluster_gcs_pcm_x_image_token_wire_valid(&grant->image, authenticated_node); } -/* INSTALL_READY is one canonical requester -> master application ACK. */ +/* INSTALL_READY is one canonical requester -> master application ACK. Both + * the V1 (104-byte, no rebase) and V2 (112-byte) exact frames are legal; the + * caller normalizes a V1 frame to rebased_own_generation == 0 before this + * check. A V1 frame carrying a nonzero rebase is impossible by construction + * and a V2 rebase must be strictly newer than the immutable identity base. */ static inline bool cluster_gcs_pcm_x_install_ready_ingress_valid(const PcmXInstallReadyPayload *ready, Size payload_length, int32 authenticated_node, uint64 current_epoch, int32 tag_master, - int32 local_node) -{ - return ready != NULL && payload_length == sizeof(*ready) && authenticated_node >= 0 - && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 - && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == local_node + int32 local_node, bool rebase_wire_active, + bool source_supports_rebase) +{ + return ready != NULL + && (payload_length == sizeof(*ready) || payload_length == PCM_X_INSTALL_READY_V1_LEN) + && (payload_length == sizeof(*ready) || ready->rebased_own_generation == 0) + && (payload_length != sizeof(*ready) || (rebase_wire_active && source_supports_rebase)) + && (ready->rebased_own_generation == 0 + || (ready->rebased_own_generation != UINT64_MAX + && ready->rebased_own_generation > ready->ref.identity.base_own_generation)) + && authenticated_node >= 0 && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT + && local_node >= 0 && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == local_node && cluster_gcs_pcm_x_transfer_ref_wire_valid(&ready->ref, current_epoch) && ready->ref.identity.node_id == authenticated_node && cluster_gcs_pcm_x_image_id_master_wire_valid(ready->image_id, local_node) @@ -1005,7 +1030,10 @@ cluster_gcs_pcm_x_commit_x_ingress_valid(const PcmXPhasePayload *commit, Size pa && commit->phase == PGRAC_IC_MSG_PCM_X_COMMIT_X && commit->flags == 0; } -/* FINAL_ACK proves the requester committed exactly one ownership generation. */ +/* FINAL_ACK proves the requester committed exactly one ownership generation. + * The wire check is only a monotonic floor against the immutable identity + * base: a published INSTALL_READY rebase may have moved the effective grant + * base, and the exact "+1" proof runs under the master ticket lock. */ static inline bool cluster_gcs_pcm_x_final_ack_ingress_valid(const PcmXFinalAckPayload *ack, Size payload_length, int32 authenticated_node, uint64 current_epoch, @@ -1018,7 +1046,7 @@ cluster_gcs_pcm_x_final_ack_ingress_valid(const PcmXFinalAckPayload *ack, Size p && ack->ref.identity.node_id == authenticated_node && cluster_gcs_pcm_x_image_id_master_wire_valid(ack->image_id, local_node) && ack->ref.identity.base_own_generation != UINT64_MAX - && ack->committed_own_generation == ack->ref.identity.base_own_generation + 1; + && ack->committed_own_generation > ack->ref.identity.base_own_generation; } /* FINAL_COMMIT_ACK is the canonical master -> requester application ACK. */ @@ -1272,6 +1300,31 @@ cluster_gcs_block_direct_state_transition_ok(ClusterGcsBlockDirectState from, return false; } +/* + * A current-master INVALIDATE that meets a node-local mirror-N + * GRANT_PENDING is also an authoritative reason for the one exact N->S + * attempt to stand down: that reader cannot be granted while this X transfer + * is invalidating the node. Keep the delivery predicate attempt-exact and + * refuse any live direct-land target. The caller only synthesizes a local + * DENIED_PENDING_X; the INVALIDATE itself still returns RETRYABLE_BUSY until + * the owning backend aborts its reservation, so no holder bit is credited. + */ +static inline bool +GcsBlockLocalPendingSDenialMatches(bool in_use, bool reply_received, bool stale, + uint8 transition_id, const BufferTag *slot_tag, + uint64 request_epoch, int32 expected_master_node, + ClusterGcsBlockDirectState direct_state, + bool direct_target_prepared, const BufferTag *invalidate_tag, + uint64 invalidate_epoch, int32 invalidate_master_node) +{ + return in_use && !reply_received && !stale && slot_tag != NULL && invalidate_tag != NULL + && transition_id == (uint8)PCM_TRANS_N_TO_S && BufferTagsEqual(slot_tag, invalidate_tag) + && request_epoch == invalidate_epoch && expected_master_node == invalidate_master_node + && !direct_target_prepared + && (direct_state == GCS_BLOCK_DIRECT_UNARMED + || direct_state == GCS_BLOCK_DIRECT_ABORTED); +} + /* ============================================================ * GcsBlockReplyStatus -- reply status code carried in @@ -1455,6 +1508,27 @@ typedef enum GcsBlockReplyStatus { * requester keeps 53R97 (Rule 8.A). */ } GcsBlockReplyStatus; +/* + * The block-request wire validator admits only legal transition ids, but the + * master's outer S/X decision and its final entry-lock transition apply are + * deliberately separated by buffer probing/copying. A concurrent handoff can + * therefore make an otherwise valid N->S/N->X request incompatible at that + * final apply. That is authority drift, not a client-terminal protocol error: + * reuse DENIED_PENDING_X's established fresh-request/token retry boundary. + * Keep every other transition's incompatibility terminal so malformed or + * structurally illegal control-plane transitions are never papered over. + */ +static inline GcsBlockReplyStatus +GcsBlockApplyRefusalStatus(PcmGcsTransitionApplyResult apply_result, + PcmLockTransition transition_id) +{ + if (apply_result == PCM_GCS_TRANSITION_PENDING_X + || (apply_result == PCM_GCS_TRANSITION_INCOMPATIBLE + && (transition_id == PCM_TRANS_N_TO_S || transition_id == PCM_TRANS_N_TO_X))) + return GCS_BLOCK_REPLY_DENIED_PENDING_X; + return GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE; +} + /* spec-5.16 D3b / r4 (spec-6.12a ㉕ extends) — every new reply status MUST be * appended as the tail value (no collision with any shipped status; r3 mis-read * a truncated enum as max 8, the real shipped max before spec-5.16 was @@ -3038,7 +3112,39 @@ StaticAssertDecl(GCS_BLOCK_DATA_SIZE == BLCKSZ, #include "cluster/cluster_pcm_lock.h" /* PcmLockMode for invalidate helper */ extern bool cluster_bufmgr_probe_block_for_gcs(BufferTag tag); extern bool cluster_bufmgr_read_storage_scn_for_gcs(BufferTag tag, SCN *out_page_scn); -extern bool cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char *dst); +/* Process-local diagnostic returned by the nonblocking holder-copy helper. + * This is observation only: the caller's bool success/deny contract and wire + * reply status remain unchanged. */ +typedef enum ClusterBufmgrGcsCopyRefusal { + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE = 0, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INVALID_ARGUMENT, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_SECOND, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_HC89_LSN_DRIFT, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_SMART_FUSION_UNCLASSIFIED, + CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INJECTED_EVICT +} ClusterBufmgrGcsCopyRefusal; + +/* A DATA worker cannot wait for BufferContent: its owner may itself be waiting + * for that worker to deliver a reply. Only the two conditional-lock misses + * are therefore retryable through the established fresh reservation/request + * boundary. Residency/current-image failures remain structural, while HC89 + * keeps its explicit one-retry hot-page bound. */ +static inline GcsBlockReplyStatus +GcsBlockMasterDirectCopyRefusalStatus(ClusterBufmgrGcsCopyRefusal refusal) +{ + if (refusal == CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST + || refusal == CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_SECOND) + return GCS_BLOCK_REPLY_DENIED_PENDING_X; + return GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; +} + +extern const char *cluster_bufmgr_gcs_copy_refusal_name(ClusterBufmgrGcsCopyRefusal refusal); +extern bool cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char *dst, + ClusterBufmgrGcsCopyRefusal *out_refusal); extern bool cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page_lsn, void **out_page_addr, BufferDesc **out_buf); @@ -3167,8 +3273,19 @@ extern bool cluster_gcs_block_local_x_upgrade_ext(BufferTag tag, bool *out_busy) * consistent), applies PCM_TRANS_X_TO_S_DOWNGRADE, flips the local * pcm_state cache X->S. False = not quiescent / not X / buffer gone / * master refused; caller falls back to the one-shot read-image ship. */ +typedef enum ClusterBufmgrGcsDowngradeOutcome { + CLUSTER_BUFMGR_GCS_DOWNGRADE_REFUSED_PRE_NOTIFY = 0, + CLUSTER_BUFMGR_GCS_DOWNGRADE_COMMITTED, + CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY +} ClusterBufmgrGcsDowngradeOutcome; extern bool cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag); +extern ClusterBufmgrGcsDowngradeOutcome cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image( + BufferTag tag, XLogRecPtr *out_page_lsn, char *dst, ClusterBufmgrGcsCopyRefusal *out_refusal); extern bool cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node); +extern ClusterBufmgrGcsDowngradeOutcome +cluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image( + BufferTag tag, int32 master_node, XLogRecPtr *out_page_lsn, char *dst, + ClusterBufmgrGcsCopyRefusal *out_refusal); /* PGRAC: GCS-race round-4c P1 — re-send the (idempotent) X->S downgrade * notify when a BAST nudge arrives for a block this node already holds in @@ -3257,14 +3374,15 @@ extern int cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, /* * Returns true if a DURABLE PCM grant was acquired (GRANTED / STORAGE_ * FALLBACK — the caller mirrors PCM ownership into buf->pcm_state). Returns - * false for a spec-5.2 D2 one-shot READ_IMAGE: the bytes were installed for - * this read only, no S holder was registered, and the caller MUST leave - * buf->pcm_state == N so the next access re-fetches. Terminal denials - * ereport(ERROR) and do not return. + * false for a spec-5.2 D2 one-shot READ_IMAGE, or for an authoritative + * DENIED_PENDING_X with *out_retry_denied set. In the latter case the caller + * must exact-abort its GRANT_PENDING reservation before waiting/re-entering. + * Terminal denials ereport(ERROR) and do not return. */ extern bool cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition transition_id, - int master_node, bool clean_eligible); + int master_node, bool clean_eligible, + bool *out_retry_denied); /* * spec-5.2 D2 (sub-case B) — local-master read-image forward. Used by @@ -3272,14 +3390,22 @@ extern bool cluster_gcs_send_block_request_and_wait(BufferDesc *buf, * block a REMOTE node holds in X and a local reader needs an N→S image. * Forwards a read-image request to the holder and installs the shipped * current image for one read. Returns false (non-durable; caller leaves - * buf->pcm_state == N); fails closed (ereport) if no image is obtained. + * buf->pcm_state == N); fails closed (ereport) if no image is obtained while + * expected remains exact. Authority drift instead sets out_retry_denied so + * bufmgr aborts/rearms GRANT_PENDING and selects a fresh holder identity. */ -extern bool cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, int32 holder_node); +extern bool cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, + const PcmAuthoritySnapshot *expected, + bool *out_retry_denied); /* PGRAC: spec-5.2 D11 — local-master writer-transfer (revoke); durable X grant. * spec-5.2a D2/D3: clean_eligible routes a clean (sequence) page through the - * flush-data-before-drop holder path + stale-holder storage-fallback recovery. */ -extern bool cluster_gcs_local_master_x_transfer_and_wait(BufferDesc *buf, int32 holder_node, - bool clean_eligible); + * flush-data-before-drop holder path + stale-holder storage-fallback recovery. + * P0-26: expected is the entry-lock authority token; authority drift returns + * retry_denied so bufmgr aborts/rearms a fresh ownership/request identity. */ +extern bool cluster_gcs_local_master_x_transfer_and_wait(BufferDesc *buf, + const PcmAuthoritySnapshot *expected, + bool clean_eligible, + bool *out_retry_denied); /* * spec-4.7 D1 — GCS/PCM block resource recovery phase. @@ -3406,6 +3532,8 @@ typedef enum GcsBlockSendFamily { } GcsBlockSendFamily; extern void cluster_gcs_block_note_send_outcome(GcsBlockSendFamily family, ClusterICSendResult rc); +extern ClusterICSendResult +cluster_gcs_block_send_direct_zero_reply(int32 dest_node, const GcsBlockReplyHeader *header); extern uint64 cluster_gcs_get_reply_send_queued_count(void); extern uint64 cluster_gcs_get_reply_send_not_admitted_count(void); @@ -3550,6 +3678,7 @@ extern uint64 cluster_gcs_get_x_vs_s_no_carrier_denied_count(void); /* PGRAC: spec-2.37 D12 — 4 NEW counter accessors for PI watermark + lost-write. */ extern uint64 cluster_gcs_get_pi_watermark_advance_count(void); extern uint64 cluster_gcs_get_pi_watermark_retire_count(void); +extern uint64 cluster_gcs_get_pi_durable_note_apply_count(void); extern uint64 cluster_gcs_get_lost_write_detected_count(void); extern uint64 cluster_gcs_get_lost_write_avoid_count(void); /* PGRAC: spec-2.41 D7 — SCN detector + redo-coverage observability accessors. */ @@ -3672,6 +3801,9 @@ extern bool cluster_gcs_pcm_x_stage_frame(uint8 msg_type, int32 dest_node_id, co extern PcmXQueueResult cluster_gcs_pcm_x_acquire_writer(BufferDesc *buf, PcmXLocalWriterClaim *claim_out, bool *claim_handed_off); +/* Diagnostic: source line of this backend's most recent non-OK + * acquire-writer exit (0 when it never failed). */ +extern int cluster_gcs_pcm_x_requester_last_fail_line(void); extern PcmXQueueResult cluster_gcs_pcm_x_writer_claim_release_and_wake_exact(const PcmXLocalWriterClaim *claim); /* Error/owner-exit cleanup adapter: never propagates ERROR. A caught ERROR diff --git a/src/include/cluster/cluster_gcs_block_dedup.h b/src/include/cluster/cluster_gcs_block_dedup.h index 3d8e220fb6..92624eefd3 100644 --- a/src/include/cluster/cluster_gcs_block_dedup.h +++ b/src/include/cluster/cluster_gcs_block_dedup.h @@ -133,8 +133,9 @@ StaticAssertDecl(sizeof(GcsBlockDedupKey) == 24, "spec-2.34 D2 GcsBlockDedupKey * cell is an outbound-admission marker: nonzero * suppresses type-50 resends until exact type 49 * clears it. It is never application completion; - * exact DRAIN removes the dedicated entry, and all - * generic reclaim paths reject non-GENERIC kinds. + * exact DRAIN publishes a replay tombstone; exact + * RETIRE removes it, and all + generic reclaim paths * reject non-GENERIC kinds. * pinned_lifetime_us the legal-request-lifetime threshold, pinned * at REGISTRATION from the requester's wire * hint (2x margin applied) or, absent a hint, @@ -152,7 +153,10 @@ typedef enum GcsBlockDedupEntryKind { GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE = 2, /* Stable bytes exist, but X->N has not yet been positively committed. * This state is retained across LMS death and is never sendable. */ - GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED = 3 + GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED = 3, + /* Exact descriptor/byte cleanup completed after local TERMINAL_DRAINED. + * Keep the binding as an ACK replay tombstone until exact RETIRE. */ + GCS_BLOCK_DEDUP_ENTRY_PCM_X_DRAINED = 4 } GcsBlockDedupEntryKind; typedef union GcsBlockDedupPayloadMeta { @@ -169,12 +173,13 @@ typedef struct GcsBlockDedupEntry { uint8 transition_id; /* 1B — HC91 collision check */ uint8 status; /* 1B — GcsBlockReplyStatus */ uint8 entry_kind; /* 1B — GcsBlockDedupEntryKind */ - uint8 _pad0; /* 1B — session @ 48 */ + uint8 request_flags; /* 1B — generic request properties; PCM-X + * source state or drained master node + 1 */ uint64 pcm_x_master_session; /* 8B — PCM-X kind only */ GcsBlockReplyHeader reply_header; /* 48B — full reply header (HC99) */ - bool has_sf_dep; /* 1B — spec-6.2 v2 dep vector present */ - uint8 sf_flags; /* 1B — GCS_BLOCK_REPLY_SF_* */ - uint8 sf_dep_count; /* 1B — non-empty dep vector entries */ + bool has_sf_dep; /* 1B — generic SF metadata; PCM-X */ + uint8 sf_flags; /* 1B — metadata cell overlays the */ + uint8 sf_dep_count; /* 1B — exact reservation token */ uint8 _pad1[5]; /* 5B — dep_vec @ 112 */ GcsBlockDedupPayloadMeta payload_meta; /* 128B — deps or exact PCM-X identity */ char block_data[GCS_BLOCK_DATA_SIZE]; /* 8192B — full page payload */ @@ -185,6 +190,10 @@ typedef struct GcsBlockDedupEntry { int64 pinned_done_linger_us; /* 8B — round-2: quarantine pinned */ } GcsBlockDedupEntry; +#define GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND UINT8_C(0x01) +#define GCS_BLOCK_DEDUP_REQUEST_F_VALID_MASK GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND +#define GCS_BLOCK_DEDUP_REQUEST_F_PINNED UINT8_C(0x80) + StaticAssertDecl(offsetof(GcsBlockDedupEntry, entry_kind) == 46, "dedup entry kind occupies established padding at offset 46"); StaticAssertDecl(offsetof(GcsBlockDedupEntry, pcm_x_master_session) == 48, @@ -208,7 +217,10 @@ typedef enum GcsBlockPcmXImageResult { GCS_BLOCK_PCM_X_IMAGE_NOT_READY, GCS_BLOCK_PCM_X_IMAGE_STALE, GCS_BLOCK_PCM_X_IMAGE_FULL, - GCS_BLOCK_PCM_X_IMAGE_INVALID + GCS_BLOCK_PCM_X_IMAGE_INVALID, + /* Immutable bytes exist, but ownership commit is still pending. This + * process-local work result is never a sendable READY classification. */ + GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING } GcsBlockPcmXImageResult; /* By-value LMS work descriptor. It deliberately carries no page bytes: @@ -217,13 +229,15 @@ typedef enum GcsBlockPcmXImageResult { typedef struct GcsBlockPcmXImageWork { GcsBlockDedupKey key; GcsBlockPcmXImageBinding binding; + uint64 reservation_token; BufferTag tag; + uint8 source_pcm_state; uint8 entry_kind; - uint8 _reserved[3]; + uint8 _reserved[2]; } GcsBlockPcmXImageWork; -StaticAssertDecl(sizeof(GcsBlockPcmXImageWork) == 184, - "PCM-X LMS work descriptor remains a by-value 184-byte token"); +StaticAssertDecl(sizeof(GcsBlockPcmXImageWork) == 200, + "PCM-X LMS work descriptor includes the exact source floor binding"); /* ============================================================ @@ -272,6 +286,17 @@ typedef enum GcsBlockDedupResult { * path rather than re-broadcast. */ } GcsBlockDedupResult; +/* Result of advancing the exact legacy-reader termination set after a + * queue-kind pending-X claim has become authoritative. NEW means one live + * N->S identity was atomically replaced with a cached retry denial; REPLAY + * returns an already-denied identity that still lacks exact DONE proof. */ +typedef enum GcsBlockPendingXDenyResult { + GCS_BLOCK_PENDING_X_DENY_INVALID = -1, + GCS_BLOCK_PENDING_X_DENY_NOT_FOUND = 0, + GCS_BLOCK_PENDING_X_DENY_NEW = 1, + GCS_BLOCK_PENDING_X_DENY_REPLAY = 2 +} GcsBlockPendingXDenyResult; + /* ============================================================ * Eager reclaim safety decision primitives (spec-7.2a; review r1). @@ -453,6 +478,23 @@ extern GcsBlockDedupResult cluster_gcs_block_dedup_lookup_or_register( uint32 requester_lifetime_hint_ms, bool requester_done_capable, GcsBlockDedupEntry *cached_reply_out); +/* Under the routed shard lock, terminate one same-tag, still-live legacy + * N->S grant/forward identity after PCM-X publishes its queue-kind claim. + * A by-value cached denial is returned for initial send or periodic replay; + * exact DONE removes it from the replay set. */ +extern GcsBlockPendingXDenyResult +cluster_gcs_block_dedup_pending_x_deny_next(int worker_id, const BufferTag *tag, + GcsBlockDedupEntry *denied_out); +extern GcsBlockPendingXDenyResult +cluster_gcs_block_dedup_pending_x_deny_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, uint8 transition_id, + GcsBlockDedupEntry *denied_out); +extern bool cluster_gcs_block_dedup_set_request_flags_exact(int worker_id, + const GcsBlockDedupKey *key, + const BufferTag *tag, + uint8 transition_id, + uint8 request_flags); + /* Dedicated PCM-X image storage over the existing dedup entry pool. Reserve * claims capacity before the revoke lifecycle starts. Materialize publishes * immutable bytes only after the caller has established its exact revoke @@ -463,8 +505,8 @@ cluster_gcs_block_dedup_pcm_x_reserve(int worker_id, const GcsBlockDedupKey *key const GcsBlockPcmXImageBinding *reserved_binding); extern GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_materialize( int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, - const GcsBlockPcmXImageBinding *ready_binding, const GcsBlockReplyHeader *reply_header, - const char *block_data); + const GcsBlockPcmXImageBinding *ready_binding, uint64 reservation_token, uint8 source_pcm_state, + const GcsBlockReplyHeader *reply_header, const char *block_data); extern GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_publish_ready_exact(int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, @@ -473,9 +515,19 @@ extern GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_lookup( int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, const GcsBlockPcmXImageBinding *expected_binding, GcsBlockDedupEntry *cached_reply_out); extern GcsBlockPcmXImageResult -cluster_gcs_block_dedup_pcm_x_release_exact(int worker_id, const GcsBlockDedupKey *key, - const BufferTag *tag, - const GcsBlockPcmXImageBinding *binding); +cluster_gcs_block_dedup_pcm_x_drain_status_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *binding); +extern GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_release_exact( + int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, + const GcsBlockPcmXImageBinding *binding, int32 drained_master_node); +extern bool cluster_gcs_block_dedup_pcm_x_retire_up_to(uint64 cluster_epoch, + int32 authenticated_master_node, + uint64 authenticated_master_session, + uint64 retire_through_ticket_id); +extern GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_preserve_finish_error_exact( + int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, + const GcsBlockPcmXImageBinding *binding, uint64 reservation_token, uint8 source_pcm_state); extern GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_next_work(int worker_id, GcsBlockPcmXImageWork *work_out); extern GcsBlockPcmXImageResult diff --git a/src/include/cluster/cluster_ic.h b/src/include/cluster/cluster_ic.h index 4f10bf143f..c8ca15c96a 100644 --- a/src/include/cluster/cluster_ic.h +++ b/src/include/cluster/cluster_ic.h @@ -368,6 +368,19 @@ typedef enum ClusterICPlane { * family. A sender must not transmit any PCM-X frame to a peer that did not * advertise this bit; an old peer would treat the type as unregistered. */ #define PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 ((uint32)0x00000200U) +/* PGRAC: A' rebase (t/400 item 2) — this binary understands the V2 (112-byte) + * INSTALL_READY frame whose trailing rebased_own_generation publishes the + * effective grant base after an interleaved revoke consumed the enqueue-time + * base. A PROTOCOL capability, advertised unconditionally. Activation is + * formation-wide: the PCM-X runtime samples every MEMBER's bit at activation + * and only then may a requester publish a rebase; without full coverage the + * V2 frame is never sent (a drifted requester keeps the fail-closed STALE + * verdict) and every INSTALL_READY stays the V1 104-byte exact frame, so an + * old master never sees a length its byte-exact table would refuse. */ +#define PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1 ((uint32)0x00000400U) +/* P0-20: this binary accepts the 104-byte PCM-X REVOKE V2 frame and binds its + * trailing required_page_scn to the immutable source-image lifecycle. */ +#define PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1 ((uint32)0x00000800U) /* * PGRAC: spec-7.2 D2 — plane + connection-epoch ride the documented-zero * pad region (capabilities precedent: occupy pad bytes, do not resize V1). diff --git a/src/include/cluster/cluster_inject.h b/src/include/cluster/cluster_inject.h index 614ab14d91..b6e258fe62 100644 --- a/src/include/cluster/cluster_inject.h +++ b/src/include/cluster/cluster_inject.h @@ -115,6 +115,9 @@ extern void cluster_injection_run(const char *name); extern bool cluster_injection_should_skip(const char *name); +/* Non-consuming armed-state probe for deterministic test setup. */ +extern bool cluster_injection_is_armed(const char *name); + /* * cluster_cr_injection_armed -- spec-3.9 D7 CR-specific armed-state peek. * Returns true iff the named CR injection point is armed; sets *out_param diff --git a/src/include/cluster/cluster_lms.h b/src/include/cluster/cluster_lms.h index c716f5d677..ba694f242d 100644 --- a/src/include/cluster/cluster_lms.h +++ b/src/include/cluster/cluster_lms.h @@ -321,6 +321,9 @@ typedef struct ClusterLmsSharedState { */ pg_atomic_uint64 worker_outbound_not_admitted_count[CLUSTER_LMS_MAX_WORKERS]; pg_atomic_uint64 worker_outbound_requeue_drop_count[CLUSTER_LMS_MAX_WORKERS]; + /* Capability-bound wire frames consumed before send because the peer's + * current HELLO record no longer matches the staged connection generation. */ + pg_atomic_uint64 worker_outbound_cap_guard_drop_count[CLUSTER_LMS_MAX_WORKERS]; } ClusterLmsSharedState; @@ -401,8 +404,22 @@ extern void cluster_lms_outbound_shmem_register(void); extern void cluster_lms_outbound_request_lwlocks(void); extern bool cluster_lms_outbound_enqueue(int worker_id, uint8 msg_type, uint32 dest_node_id, const void *payload, uint16 payload_len); +extern bool cluster_lms_outbound_enqueue_cap_bound(int worker_id, uint8 msg_type, + uint32 dest_node_id, const void *payload, + uint16 payload_len, uint32 required_capability, + uint32 connection_generation); +struct GcsBlockReplyHeader; +extern bool cluster_lms_outbound_enqueue_zero_block_reply(int worker_id, uint32 dest_node_id, + const struct GcsBlockReplyHeader *header, + bool direct_land); extern int cluster_lms_outbound_drain_send(int worker_id); extern uint32 cluster_lms_outbound_depth(int worker_id); +extern void cluster_lms_note_pcm_x_image_ready_boundary(uint8 msg_type, const char *boundary, + int result, int runtime_state, + bool fence_enforcing, bool fence_allowed, + uint32 dest_node_id, uint64 request_id, + uint64 ticket_id, uint64 grant_generation, + uint64 image_id); /* * Read-only accessors for SQL view + diagnostics. @@ -473,8 +490,10 @@ extern uint64 cluster_lms_obs_serve_hist_bound_us(int bucket); * requeue-drop; get_* readers follow the -1 = pool aggregate idiom). */ extern void cluster_lms_obs_note_outbound_not_admitted(int worker_id); extern void cluster_lms_obs_note_outbound_requeue_drop(int worker_id); +extern void cluster_lms_obs_note_outbound_cap_guard_drop(int worker_id); extern uint64 cluster_lms_obs_get_outbound_not_admitted(int worker_id); extern uint64 cluster_lms_obs_get_outbound_requeue_drop(int worker_id); +extern uint64 cluster_lms_obs_get_outbound_cap_guard_drop(int worker_id); /* spec-7.3 D8 (Q8) — apply cluster.lms_nice to the calling LMS process * (setpriority, best-effort; 0 = leave the inherited priority alone). */ diff --git a/src/include/cluster/cluster_pcm_lock.h b/src/include/cluster/cluster_pcm_lock.h index 70235256f2..0301a2f933 100644 --- a/src/include/cluster/cluster_pcm_lock.h +++ b/src/include/cluster/cluster_pcm_lock.h @@ -104,6 +104,16 @@ typedef enum PcmLockTransition { } PcmLockTransition; #define PCM_TRANSITION_COUNT 9 +/* Exact master-side apply verdict. PENDING_X is distinct from a structural + * state incompatibility so the block data plane can return its retryable + * DENIED_PENDING_X status after the final, entry-lock-serialized admission + * check closes a preflight-to-apply race. */ +typedef enum PcmGcsTransitionApplyResult { + PCM_GCS_TRANSITION_APPLIED = 0, + PCM_GCS_TRANSITION_INCOMPATIBLE = 1, + PCM_GCS_TRANSITION_PENDING_X = 2 +} PcmGcsTransitionApplyResult; + /* * GrdEntry -- opaque per-block global lock state (master node). @@ -171,6 +181,13 @@ typedef enum PcmXGrdHandoffResult { PCM_X_GRD_HANDOFF_BAD_STATE, PCM_X_GRD_HANDOFF_INVALID } PcmXGrdHandoffResult; +typedef enum PcmXTransferCommitResult { + PCM_X_TRANSFER_COMMIT_OK = 0, + PCM_X_TRANSFER_COMMIT_NOT_FOUND, + PCM_X_TRANSFER_COMMIT_STALE, + PCM_X_TRANSFER_COMMIT_BAD_STATE +} PcmXTransferCommitResult; + typedef enum PcmPendingXReserveResult { PCM_PENDING_X_RESERVE_INVALID = -2, @@ -315,9 +332,12 @@ extern void cluster_pcm_lock_acquire(BufferTag tag, PcmLockMode mode); * Returns true if a DURABLE PCM grant was recorded (caller mirrors ownership * into buf->pcm_state). Returns false for a spec-5.2 D2 one-shot READ_IMAGE: * bytes were installed for this read only and the caller MUST leave - * buf->pcm_state == N so the next access re-fetches. + * buf->pcm_state == N so the next access re-fetches. A false return with + * *out_retry_denied set is instead a queue-arbitration retry boundary; the + * caller must exact-abort and replace its GRANT_PENDING reservation. */ -extern bool cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode); +extern bool cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode, + bool *out_retry_denied); /* * PGRAC: spec-5.2a D2 — clean-page X-transfer arm (backend-local, one-shot). @@ -371,13 +391,13 @@ extern bool cluster_pcm_clean_page_xfer_consume(void); extern void cluster_pcm_lock_unlock_content_buffer(BufferDesc *buf, PcmLockMode mode); extern void cluster_pcm_lock_release_saved_tag_for_eviction(BufferTag tag, PcmLockMode mode); extern void cluster_pcm_lock_release_buffer_for_eviction(BufferDesc *buf, PcmLockMode mode); -/* PGRAC: spec-5.2 D11 — local master records self as new X holder after a - * writer-transfer (the remote holder shipped + released its X). S3 forensics - * step 1b: the shipping holder + wire request_id/epoch ride along so the - * watermark-advance provenance table records WHO produced the advance. */ -extern void cluster_pcm_lock_master_take_x_after_transfer(BufferTag tag, XLogRecPtr page_lsn, - SCN page_scn, int32 holder_node, - uint64 request_id, uint64 epoch); +/* PGRAC: spec-5.2 D11 / P0-26 — commit a local-master X transfer only if the + * full remote-X authority sampled before the wire round is unchanged. The + * exact source identity/generation/session closes late-reply and concurrent + * queue-handoff races; STALE is a fresh-request retry boundary. */ +extern PcmXTransferCommitResult cluster_pcm_lock_master_take_x_after_transfer( + BufferTag tag, const PcmAuthoritySnapshot *expected, XLogRecPtr page_lsn, SCN page_scn, + int32 holder_node, uint32 requester_procno, uint64 request_id, uint64 epoch); /* PGRAC: spec-5.2 D11 path B — master==holder==self ships its X image to a * remote requester and records the requester as the new X holder (single-phase * writer-transfer-revoke; caller drops self's copy no-wire before calling). @@ -419,6 +439,7 @@ extern void cluster_pcm_lock_downgrade(BufferTag tag, PcmLockMode target_mode, b */ extern PcmLockMode cluster_pcm_lock_query(BufferTag tag); extern bool cluster_pcm_lock_authority_snapshot(BufferTag tag, PcmAuthoritySnapshot *out); +extern bool cluster_pcm_lock_authority_matches(BufferTag tag, const PcmAuthoritySnapshot *expected); extern PcmXGrdHandoffResult cluster_pcm_lock_queue_handoff_x_exact(const PcmXGrdHandoffToken *token); extern int cluster_pcm_grd_count(void); @@ -448,6 +469,9 @@ extern void cluster_pcm_grd_init(void); extern bool cluster_pcm_transition_legal(PcmState from, PcmState to, PcmLockTransition trans); extern void cluster_pcm_transition_apply(struct GrdEntry *entry, PcmLockTransition trans, int holder_node_id); +extern PcmGcsTransitionApplyResult +cluster_pcm_lock_apply_gcs_transition_result(BufferTag tag, PcmLockTransition trans, + int holder_node_id); extern bool cluster_pcm_lock_apply_gcs_transition(BufferTag tag, PcmLockTransition trans, int holder_node_id); diff --git a/src/include/cluster/cluster_pcm_own.h b/src/include/cluster/cluster_pcm_own.h index 158d6477c3..ac7300730b 100644 --- a/src/include/cluster/cluster_pcm_own.h +++ b/src/include/cluster/cluster_pcm_own.h @@ -61,13 +61,16 @@ #define PCM_OWN_FLAG_REVOKING ((uint32)0x2) /* a revoke (downgrade/invalidate) started */ typedef struct ClusterPcmOwnEntry { - pg_atomic_uint64 generation; /* monotone; bumped on every committed transition */ - pg_atomic_uint64 reservation_token; /* monotone; active iff a transient flag is set */ - pg_atomic_uint32 flags; /* PCM_OWN_FLAG_* */ - uint32 _pad; /* keep 24B aligned */ + pg_atomic_uint64 generation; /* monotone; bumped on every committed transition */ + pg_atomic_uint64 reservation_token; /* monotone; active iff a transient flag is set */ + pg_atomic_uint64 writer_activation_token; /* committed X not yet activated under content X */ + pg_atomic_uint32 flags; /* PCM_OWN_FLAG_* */ + uint32 _pad; /* keep 32B aligned */ } ClusterPcmOwnEntry; -StaticAssertDecl(sizeof(ClusterPcmOwnEntry) == 24, "ClusterPcmOwnEntry must remain 24 bytes"); +StaticAssertDecl(sizeof(ClusterPcmOwnEntry) == 32, "ClusterPcmOwnEntry must remain 32 bytes"); +StaticAssertDecl(offsetof(ClusterPcmOwnEntry, writer_activation_token) == 16, + "writer activation fence offset must remain stable"); typedef enum ClusterPcmOwnResult { CLUSTER_PCM_OWN_OK = 0, @@ -130,6 +133,18 @@ extern ClusterPcmOwnResult cluster_pcm_own_revoke_commit_exact(int buf_id, uint64 expected_generation, uint64 reservation_token, uint64 *out_committed_generation); +/* X-only combined linearization: clear the exact GRANT_PENDING lifecycle, + * bump its ownership generation, and publish the writer activation fence + * before the caller releases the BufferDesc header lock. */ +extern ClusterPcmOwnResult +cluster_pcm_own_writer_grant_commit_exact(int buf_id, uint64 expected_generation, + uint64 reservation_token, + uint64 *out_committed_generation); +/* Clear only the exact committed generation/token after content-EXCLUSIVE + * activation proof or after exact queue-claim cleanup. */ +extern ClusterPcmOwnResult cluster_pcm_own_writer_activation_clear_exact(int buf_id, + uint64 expected_generation, + uint64 reservation_token); /* PCM-X retained-image revoke: commit advances the ownership generation but * deliberately leaves the exact REVOKING token live until DRAIN proves the * immutable image record is no longer needed. Release clears only that @@ -175,6 +190,14 @@ cluster_pcm_own_reservation_token_get(int buf_id) return pg_atomic_read_u64(&ClusterPcmOwnArray[buf_id].reservation_token); } +static inline uint64 +cluster_pcm_own_writer_activation_token_get(int buf_id) +{ + if (ClusterPcmOwnArray == NULL || buf_id < 0) + return 0; + return pg_atomic_read_u64(&ClusterPcmOwnArray[buf_id].writer_activation_token); +} + static inline void cluster_pcm_own_flags_apply(int buf_id, uint32 set, uint32 clear) { diff --git a/src/include/cluster/cluster_pcm_x_bufmgr.h b/src/include/cluster/cluster_pcm_x_bufmgr.h index 6bd548d26c..65306ef698 100644 --- a/src/include/cluster/cluster_pcm_x_bufmgr.h +++ b/src/include/cluster/cluster_pcm_x_bufmgr.h @@ -21,6 +21,7 @@ #ifndef CLUSTER_PCM_X_BUFMGR_H #define CLUSTER_PCM_X_BUFMGR_H +#include "access/transam.h" #include "access/xlogdefs.h" #include "cluster/cluster_pcm_own.h" #include "cluster/cluster_pcm_x_convert.h" @@ -33,6 +34,17 @@ * becomes a transaction ERROR merely because RETIRE owns the short gate. */ #define CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS 5 +/* FSM pages are advisory and may be consumed without a content lock, so they + * are outside PCM/PCM-X ownership. Every other fork keeps the relation-level + * user/shared-catalog tracking policy. */ +static inline bool +cluster_pcm_x_buffer_tag_tracked(const BufferTag *tag, bool shared_catalog) +{ + if (tag == NULL || tag->forkNum == FSM_FORKNUM) + return false; + return shared_catalog || tag->relNumber >= (RelFileNumber)FirstNormalObjectId; +} + typedef enum ClusterPcmXHolderRetryAction { CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE = 0, CLUSTER_PCM_X_HOLDER_RETRY_WAIT, @@ -74,7 +86,7 @@ cluster_pcm_x_holder_register_retry_action(PcmXQueueResult result, bool runtime_ if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) return CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE; if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BARRIER_CLOSED - || (result == PCM_X_QUEUE_NOT_READY && runtime_active)) + || (runtime_active && (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY))) return CLUSTER_PCM_X_HOLDER_RETRY_WAIT; return CLUSTER_PCM_X_HOLDER_RETRY_FAIL; } @@ -113,12 +125,50 @@ typedef struct ClusterPcmOwnSnapshot { BufferTag tag; uint64 generation; uint64 reservation_token; + /* Diagnostic projection of the grant->content activation fence. It is + * sampled under the same BufferDesc header lock as the authoritative + * ownership tuple, but is not part of snapshot matching or wire state. */ + uint64 writer_activation_token; uint32 flags; uint8 pcm_state; uint8 _reserved[3]; } ClusterPcmOwnSnapshot; -StaticAssertDecl(sizeof(ClusterPcmOwnSnapshot) == 48, "ClusterPcmOwnSnapshot must remain 48 bytes"); +StaticAssertDecl(sizeof(ClusterPcmOwnSnapshot) == 56, "ClusterPcmOwnSnapshot must remain 56 bytes"); + +/* + * Process-local evidence for a reversible finish-revoke refusal. The DATA + * worker fills this from the same header-lock sample that chose BUSY, so GCS + * can correlate the exact ticket/image with the actual VM/FSM pin, I/O, or + * ownership-lifecycle branch. This is diagnostics only; it is never sent on + * wire or persisted. + */ +typedef enum ClusterPcmOwnFinishRefusalReason { + CLUSTER_PCM_OWN_FINISH_REFUSAL_NONE = 0, + CLUSTER_PCM_OWN_FINISH_REFUSAL_VM_FSM_PINNED, + CLUSTER_PCM_OWN_FINISH_REFUSAL_IO_IN_PROGRESS, + CLUSTER_PCM_OWN_FINISH_REFUSAL_LIVE_FLAGS +} ClusterPcmOwnFinishRefusalReason; + +typedef struct ClusterPcmOwnFinishRefusal { + uint64 live_token; + uint32 shared_refcount; + uint32 live_flags; + ClusterPcmOwnFinishRefusalReason reason; + bool bm_io_in_progress; +} ClusterPcmOwnFinishRefusal; + +/* Process-local reason for a retryable S-source image preparation refusal. + * DIRTY_FLUSHED records forward progress; the other values identify the + * exact nonblocking gate that asked the image pump to retry. */ +typedef enum ClusterPcmOwnSourcePrepareRefusal { + CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_NONE = 0, + CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_BEGIN_REVOKE, + CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_CONTENT_LOCK, + CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_DIRTY_FLUSHED, + CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_DIRTY_RACED, + CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_IO_IN_PROGRESS +} ClusterPcmOwnSourcePrepareRefusal; /* The queue returns execution authority before bufmgr takes content EXCLUSIVE. * Bind that claim to the one committed ownership generation and require the @@ -131,7 +181,14 @@ cluster_pcm_x_writer_grant_snapshot_exact(const PcmXLocalWriterClaim *claim, return claim != NULL && granted != NULL && live != NULL && claim->flags == 0 && claim->writer.flags == 0 && claim->claim_generation != 0 && claim->writer.identity.base_own_generation != UINT64_MAX - && granted->generation == claim->writer.identity.base_own_generation + 1 + && claim->grant_base_own_generation != UINT64_MAX + /* A' rebase: the requester copies the published effective grant + * base into the claim; zero keeps the enqueue-time identity base. */ + && granted->generation + == (claim->grant_base_own_generation != 0 + ? claim->grant_base_own_generation + : claim->writer.identity.base_own_generation) + + 1 && granted->reservation_token != 0 && granted->flags == 0 && granted->pcm_state == (uint8)PCM_STATE_X && BufferTagsEqual(&granted->tag, &claim->writer.identity.tag) @@ -165,6 +222,31 @@ cluster_pcm_x_cached_cover_bypasses_queue(bool local_cache, bool requested_x, ui return local_cache && requested_x && pcm_state == (uint8)PCM_STATE_X && flags == 0; } +/* Revalidate a cached cover after the caller has acquired content authority. + * A stable current S/X is the node-level authority for an S read even when a + * complete ownership round advanced the generation while this backend waited: + * the page-image install/transition is serialized by the content lock already + * held by the caller. Treating that successor S as stale would open a fresh + * legacy reservation from S (the forbidden S_NEW shape). X writers retain + * the stricter generation-exact rule and re-enter the convert queue after any + * ownership round. */ +static inline bool +cluster_pcm_x_cached_cover_reverify_accepts(uint8 requested_state, uint64 captured_generation, + uint64 current_generation, uint8 current_state, + uint32 current_flags) +{ + bool covers; + + if (requested_state != (uint8)PCM_STATE_S && requested_state != (uint8)PCM_STATE_X) + return false; + if (current_flags != 0) + return false; + covers = current_state == (uint8)PCM_STATE_X + || (requested_state == (uint8)PCM_STATE_S && current_state == (uint8)PCM_STATE_S); + return covers + && (requested_state == (uint8)PCM_STATE_S || current_generation == captured_generation); +} + /* ConditionalLockBuffer cannot initiate a PCM conversion. Preserve native * PostgreSQL behavior while PCM is inactive and for relations outside the * coherence domain; an active tracked page must already hold exact X. Live @@ -191,7 +273,17 @@ typedef enum ClusterPcmXGrantReservationKind { * acquisition allocates the next monotonic token. A requester acting as its * own N/S/X image source instead reuses the already-live revoke token and * changes only its role. Every shape retains tag/generation/pcm_state until - * the single grant commit. */ + * the single grant commit. + * + * A fresh-token S base (an S holder entering the legacy master acquire, the + * loop9/loop10b "S_NEW" tuple) is deliberately NOT a legal finish shape: + * writer conversions are ordered by the convert queue's FIFO/WFG, and + * legalizing the legacy S-base short cut here would let a stale-cover + * fallback bypass that arbitration entirely (the original S3 unordered + * multi-writer defect). The stale-cover path must re-enter the queue + * instead; this classifier keeps refusing the bypass. A fresh-token X base + * is equally unenumerated: a live X cover never re-acquires through this + * path (cluster_pcm_x_cached_cover_bypasses_queue). */ static inline ClusterPcmXGrantReservationKind cluster_pcm_x_grant_reservation_kind(const ClusterPcmOwnSnapshot *live, const ClusterPcmOwnSnapshot *base, uint64 reservation_token) @@ -254,6 +346,42 @@ cluster_pcm_x_current_image_shape(uint8 pcm_state, uint8 buffer_type, bool valid || (pcm_state == (uint8)PCM_STATE_X && buffer_type == (uint8)BUF_TYPE_XCUR)); } +/* Post-release retag of a retained PI whose exact write-fence token was just + * released, applied under the same header-lock hold. With no pins the image + * is dropped -- !BM_VALID plus a BUF_TYPE_CURRENT retag makes the next + * ordinary read reload the current page bytes. With a pre-existing pin the + * frozen bytes must neither vanish nor change under the pin holder + * (PostgreSQL pin contract): keep the established PI+BM_VALID + * never-write/never-serve N mirror (the passive-pin invalidate release + * shape). The next S acquire begins an exact GRANT_PENDING install that may + * overwrite it and republish CURRENT, an X convert rides the convert queue, + * and eviction retags once the last pin drains. Returns true when the image + * was dropped. */ +static inline bool +cluster_pcm_x_retained_release_retag(uint8 *buffer_type, uint32 *buf_state) +{ + if (BUF_STATE_GET_REFCOUNT(*buf_state) == 0) { + *buf_state &= ~BM_VALID; + *buffer_type = (uint8)BUF_TYPE_CURRENT; + return true; + } + return false; +} + +/* A frozen PI mirror kept for pre-existing pins regains CURRENT only while + * the exact legacy grant lifecycle that just proved its bytes (a shipped + * install, a storage refresh, or an SCN PASS proof) is still open: pcm N + + * live GRANT_PENDING + nonzero token + BM_VALID + PI. Every other shape + * stays frozen so the grant-finish valid-image gate keeps failing closed on + * an unproven stale cover. */ +static inline bool +cluster_pcm_x_grant_pending_republish_shape(uint8 pcm_state, uint32 flags, uint64 token, bool valid, + uint8 buffer_type) +{ + return pcm_state == (uint8)PCM_STATE_N && flags == PCM_OWN_FLAG_GRANT_PENDING && token != 0 + && valid && buffer_type == (uint8)BUF_TYPE_PI; +} + /* Token zero is valid before the first reservation; a completed reservation * leaves a nonzero monotonic token idle. In both cases flags alone say * whether a lifecycle is currently active. */ @@ -276,6 +404,7 @@ cluster_bufmgr_pcm_own_snapshot_by_tag(const BufferTag *tag, int *out_buffer_id, * LockBuffer/W1. GRANT_PENDING image installation is permitted; a live * source REVOKING lifecycle or retained PI+VALID descriptor is not. */ extern bool cluster_bufmgr_pcm_x_content_write_permitted(BufferDesc *buf); +extern void cluster_bufmgr_pcm_own_republish_grant_pending_image(BufferDesc *buf); /* Called only after the requester has proved the exact remote master's S->N * RELEASE application ACK. Atomically normalizes the matching descriptor * tuple and returns the fresh N snapshot used by the later PREPARE leg. */ @@ -347,18 +476,40 @@ cluster_bufmgr_pcm_own_abort_n_revoke(BufferDesc *buf, extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_begin_s_revoke(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_s, ClusterPcmOwnSnapshot *out_revoking); +/* Freeze one exact S source and choose the newest safe bytes from its clean + * current image and verified shared storage. A valid required_page_scn is a + * hard floor; V1's zero floor still prefers the newer observed copy. */ +extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_prepare_s_source_image( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_s, SCN required_page_scn, + ClusterPcmOwnSnapshot *out_revoking, char block_data[BLCKSZ], XLogRecPtr *out_page_lsn, + uint64 *out_page_scn, ClusterPcmOwnSourcePrepareRefusal *out_refusal); extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_abort_s_revoke(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking); extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_finish_revoke_retain( BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, XLogRecPtr expected_lsn, - ClusterPcmOwnSnapshot *out_retained); + ClusterPcmOwnSnapshot *out_retained, ClusterPcmOwnFinishRefusal *out_refusal); extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, uint64 source_generation); -/* Read-only DRAIN proof for a sole-requester source that committed S->X in - * place. Exact base+1 X/XCUR authority means DRAIN may release only the - * immutable dedup record and must not invalidate this descriptor. */ -extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, - uint64 source_generation); +/* Process-local descriptor evidence captured by the DRAIN-time probe below + * so the caller can log the observed shape as diagnostics. The fields are + * one header-locked snapshot; never persisted or sent on wire. */ +typedef struct ClusterPcmOwnSelfHandoffSample { + uint64 live_generation; + uint64 live_token; + uint32 own_flags; + uint8 pcm_state; + uint8 buffer_type; + bool bm_valid; + bool buffer_found; +} ClusterPcmOwnSelfHandoffSample; + +/* Read-only corruption probe for a delayed sole-requester source DRAIN. The + * release authority is the protocol completion certificate; the descriptor + * may legitimately have moved on or been evicted, so this only reports a + * structurally malformed flags/token shape (and never mutates anything). */ +extern ClusterPcmOwnResult +cluster_bufmgr_pcm_own_self_handoff_probe(const BufferTag *tag, + ClusterPcmOwnSelfHandoffSample *sample_out); #endif /* CLUSTER_PCM_X_BUFMGR_H */ diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h index 068ddee7cb..35360cd2fe 100644 --- a/src/include/cluster/cluster_pcm_x_convert.h +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -32,7 +32,8 @@ #define PCM_X_PROTOCOL_NODE_LIMIT 32 #define PCM_X_SHMEM_REGION_NAME "pgrac cluster pcm convert queue" #define PCM_X_SHMEM_MAGIC ((uint32)0x50435851) /* "PCXQ" */ -#define PCM_X_SHMEM_LAYOUT_VERSION ((uint32)12) +/* 13: A' rebase slot growth (ticket 392 / local tag 760 / header stats). */ +#define PCM_X_SHMEM_LAYOUT_VERSION ((uint32)14) #define PCM_X_INVALID_SLOT_INDEX ((Size) - 1) #define PCM_X_LOCK_PARTITIONS NUM_BUFFER_PARTITIONS #define PCM_X_LWLOCK_COUNT (1 + 2 * PCM_X_LOCK_PARTITIONS) @@ -193,6 +194,13 @@ typedef struct PcmXRevokePayload { uint64 image_id; } PcmXRevokePayload; +/* P0-20 source-floor extension. V1 remains the byte-exact 96-byte prefix; + * V2 is sent only to a connection that advertised SOURCE_FLOOR_V1. */ +typedef struct PcmXRevokePayloadV2 { + PcmXRevokePayload v1; + uint64 required_page_scn; +} PcmXRevokePayloadV2; + typedef struct PcmXGrantPayload { PcmXTicketRef ref; PcmXImageToken image; @@ -204,8 +212,16 @@ typedef struct PcmXInstallReadyPayload { uint32 result; uint16 phase; uint16 flags; + /* A' rebase: nonzero publishes the effective grant base exactly once (an + * interleaved revoke consumed ref.identity.base_own_generation while the + * request was queued). ref stays the immutable admission-time locator. + * Only sent as the V2 (112-byte) frame when the whole formation + * advertises PCM_X_REBASE_V1; the V1 frame is the first 104 bytes. */ + uint64 rebased_own_generation; } PcmXInstallReadyPayload; +#define PCM_X_INSTALL_READY_V1_LEN ((Size)104) + typedef struct PcmXFinalAckPayload { PcmXTicketRef ref; uint64 image_id; @@ -250,8 +266,14 @@ StaticAssertDecl(sizeof(PcmXPrehandleCancelPayload) == 80, "PCM-X pre-handle CAN StaticAssertDecl(sizeof(PcmXAdmitAckPayload) == 112, "PCM-X ADMIT_ACK ABI"); StaticAssertDecl(sizeof(PcmXPhasePayload) == 96, "PCM-X phase ABI"); StaticAssertDecl(sizeof(PcmXRevokePayload) == 96, "PCM-X revoke ABI"); +StaticAssertDecl(sizeof(PcmXRevokePayloadV2) == 104, "PCM-X revoke V2 ABI"); +StaticAssertDecl(offsetof(PcmXRevokePayloadV2, required_page_scn) == sizeof(PcmXRevokePayload), + "PCM-X revoke V2 preserves the complete V1 prefix"); StaticAssertDecl(sizeof(PcmXGrantPayload) == 128, "PCM-X grant ABI"); -StaticAssertDecl(sizeof(PcmXInstallReadyPayload) == 104, "PCM-X INSTALL_READY ABI"); +StaticAssertDecl(sizeof(PcmXInstallReadyPayload) == 112, "PCM-X INSTALL_READY ABI"); +StaticAssertDecl(offsetof(PcmXInstallReadyPayload, rebased_own_generation) + == PCM_X_INSTALL_READY_V1_LEN, + "PCM-X INSTALL_READY V1 prefix ABI"); StaticAssertDecl(sizeof(PcmXFinalAckPayload) == 104, "PCM-X FINAL_ACK ABI"); StaticAssertDecl(sizeof(PcmXBlockerSetHeaderPayload) == 104, "PCM-X blocker header ABI"); StaticAssertDecl(sizeof(PcmXBlockerChunkPayload) == 128, "PCM-X blocker chunk ABI"); @@ -328,9 +350,11 @@ typedef struct PcmXRuntimeSnapshot { uint64 master_session_incarnation; uint32 gate_generation; PcmXRuntimeState state; + /* A' rebase: formation-wide PCM_X_REBASE_V1 coverage at activation. */ + bool rebase_wire_active; } PcmXRuntimeSnapshot; -StaticAssertDecl(sizeof(PcmXRuntimeSnapshot) == 16, "PCM-X runtime snapshot ABI"); +StaticAssertDecl(sizeof(PcmXRuntimeSnapshot) == 24, "PCM-X runtime snapshot ABI"); typedef enum PcmXMasterTicketState { PCM_XT_FREE = PCM_X_SLOT_FREE, @@ -426,6 +450,25 @@ typedef enum PcmXQueueResult { PCM_X_QUEUE_BARRIER_CLOSED } PcmXQueueResult; +/* Process-local classification for the holder-side IMAGE_READY arm. This is + * diagnostic evidence only: the queue result remains the protocol verdict and + * no value from this enum is persisted in shared memory or sent on the wire. */ +typedef enum PcmXLocalImageReadyRefusal { + PCM_X_LOCAL_IMAGE_READY_REFUSAL_NONE = 0, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_INVALID, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_RUNTIME, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_DIRECTORY, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_TAG_SLOT, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_LOCAL_GATE, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_RUNTIME_RECHECK, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_IDENTITY, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_ACTIVE_HOLDER, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_RELIABLE_LEG, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_BAD_PHASE, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_COUNTER, + PCM_X_LOCAL_IMAGE_READY_REFUSAL_GATE_RELEASE +} PcmXLocalImageReadyRefusal; + /* A generation-exact type-47 replay can prove the blocker set DUPLICATE * after the ticket has already advanced beyond ACTIVE_PROBE. Its type-48 * ACK is still safe to replay; only structural corruption warrants closing @@ -563,6 +606,9 @@ typedef struct PcmXLocalWriterClaim { uint32 local_round; uint16 role; uint16 flags; + /* A' rebase: the requester copies the published effective grant base here + * (0 = no drift) so the bufmgr grant-snapshot cross-check stays exact. */ + uint64 grant_base_own_generation; } PcmXLocalWriterClaim; /* @@ -586,14 +632,17 @@ typedef struct PcmXLocalProgress { uint64 master_session_incarnation; int32 master_node; uint32 reserved; + /* A' rebase: the tag's published effective grant base (0 = no drift). */ + uint64 grant_base_own_generation; } PcmXLocalProgress; -StaticAssertDecl(sizeof(PcmXLocalProgress) == 240, "PCM-X local progress ABI"); +StaticAssertDecl(sizeof(PcmXLocalProgress) == 248, "PCM-X local progress ABI"); /* Read-only process-local view of the independent holder transfer lane. */ typedef struct PcmXLocalHolderProgress { PcmXTicketRef ref; PcmXImageToken image; + uint64 required_page_scn; uint64 reliable_state_sequence; uint16 pending_opcode; uint16 last_response_opcode; @@ -605,7 +654,7 @@ typedef struct PcmXLocalHolderProgress { uint32 reserved; } PcmXLocalHolderProgress; -StaticAssertDecl(sizeof(PcmXLocalHolderProgress) == 160, "PCM-X local holder progress ABI"); +StaticAssertDecl(sizeof(PcmXLocalHolderProgress) == 168, "PCM-X local holder progress ABI"); typedef struct PcmXLocalCutoff { PcmXSlotRef tag_slot; @@ -750,7 +799,7 @@ StaticAssertDecl(sizeof(PcmXLocalFollowerWfgSnapshot) == 248, "PCM-X local follower WFG snapshot process-local ABI"); StaticAssertDecl(offsetof(PcmXLocalFollowerWfgSnapshot, waiter_graph_generation) == 192, "PCM-X local follower sampled graph offset"); -StaticAssertDecl(sizeof(PcmXLocalWriterClaim) == 144, "PCM-X local writer claim process-local ABI"); +StaticAssertDecl(sizeof(PcmXLocalWriterClaim) == 152, "PCM-X local writer claim process-local ABI"); StaticAssertDecl(offsetof(PcmXLocalWriterClaim, claim_generation) == 128, "PCM-X local writer claim generation offset"); StaticAssertDecl(sizeof(PcmXLocalCutoff) == 48, "PCM-X local cutoff process-local ABI"); @@ -920,6 +969,12 @@ typedef struct PcmXMasterTicketSlot { uint32 involved_nodes_bitmap; uint32 drained_nodes_bitmap; uint32 retire_acked_nodes_bitmap; + /* A' rebase (t/400 item 2): effective grant base published exactly once by + * INSTALL_READY when an interleaved revoke consumed the enqueue-time + * base_own_generation on the requester node. Zero = no drift; the wire + * ref identity stays immutable and every exact committed==base+1 check + * runs against the effective base instead. */ + uint64 grant_base_own_generation; } PcmXMasterTicketSlot; typedef struct PcmXBlockerSlot { @@ -967,10 +1022,16 @@ typedef struct PcmXLocalTagSlot { uint64 committed_own_generation; PcmXTicketRef holder_ref; PcmXImageToken holder_image; + uint64 holder_required_page_scn; PcmXReliableLegState holder_reliable; uint64 holder_terminal_drain_generation; PcmXTicketRef blocker_snapshot_ref; PcmXReliableLegState blocker_snapshot_reliable; + /* A' rebase (t/400 item 2): effective grant base published exactly once + * (pre-reservation) when an interleaved revoke consumed the enqueue-time + * base_own_generation. Zero = no drift. Writer-round evidence: cleared + * with image/committed_own_generation, never with the holder lane. */ + uint64 grant_base_own_generation; } PcmXLocalTagSlot; typedef struct PcmXLocalMembershipSlot { @@ -1002,9 +1063,13 @@ StaticAssertDecl(sizeof(PcmXReliableLegState) == 56, "PCM-X reliable leg ABI"); StaticAssertDecl(sizeof(PcmXMasterTagSlot) == 120, "PCM-X master tag slot ABI"); StaticAssertDecl(offsetof(PcmXMasterTagSlot, outstanding_ticket_count) == 112, "PCM-X master outstanding-ticket count offset"); -StaticAssertDecl(sizeof(PcmXMasterTicketSlot) == 384, "PCM-X master ticket slot ABI"); +StaticAssertDecl(sizeof(PcmXMasterTicketSlot) == 392, "PCM-X master ticket slot ABI"); +StaticAssertDecl(offsetof(PcmXMasterTicketSlot, grant_base_own_generation) == 384, + "PCM-X master ticket grant-base offset"); StaticAssertDecl(sizeof(PcmXBlockerSlot) == 128, "PCM-X blocker slot ABI"); -StaticAssertDecl(sizeof(PcmXLocalTagSlot) == 752, "PCM-X local tag slot ABI"); +StaticAssertDecl(sizeof(PcmXLocalTagSlot) == 768, "PCM-X local tag slot ABI"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, grant_base_own_generation) == 760, + "PCM-X local tag grant-base offset"); StaticAssertDecl(offsetof(PcmXLocalTagSlot, membership_count) == 384, "PCM-X local membership count offset"); StaticAssertDecl(offsetof(PcmXLocalTagSlot, closed_round_member_count) == 392, @@ -1016,13 +1081,15 @@ StaticAssertDecl(offsetof(PcmXLocalTagSlot, committed_own_generation) == 408, StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_ref) == 416, "PCM-X local holder ticket offset"); StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_image) == 504, "PCM-X local holder image offset"); -StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_reliable) == 544, +StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_required_page_scn) == 544, + "PCM-X local holder source-floor offset"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_reliable) == 552, "PCM-X local holder reliable-leg offset"); -StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_terminal_drain_generation) == 600, +StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_terminal_drain_generation) == 608, "PCM-X local holder terminal drain generation offset"); -StaticAssertDecl(offsetof(PcmXLocalTagSlot, blocker_snapshot_ref) == 608, +StaticAssertDecl(offsetof(PcmXLocalTagSlot, blocker_snapshot_ref) == 616, "PCM-X local blocker snapshot ticket offset"); -StaticAssertDecl(offsetof(PcmXLocalTagSlot, blocker_snapshot_reliable) == 696, +StaticAssertDecl(offsetof(PcmXLocalTagSlot, blocker_snapshot_reliable) == 704, "PCM-X local blocker snapshot reliable-leg offset"); StaticAssertDecl(sizeof(PcmXLocalMembershipSlot) == 168, "PCM-X local membership slot ABI"); StaticAssertDecl(offsetof(PcmXLocalMembershipSlot, admitted_round) == 160, @@ -1120,6 +1187,9 @@ typedef struct PcmXStats { pg_atomic_uint64 own_abort_count; pg_atomic_uint64 own_busy_count; pg_atomic_uint64 own_corrupt_count; + /* t/400 L3 item 3: BARRIER_CLOSED refusals handed back to a + * barrier-aware LockBuffer caller instead of escaping as ERROR. */ + pg_atomic_uint64 barrier_unwind_count; } PcmXStats; /* Process-local copy used by debug views and acceptance gates. */ @@ -1155,6 +1225,7 @@ typedef struct PcmXStatsSnapshot { uint64 own_abort_count; uint64 own_busy_count; uint64 own_corrupt_count; + uint64 barrier_unwind_count; } PcmXStatsSnapshot; /* Optional peer authority installed while the runtime gate is ACTIVATING. */ @@ -1177,6 +1248,9 @@ typedef struct PcmXOutboundTargetFrontier { uint64 next_prehandle_sequence; } PcmXOutboundTargetFrontier; +/* Bytes reserved for the "file:line" fail-closed provenance string. */ +#define PCM_X_FAIL_CLOSED_SITE_LEN 80 + typedef struct PcmXShmemHeader { PcmXShmemLayout layout; PcmXAllocatorState allocator[PCM_X_ALLOC_COUNT]; @@ -1197,13 +1271,47 @@ typedef struct PcmXShmemHeader { PcmXPeerFrontier peer_frontiers[PCM_X_PROTOCOL_NODE_LIMIT]; PcmXStats stats; PcmXOutboundTargetFrontier outbound_targets[PCM_X_PROTOCOL_NODE_LIMIT]; + /* "file:line" of the call site that most recently won the transition into + * PCM_X_RUNTIME_RECOVERY_BLOCKED. Single-writer by construction: only the + * CAS winner inside pcm_x_runtime_fail_closed_at() writes it, and two + * successful transitions are always separated by a full reactivation. + * Readers are diagnostic-only (pg_cluster_state dump) and tolerate an + * in-progress overwrite after such a reactivation. */ + char fail_closed_site[PCM_X_FAIL_CLOSED_SITE_LEN]; + /* Terminal-kick termination evidence: the arm (PCM_X_TERMINAL_NOTE_*) and + * result that ended the most recent terminal kick on this node, with a + * same-shape repetition count. Last-writer-wins diagnostics only. */ + uint32 terminal_note_op; + uint32 terminal_note_result; + uint32 terminal_note_count; + uint32 terminal_note_reserved; + uint64 terminal_note_ticket; + /* A' rebase: nonzero only when EVERY member of the bound formation + * advertised PCM_X_REBASE_V1 at activation. Written by the activating + * formation tick strictly before the ACTIVE gate publish and immutable + * for the runtime's lifetime; readers consume it through the snapshot's + * double-gate sample. Gates rebase publication and the V2 INSTALL_READY + * frame, never the base protocol. */ + uint32 rebase_wire_active; + uint32 rebase_reserved; } PcmXShmemHeader; +/* Terminal-kick note arms (diagnostic identifiers, not protocol state). */ +#define PCM_X_TERMINAL_NOTE_NONE 0 +#define PCM_X_TERMINAL_NOTE_DETACH 1 +#define PCM_X_TERMINAL_NOTE_CANCEL_CLEANUP 2 +#define PCM_X_TERMINAL_NOTE_ARM_DRAIN 3 +#define PCM_X_TERMINAL_NOTE_ARM_RETIRE 4 +#define PCM_X_TERMINAL_NOTE_DRAIN_STAGE 5 +#define PCM_X_TERMINAL_NOTE_RETIRE_STAGE 6 +#define PCM_X_TERMINAL_NOTE_RETIRE_LOCAL 7 +#define PCM_X_TERMINAL_NOTE_RETIRE_ACK_RESOLVE 8 + StaticAssertDecl(sizeof(PcmXShmemLayout) == 440, "PCM-X shmem layout ABI"); StaticAssertDecl(sizeof(PcmXAllocatorState) == 32, "PCM-X allocator state ABI"); StaticAssertDecl(sizeof(PcmXPeerFrontier) == 48, "PCM-X peer frontier ABI"); -StaticAssertDecl(sizeof(PcmXStats) == 176, "PCM-X stats ABI"); -StaticAssertDecl(sizeof(PcmXStatsSnapshot) == 224, "PCM-X stats snapshot ABI"); +StaticAssertDecl(sizeof(PcmXStats) == 184, "PCM-X stats ABI"); +StaticAssertDecl(sizeof(PcmXStatsSnapshot) == 232, "PCM-X stats snapshot ABI"); StaticAssertDecl(sizeof(PcmXPeerBinding) == 16, "PCM-X peer binding ABI"); StaticAssertDecl(sizeof(PcmXOutboundTargetFrontier) == 32, "PCM-X outbound target frontier ABI"); StaticAssertDecl(offsetof(PcmXOutboundTargetFrontier, mint_gate) == 0, @@ -1221,9 +1329,9 @@ StaticAssertDecl(offsetof(PcmXShmemHeader, local_locks) == 17280, "PCM-X local l StaticAssertDecl(offsetof(PcmXShmemHeader, peer_frontiers) == 33664, "PCM-X peer frontier array offset"); StaticAssertDecl(offsetof(PcmXShmemHeader, stats) == 35200, "PCM-X stats offset"); -StaticAssertDecl(offsetof(PcmXShmemHeader, outbound_targets) == 35376, +StaticAssertDecl(offsetof(PcmXShmemHeader, outbound_targets) == 35384, "PCM-X outbound target frontier array offset"); -StaticAssertDecl(sizeof(PcmXShmemHeader) == 36400, "PCM-X shmem header ABI"); +StaticAssertDecl(sizeof(PcmXShmemHeader) == 36520, "PCM-X shmem header ABI"); typedef enum PcmXAttachResult { PCM_X_ATTACH_OK = 0, @@ -1267,13 +1375,25 @@ extern PcmXRuntimeSnapshot cluster_pcm_x_runtime_snapshot(void); extern bool cluster_pcm_x_stats_snapshot(PcmXStatsSnapshot *snapshot_out); extern void cluster_pcm_x_stats_note_enqueue(void); extern void cluster_pcm_x_stats_note_wait(void); +typedef bool (*PcmXPreSleepRevalidateCallback)(void *callback_arg); +typedef PcmXQueueResult (*PcmXPreSleepResultCallback)(void *callback_arg); +typedef void (*PcmXWaitCallback)(void *callback_arg); +extern bool cluster_pcm_x_requester_wait_once(PcmXPreSleepRevalidateCallback revalidate, + PcmXWaitCallback wait, void *callback_arg); +extern PcmXQueueResult +cluster_pcm_x_requester_wait_once_result(PcmXPreSleepResultCallback revalidate, + PcmXWaitCallback wait, void *callback_arg); extern void cluster_pcm_x_stats_note_queue_result(PcmXQueueResult result); extern void cluster_pcm_x_stats_note_own_begin(void); extern void cluster_pcm_x_stats_note_own_commit(void); extern void cluster_pcm_x_stats_note_own_abort(void); extern void cluster_pcm_x_stats_note_own_busy(void); extern void cluster_pcm_x_stats_note_own_corrupt(void); +extern void cluster_pcm_x_stats_note_barrier_unwind(void); extern bool cluster_pcm_x_runtime_activate(uint64 master_session_incarnation); +/* A' rebase: publish formation-wide PCM_X_REBASE_V1 coverage. Legal only + * from the activating formation tick, strictly before activate_bound(). */ +extern void cluster_pcm_x_runtime_set_rebase_wire_active(bool active); extern bool cluster_pcm_x_runtime_activate_bound(uint64 master_session_incarnation, const PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_LIMIT]); @@ -1284,8 +1404,24 @@ extern bool cluster_pcm_x_runtime_reset_activating(uint32 expected_gate_generati extern bool cluster_pcm_x_runtime_transition(PcmXRuntimeState expected, PcmXRuntimeState desired); /* Cross-layer fail-closed seam for adapters that have already consumed an * irreversible external side effect and cannot safely retry it as ordinary - * queue work. */ -extern void cluster_pcm_x_runtime_fail_closed(void); + * queue work. The macro records the fusing arm (file:line) so a fused node + * can be diagnosed post-mortem without per-site plumbing. */ +extern void cluster_pcm_x_runtime_fail_closed_at(const char *site_file, int site_line); +#define cluster_pcm_x_runtime_fail_closed() cluster_pcm_x_runtime_fail_closed_at(__FILE__, __LINE__) +/* Copies the recorded fail-closed site into buf; false when never fused. */ +extern bool cluster_pcm_x_runtime_fail_closed_site(char *buf, Size buflen); +/* Diagnostic iterators over live master tag / occupied ticket slots for the + * pg_cluster_state dump; racy-tolerant, never used for protocol decisions. */ +extern bool cluster_pcm_x_master_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, + Size buflen); +extern bool cluster_pcm_x_master_ticket_debug_next(Size *cursor_io, Size *index_out, char *buf, + Size buflen); +extern bool cluster_pcm_x_local_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, + Size buflen); +/* Terminal-kick termination note: which arm/result ended the latest kick. */ +extern void cluster_pcm_x_terminal_note(uint32 op, uint32 result, uint64 ticket_id); +extern bool cluster_pcm_x_terminal_note_read(uint32 *op_out, uint32 *result_out, uint64 *ticket_out, + uint32 *count_out); extern PcmXStepResult cluster_pcm_x_master_step(PcmXMasterTicketState current, PcmXMasterEvent event, uint32 guards, PcmXMasterTicketState *next); @@ -1325,6 +1461,19 @@ extern PcmXQueueResult cluster_pcm_x_master_drive_work_next(Size *cursor_io, Siz extern PcmXQueueResult cluster_pcm_x_master_drive_snapshot_exact(const BufferTag *tag, uint64 cluster_epoch, PcmXMasterDriveSnapshot *snapshot_out); +extern PcmXQueueResult +cluster_pcm_x_master_commit_retry_exact(const PcmXMasterDriveSnapshot *expected, uint64 now_ms, + uint64 next_retry_deadline_ms, PcmXPhasePayload *commit_out, + PcmXMasterDriveSnapshot *snapshot_out); +extern PcmXQueueResult cluster_pcm_x_master_invalidate_busy_backoff_exact( + const PcmXMasterDriveSnapshot *expected, int32 busy_node, uint64 now_ms, uint64 retry_delay_ms, + PcmXMasterDriveSnapshot *snapshot_out); +typedef bool (*PcmXStageFrameCallback)(uint8 msg_type, int32 dest_node_id, const void *payload, + Size payload_len, void *callback_arg); +extern PcmXQueueResult +cluster_pcm_x_master_commit_retry_drive(const PcmXMasterDriveSnapshot *expected, uint64 now_ms, + uint64 retry_delay_ms, PcmXStageFrameCallback stage_frame, + void *callback_arg, PcmXMasterDriveSnapshot *snapshot_out); extern PcmXQueueResult cluster_pcm_x_master_drive_bitmap_replace_exact( const PcmXMasterDriveSnapshot *expected, uint32 pending_s_holders_bitmap, uint32 acked_s_holders_bitmap, PcmXMasterDriveSnapshot *snapshot_out); @@ -1378,10 +1527,9 @@ extern PcmXQueueResult cluster_pcm_x_master_image_ready_exact(const PcmXGrantPay int32 authenticated_node, uint64 authenticated_session, PcmXGrantPayload *prepare_out); -extern PcmXQueueResult -cluster_pcm_x_master_install_ready_exact(const PcmXInstallReadyPayload *install_ready, - int32 authenticated_node, uint64 authenticated_session, - PcmXPhasePayload *commit_out); +extern PcmXQueueResult cluster_pcm_x_master_install_ready_exact( + const PcmXInstallReadyPayload *install_ready, uint64 rebased_own_generation, + int32 authenticated_node, uint64 authenticated_session, PcmXPhasePayload *commit_out); extern PcmXQueueResult cluster_pcm_x_master_final_ack_exact(const PcmXFinalAckPayload *final_ack, int32 authenticated_node, uint64 authenticated_session, @@ -1424,6 +1572,8 @@ extern PcmXQueueResult cluster_pcm_x_master_retire_ack_resolve_exact(const PcmXR uint64 authenticated_session, PcmXTicketRef *ref_out); extern PcmXQueueResult cluster_pcm_x_master_terminal_work_next(PcmXTicketRef *ref_out); +extern PcmXQueueResult cluster_pcm_x_master_terminal_work_next_after(uint64 after_ticket_id, + PcmXTicketRef *ref_out); extern PcmXQueueResult cluster_pcm_x_master_detach_terminal_exact(const PcmXTicketRef *ref); extern PcmXQueueResult cluster_pcm_x_local_join_begin(const PcmXWaitIdentity *identity, int32 master_node, @@ -1489,9 +1639,16 @@ extern PcmXQueueResult cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, int32 authenticated_master_node, uint64 authenticated_master_session); +extern PcmXQueueResult cluster_pcm_x_local_holder_revoke_apply_floor_exact( + const PcmXRevokePayload *revoke, uint64 required_page_scn, int32 authenticated_master_node, + uint64 authenticated_master_session); extern PcmXQueueResult cluster_pcm_x_local_holder_image_ready_arm_exact( const PcmXGrantPayload *image_ready, int32 authenticated_master_node, uint64 authenticated_master_session, PcmXGrantPayload *replay_out); +extern PcmXQueueResult cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( + const PcmXGrantPayload *image_ready, int32 authenticated_master_node, + uint64 authenticated_master_session, PcmXGrantPayload *replay_out, + PcmXLocalImageReadyRefusal *refusal_out); extern PcmXQueueResult cluster_pcm_x_local_enqueue_arm_exact(const PcmXLocalHandle *leader, PcmXEnqueuePayload *payload_out, PcmXLocalReliableToken *token_out); @@ -1511,6 +1668,12 @@ extern PcmXQueueResult cluster_pcm_x_local_prepare_grant_exact(const PcmXLocalHa const PcmXGrantPayload *prepare, int32 authenticated_node, uint64 authenticated_session); +/* A' rebase: one-shot pre-reservation publication of the effective grant base + * for the prepared transfer. Same value replays as DUPLICATE; a different + * second value is evidence divergence and fails closed. */ +extern PcmXQueueResult +cluster_pcm_x_local_grant_rebase_publish_exact(const PcmXLocalHandle *leader, + uint64 rebased_own_generation); extern PcmXQueueResult cluster_pcm_x_local_install_ready_arm_exact( const PcmXLocalHandle *leader, const PcmXTicketRef *expected_ref, const PcmXImageToken *expected_image, PcmXInstallReadyPayload *install_ready_out, @@ -1555,6 +1718,24 @@ extern PcmXQueueResult cluster_pcm_x_local_terminal_publish_exact( extern PcmXQueueResult cluster_pcm_x_local_drain_poll_exact(const PcmXDrainPollPayload *poll, int32 authenticated_master_node, uint64 authenticated_master_session); + +/* Process-local completion certificate captured under the tag lock by the + * first (writer-lane) DRAIN consumption. The exact protocol ledger - not + * the live buffer descriptor, whose per-buf_id generation may legitimately + * move on or vanish behind eviction - is the release authority for the + * self-source immutable image record. Never persisted or sent on wire. */ +typedef struct PcmXLocalDrainCertificate { + PcmXTicketRef ref; + PcmXImageToken image; + uint64 committed_own_generation; + uint64 master_session_incarnation; + int32 master_node; + bool valid; +} PcmXLocalDrainCertificate; + +extern PcmXQueueResult cluster_pcm_x_local_drain_poll_certificate_exact( + const PcmXDrainPollPayload *poll, int32 authenticated_master_node, + uint64 authenticated_master_session, PcmXLocalDrainCertificate *certificate_out); extern PcmXQueueResult cluster_pcm_x_local_retire_up_to_exact(const PcmXRetirePayload *request, int32 authenticated_master_node, uint64 authenticated_master_session); @@ -1632,6 +1813,9 @@ extern PcmXQueueResult cluster_pcm_x_local_probe_freeze_snapshot_exact( * ForEachLWLockHeldByMe callback, then rejects any tag whose local PROBE * barrier is closed. BUSY/CORRUPT are closed decisions, never "inactive". */ extern PcmXQueueResult cluster_pcm_x_nested_wait_guard_before_block(void); +/* Diagnostic: the held content-lock tag and result of this backend's most + * recent nested-guard refusal; false when the guard never refused. */ +extern bool cluster_pcm_x_nested_wait_guard_last_block(BufferTag *tag_out, int *result_out); extern PcmXQueueResult cluster_pcm_x_local_holder_snapshot_revalidate(const BufferTag *tag, const PcmXLocalHolderSnapshot *snapshot); diff --git a/src/include/cluster/cluster_pcm_x_image_fetch.h b/src/include/cluster/cluster_pcm_x_image_fetch.h index d86b18f170..c7ada654b7 100644 --- a/src/include/cluster/cluster_pcm_x_image_fetch.h +++ b/src/include/cluster/cluster_pcm_x_image_fetch.h @@ -24,6 +24,18 @@ #ifdef USE_PGRAC_CLUSTER +typedef enum PcmXImageFetchRequestRefusal { + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_NONE = 0, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_ARGUMENT, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_ENVELOPE, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_REQUEST, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_MASTER, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_REF, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_LEG, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_IMAGE, + PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_REQUESTER_ID +} PcmXImageFetchRequestRefusal; + extern bool cluster_pcm_x_image_fetch_build_request(const PcmXLocalProgress *progress, int32 requester_node, int32 requester_backend_id, @@ -33,6 +45,10 @@ extern bool cluster_pcm_x_image_fetch_request_exact(const ClusterICEnvelope *env const PcmXLocalHolderProgress *holder, int32 holder_node, int32 current_master_node, uint64 current_epoch); +extern bool cluster_pcm_x_image_fetch_request_exact_diagnosed( + const ClusterICEnvelope *env, const GcsBlockRequestPayload *request, + const PcmXLocalHolderProgress *holder, int32 holder_node, int32 current_master_node, + uint64 current_epoch, PcmXImageFetchRequestRefusal *refusal_out); extern bool cluster_pcm_x_image_fetch_reply_exact(const GcsBlockReplyHeader *reply, const char *block_data, const PcmXLocalProgress *progress, diff --git a/src/include/cluster/cluster_sf_dep.h b/src/include/cluster/cluster_sf_dep.h index a02d0e4ef3..24acea633d 100644 --- a/src/include/cluster/cluster_sf_dep.h +++ b/src/include/cluster/cluster_sf_dep.h @@ -103,6 +103,40 @@ cluster_sf_peer_cap_generation_for_bits(const ClusterSfPeerCap *cap, uint32 requ return true; } +/* review P0-2: one record-coherent sample of a capability family. True iff + * every required bit is present on the CURRENT connection's record; the + * optional-bit presence and the record generation come from the SAME read, + * so a caller sampling this on both sides of an authority pass rejects any + * reconnect in between (the generation moves) instead of pairing a stale + * connection's optional bit with a fresh session. False zeroes both + * outputs. */ +static inline bool +cluster_sf_peer_cap_family_sample(const ClusterSfPeerCap *cap, uint32 required_bits, + uint32 optional_bits, bool *optional_out, uint32 *generation_out) +{ + if (optional_out != NULL) + *optional_out = false; + if (!cluster_sf_peer_cap_generation_for_bits(cap, required_bits, generation_out)) + return false; + if (optional_out != NULL) + *optional_out = (cluster_sf_peer_cap_bits(cap) & optional_bits) == optional_bits + && optional_bits != 0; + return true; +} + +/* Exact pre-send fence for a capability-bound frame. Both the required + * bits and generation are read from one immutable record snapshot by the + * caller holding the capability-store lock. */ +static inline bool +cluster_sf_peer_cap_generation_matches_exact(const ClusterSfPeerCap *cap, uint32 required_bits, + uint32 expected_generation) +{ + uint32 observed_generation; + + return cluster_sf_peer_cap_generation_for_bits(cap, required_bits, &observed_generation) + && observed_generation == expected_generation; +} + /* Returns true iff the record was live for exactly this generation and got * cleared; a mismatch (older/newer generation, already invalid) is a no-op. */ static inline bool @@ -236,6 +270,17 @@ extern bool cluster_sf_peer_supports_xid_native_disable(int32 peer_id); extern bool cluster_sf_peer_supports_xid_authority_flock(int32 peer_id); extern bool cluster_sf_peer_supports_gcs_inval_busy(int32 peer_id); extern bool cluster_sf_peer_supports_pcm_x_convert(int32 peer_id); +extern bool cluster_sf_peer_supports_pcm_x_rebase(int32 peer_id); +extern bool cluster_sf_peer_supports_pcm_x_source_floor(int32 peer_id); +extern bool cluster_sf_peer_pcm_x_source_floor_sample(int32 peer_id, bool *source_floor_out, + uint32 *generation_out); +extern bool cluster_sf_peer_capability_generation_matches(int32 peer_id, + uint32 required_capabilities, + uint32 expected_generation); +/* review P0-2: lock-coherent (CONVERT supported, REBASE bit, record + * generation) triple for the formation collector's double sample. */ +extern bool cluster_sf_peer_pcm_x_capability_sample(int32 peer_id, bool *rebase_out, + uint32 *generation_out); extern bool cluster_sf_peer_pcm_x_connection_generation(int32 peer_id, uint32 *generation); extern void cluster_sf_note_peer_disconnected_gen(int32 peer_id, uint32 generation); extern void cluster_sf_note_peer_disconnected(int32 peer_id); diff --git a/src/include/cluster/cluster_tt_local.h b/src/include/cluster/cluster_tt_local.h index ffd051ec5e..fc3286eb67 100644 --- a/src/include/cluster/cluster_tt_local.h +++ b/src/include/cluster/cluster_tt_local.h @@ -39,6 +39,7 @@ #include "c.h" #include "access/transam.h" +#include "cluster/cluster_itl_slot.h" /* UBA */ #include "cluster/cluster_scn.h" /* SCN */ #include "cluster/cluster_tt_status.h" /* ClusterTTStatus */ @@ -107,6 +108,15 @@ extern void cluster_tt_local_record_abort(TransactionId xid); */ extern void cluster_tt_local_record_active(TransactionId xid); +/* + * Publish ACTIVE for an ordinary heap data write using the exact UBA carried + * by the page's ITL slot. Record-extent rollover can make the UBA's record + * segment differ from the transaction binding's canonical TT segment; this + * entry point registers that exact-key alias so commit/abort later publish the + * same terminal status to it. `uba` must carry the binding's slot offset. + */ +extern void cluster_tt_local_record_data_active(TransactionId xid, UBA uba); + /* * Counters / introspection used by test_cluster_tt_status (D9 T17). */ diff --git a/src/include/cluster/cluster_visibility_resolve.h b/src/include/cluster/cluster_visibility_resolve.h index 8efec94a65..50dfb0bbd8 100644 --- a/src/include/cluster/cluster_visibility_resolve.h +++ b/src/include/cluster/cluster_visibility_resolve.h @@ -184,6 +184,10 @@ extern ClusterVisVerdict cluster_vis_self_verdict(ClusterTTStatus status); extern ClusterVisVerdict cluster_vis_toast_verdict(ClusterTTStatus status); /* OBS-2 Update: xmin gate then xmax outcome. */ +extern bool cluster_vis_xmin_needs_resolution(uint16 infomask); + +/* P0-28: local GlobalVis is not a shared-storage pruning horizon. */ +extern bool cluster_vis_prune_must_defer(bool storage_mode, bool cluster_horizon_available); extern ClusterVisVerdict cluster_vis_update_xmin_verdict(ClusterTTStatus status); extern ClusterVisVerdict cluster_vis_update_xmax_verdict(ClusterTTStatus status, bool is_delete); diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index 2a0a63e2e7..6fecd6fbf2 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -243,6 +243,9 @@ extern void MarkBufferDirtyHint(Buffer buffer, bool buffer_std); extern void UnlockBuffers(void); extern void LockBuffer(Buffer buffer, int mode); +/* PGRAC: EXCLUSIVE lock that hands a nested-guard BARRIER_CLOSED refusal + * back to the caller (false, nothing held) instead of raising an ERROR. */ +extern bool ClusterLockBufferExclusiveBarrierAware(Buffer buffer); /* PGRAC: operation-scoped PCM-X direct-init entrances for zero VM/FSM pages. */ extern void LockBufferForVisibilityMapPageInit(Buffer buffer); extern void LockBufferForFreeSpaceMapPageInit(Buffer buffer); diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index 0651600fe3..d8f96d0211 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -57,7 +57,7 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '183', 'pg_stat_cluster_injections returns 183 rows (branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-writer-cached-x-stall + cluster-pcm-restore-aba-window + cluster-pcm-grant-finalize-window + cluster-pcm-grant-finalize-deliver-invalidate + cluster-pcm-drop-prepin-window + cluster-pcm-restore-aba-force-round; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 horizon report-drop/epoch-fence; spec-5.22d D4-8 +1 authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '184', 'pg_stat_cluster_injections returns 184 rows (P0-20 +1 cluster-pcm-x-retain-flush-error; branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-writer-cached-x-stall + cluster-pcm-restore-aba-window + cluster-pcm-grant-finalize-window + cluster-pcm-grant-finalize-deliver-invalidate + cluster-pcm-drop-prepin-window + cluster-pcm-restore-aba-force-round; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 horizon report-drop/epoch-fence; spec-5.22d D4-8 +1 authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- @@ -85,8 +85,8 @@ 'postgres', 'SELECT string_agg(name, \',\' ORDER BY name) FROM pg_stat_cluster_injections' ), - 'cluster-boc-event-publish,cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-done-drop,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-duplicate-grant-reply,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-fallback-refresh-stale,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-master-direct-fallback-storage-stale,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-stale-ship-resident,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-gcs-block-yield-notify-drop,cluster-gcs-xfer-copy-drop-window,cluster-ges-master-work-queue-full,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-conn-reset,cluster-lms-cr-construct,cluster-lms-cr-fence-recheck,cluster-lms-cr-fence-refuse,cluster-lms-data-dispatch,cluster-lms-undo-fetch,cluster-mxid-halfspace-hard-limit,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-drop-prepin-window,cluster-pcm-grant-finalize-deliver-invalidate,cluster-pcm-grant-finalize-window,cluster-pcm-release-pre,cluster-pcm-restore-aba-force-round,cluster-pcm-restore-aba-window,cluster-pcm-writer-cached-x-stall,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-undo-authority-block0-prove,cluster-undo-authority-scan,cluster-undo-horizon-epoch-fence,cluster-undo-horizon-report-drop,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', - '180 injection point names match the full registry (S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 cluster-undo-horizon-report-drop + cluster-undo-horizon-epoch-fence; spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + 'cluster-boc-event-publish,cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-done-drop,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-duplicate-grant-reply,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-fallback-refresh-stale,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-master-direct-fallback-storage-stale,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-stale-ship-resident,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-gcs-block-yield-notify-drop,cluster-gcs-xfer-copy-drop-window,cluster-ges-master-work-queue-full,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-conn-reset,cluster-lms-cr-construct,cluster-lms-cr-fence-recheck,cluster-lms-cr-fence-refuse,cluster-lms-data-dispatch,cluster-lms-undo-fetch,cluster-mxid-halfspace-hard-limit,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-drop-prepin-window,cluster-pcm-grant-finalize-deliver-invalidate,cluster-pcm-grant-finalize-window,cluster-pcm-release-pre,cluster-pcm-restore-aba-force-round,cluster-pcm-restore-aba-window,cluster-pcm-writer-cached-x-stall,cluster-pcm-x-retain-flush-error,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-undo-authority-block0-prove,cluster-undo-authority-scan,cluster-undo-horizon-epoch-fence,cluster-undo-horizon-report-drop,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', + '184 injection point names match the full registry (P0-20 +1 cluster-pcm-x-retain-flush-error; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2 cluster-undo-horizon-report-drop + cluster-undo-horizon-epoch-fence; spec-5.22d D4-8 +1 cluster-undo-authority-block0-prove; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/017_debug.pl b/src/test/cluster_tap/t/017_debug.pl index 1f217dabb4..3f8fa9189c 100644 --- a/src/test/cluster_tap/t/017_debug.pl +++ b/src/test/cluster_tap/t/017_debug.pl @@ -128,15 +128,15 @@ 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.fault_type'}), - '183', - 'all 183 injection points have a .fault_type entry (branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '184', + 'all 184 injection points have a .fault_type entry (P0-20 +1 cluster-pcm-x-retain-flush-error; branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is( $node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.hits'}), - '183', - 'all 183 injection points have a .hits entry (branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '184', + 'all 184 injection points have a .hits entry (P0-20 +1 cluster-pcm-x-retain-flush-error; branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; merge sum with 7.x lanes) under inject category (spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/024_pcm_lock.pl b/src/test/cluster_tap/t/024_pcm_lock.pl index 4d8520341e..1a26f2b098 100644 --- a/src/test/cluster_tap/t/024_pcm_lock.pl +++ b/src/test/cluster_tap/t/024_pcm_lock.pl @@ -55,14 +55,14 @@ # ---------- -# L1: pg_cluster_state.pcm category has 58 keys, including the 30-key PCM-X FIFO surface. +# L1: pg_cluster_state.pcm category has 60 keys, including the 32-key PCM-X FIFO surface. # activates the state-machine diagnostics. # ---------- is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='pcm'}), - '58', - 'L1 pg_cluster_state.pcm category has 58 keys (existing 28 + PCM-X FIFO/ownership/runtime 30)'); + '60', + 'L1 pg_cluster_state.pcm category has 60 keys (existing 28 + PCM-X FIFO/ownership/runtime 31 + fail-closed site)'); # ---------- diff --git a/src/test/cluster_tap/t/030_acceptance.pl b/src/test/cluster_tap/t/030_acceptance.pl index 28f1736087..78788b9e8b 100644 --- a/src/test/cluster_tap/t/030_acceptance.pl +++ b/src/test/cluster_tap/t/030_acceptance.pl @@ -325,12 +325,12 @@ # ============================================================ -# §M error injection 112 注入点 + 5 fault types (6 tests) +# §M error injection 184 注入点 + 7 fault types (6 tests) # ============================================================ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '183', 'M1 183 injection points (branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + '184', 'M1 184 injection points (P0-20 +1 cluster-pcm-x-retain-flush-error; branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql('postgres', q{SELECT string_agg(name, ',' ORDER BY name) FROM pg_stat_cluster_injections WHERE name LIKE 'cluster-init-%'}), @@ -360,8 +360,8 @@ 'postgres', q{SELECT count(DISTINCT key) FROM pg_cluster_state WHERE category='inject' AND (key LIKE '%.fault_type' OR key LIKE '%.hits')} - ) eq '366', - 'M5 inject category has 183×2 = 366 sub-keys (.fault_type + .hits; branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + ) eq '368', + 'M5 inject category has 184×2 = 368 sub-keys (.fault_type + .hits; P0-20 +1 cluster-pcm-x-retain-flush-error; branch-1 +2 cluster-gcs-block-stale-ship-resident + cluster-gcs-block-master-direct-fallback-storage-stale; S3 forensics step 1a +1 cluster-ges-master-work-queue-full; ownership-gen wave +6 cluster-pcm-*; serve-stall round-6 +1 cluster-gcs-xfer-copy-drop-window; gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->get_cluster_state_value('inject', 'armed_count'), '0', 'M6 inject.armed_count starts at 0 in fresh backend'); diff --git a/src/test/cluster_tap/t/108_pcm_state_machine.pl b/src/test/cluster_tap/t/108_pcm_state_machine.pl index 0081dafc7b..e0a885f1ae 100644 --- a/src/test/cluster_tap/t/108_pcm_state_machine.pl +++ b/src/test/cluster_tap/t/108_pcm_state_machine.pl @@ -48,8 +48,8 @@ my $pcm_category_rows = $node_default->safe_psql( 'postgres', "SELECT count(*) FROM pg_cluster_state WHERE category = 'pcm'"); -is($pcm_category_rows, '58', - 'L1 pg_cluster_state pcm category has 58 rows (existing 28 + PCM-X FIFO/ownership/runtime 30)'); +is($pcm_category_rows, '60', + 'L1 pg_cluster_state pcm category has 60 rows (existing 28 + PCM-X FIFO/ownership/runtime 31 + fail-closed site)'); # L3 — api_state shows "active" when GUC=-1 default my $api_state_default = $node_default->safe_psql( diff --git a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl index 88cca599f6..cc781e1242 100644 --- a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl +++ b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl @@ -17,9 +17,8 @@ # L5 S barrier injection — DENIED_PENDING_X surfaces under # cluster-gcs-block-starvation-force-denied inject; reader # sees starvation_denied_pending_x_count tick -# L6 53R92 ERRCODE_CLUSTER_GCS_BLOCK_STARVATION_EXHAUSTED triggered -# when reader exhausts cluster.gcs_block_starvation_max_retries -# (set to 0 via runtime GUC so first inject denial → ereport) +# L6 a zero legacy starvation budget cannot surface a client error; +# one forced denial exact-aborts and re-enters with fresh identities # L7 53R91 ERRCODE_CLUSTER_GCS_BLOCK_INVALIDATE_TIMEOUT triggered # via cluster-gcs-block-invalidate-stall-ack inject (holder # never acks); budget exhaustion → DENIED_INVALIDATE_TIMEOUT @@ -161,7 +160,7 @@ sub gcs_int "L5 starvation_denied_pending_x_count tick under inject " . "(before=$r1, after=$r2)"); - # L6: 53R92 budget exhaustion — set retries=0 so first deny errors. + # L6: the legacy zero budget cannot turn queue competition into ERROR. $pair->node0->safe_psql('postgres', 'SET cluster.gcs_block_starvation_max_retries = 0'); $pair->inject_skip_set($pair->node0, @@ -173,10 +172,8 @@ sub gcs_int 'cluster-gcs-block-starvation-force-denied', 0); $pair->node0->safe_psql('postgres', 'RESET cluster.gcs_block_starvation_max_retries'); - # Result may be PASS if the local backend's transition doesn't - # route through GCS (single-node ish) — keep cmp_ok permissive. - cmp_ok($rc, '>=', 0, - "L6 starvation retry exhaustion exit_code observed (rc=$rc)"); + is($rc, 0, "L6 pending-X retry stays internal with legacy budget zero") + or diag("L6 stdout=[$stdout] stderr=[$stderr]"); # L7: 53R91 — invalidate ack timeout via stall inject (no X transfer # attempted in 2-node-S baseline, so we just observe the inject is diff --git a/src/test/cluster_tap/t/390_gcs_block_race_convergence_3node.pl b/src/test/cluster_tap/t/390_gcs_block_race_convergence_3node.pl index 76923c2ce0..1d82f13e4e 100644 --- a/src/test/cluster_tap/t/390_gcs_block_race_convergence_3node.pl +++ b/src/test/cluster_tap/t/390_gcs_block_race_convergence_3node.pl @@ -189,10 +189,9 @@ sub read_retry # ============================================================ # L2b (RC-B, injection-deterministic): force ONE DENIED_PENDING_X on a -# cross-node N->S read. The reader's backoff retry reuses the same -# (request_id, epoch) key; the master must re-evaluate it (deny released -# the dedup entry), not swallow it as IN_FLIGHT_DUPLICATE for a full -# cluster.gcs_reply_timeout_ms round. +# cross-node N->S read. The reader exact-aborts the denied reservation and +# re-enters with a fresh request_id/token; the old denial remains cached until +# exact DONE and cannot swallow or authorize the successor identity. # ============================================================ my $base_timeout2 = gcs_sum($triple, 'block_timeout_count'); my $base_starve = gcs_sum($triple, 'starvation_denied_pending_x_count'); diff --git a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl index 1754c03a17..56014ac1ba 100644 --- a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl +++ b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl @@ -66,6 +66,7 @@ sub state_int ); my @pcm_x_lmd_keys = qw( + wait_edge_count pcm_convert_wfg_replace_count pcm_convert_wfg_remove_count pcm_convert_wfg_replace_fail_count @@ -98,8 +99,6 @@ sub state_int my @zero_pcm_failure_keys = qw( pcm_x_queue_cancel_count pcm_x_queue_full_count - pcm_x_queue_stale_count - pcm_x_queue_miss_count pcm_x_queue_recovery_blocked_count pcm_x_queue_activating_reset_count pcm_x_own_abort_count @@ -108,7 +107,6 @@ sub state_int my @positive_wfg_keys = qw( pcm_convert_wfg_replace_count - pcm_convert_wfg_remove_count ); sub exact_key_count @@ -191,6 +189,28 @@ sub wait_for_pcm_gauges_zero return (0, \@snapshots); } +sub wait_for_node_state_gt +{ + my ($node, $category, $key, $before, $timeout_seconds) = @_; + my $deadline = time() + $timeout_seconds; + my $last_value; + + do + { + my ($rc, $out, $err) = $node->psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='$category' AND key='$key'}, + timeout => 3); + if (defined($rc) && $rc == 0 && defined($out) && $out =~ /\A\d+\z/) + { + $last_value = $out + 0; + return (1, $last_value) if $last_value > $before; + } + usleep(100_000); + } while (time() < $deadline); + + return (0, $last_value); +} + my $warmup_error_count = 0; sub write_retry @@ -213,6 +233,30 @@ sub write_retry return 0; } +sub wait_for_lms_finish_flush_reload +{ + my ($node, $log_offset, $expected_workers, $expected_armed, + $expected_value, $timeout_seconds) = @_; + my $deadline = time() + $timeout_seconds; + my $last_log = ''; + + do + { + $last_log = substr(slurp_file($node->logfile), $log_offset); + my %ready_workers; + while ($last_log =~ /cluster_lms: DATA worker=(\d+) applied PCM-X finish Flush injection config: pid=\d+ armed=(true|false) value="([^"]*)"/g) + { + my ($worker_id, $armed, $value) = ($1, $2, $3); + next unless $armed eq $expected_armed && $value eq $expected_value; + $ready_workers{$worker_id} = 1; + } + return (1, $last_log) if scalar(keys %ready_workers) == $expected_workers; + usleep(100_000); + } while (time() < $deadline); + + return (0, $last_log); +} + sub write_file { my ($path, $contents) = @_; @@ -234,6 +278,7 @@ sub write_file 'cluster.xid_striping = on', 'cluster.crossnode_runtime_visibility = on', 'cluster.page_scn_shortcut = on', + 'cluster.past_image = on', 'cluster.crossnode_write_write = on', 'cluster.undo_gcs_coherence = on', 'cluster.crossnode_cr_data_plane = on', @@ -286,6 +331,14 @@ sub write_file CREATE TABLE pcm_xq_self ( id integer, v bigint NOT NULL + ) WITH (fillfactor = 100); + CREATE TABLE pcm_xq_dirty_retain ( + id integer, + v bigint NOT NULL + ) WITH (fillfactor = 100); + CREATE TABLE pcm_xq_flush_error ( + id integer, + v bigint NOT NULL ) WITH (fillfactor = 100) }); } @@ -313,7 +366,58 @@ sub write_file ok(write_retry($quad->node0, q{SELECT count(*) FROM pcm_xq_hot WHERE id BETWEEN 1 AND 4}), 'L2 seed owner installed committed visibility hints'); +my @l2s_checkpoint_log_offsets = map { (-s $quad->node($_)->logfile) // 0 } (0 .. 3); +# The hot main page is guaranteed dirty by seed/VACUUM and hashes to remote +# master node1 on shard1. That single consumed note deterministically proves +# the P0-24 DATA-plane route. The VM page also hashes to node3, but its +# checkpoint write-note is conditional on still holding PCM S/X at flush; N +# is legal, so waiting for node3 would assert a non-contractual side effect. +my %l2s_pi_durable_applied_before = map { + $_ => state_int($quad->node($_), 'gcs', 'pi_durable_note_apply_count') +} (1); +my @l2s_pi_route_before_by_node = map { + { + misroute => state_int($quad->node($_), 'gcs', + 'dedup_misroute_failclosed_count'), + requeue_drop => state_int($quad->node($_), 'lms', + 'lms_outbound_requeue_drop_count'), + } +} (0 .. 3); ok(write_retry($quad->node0, 'CHECKPOINT'), 'L2 seed checkpointed'); +for my $target (1) +{ + my ($applied, $after) = wait_for_node_state_gt( + $quad->node($target), 'gcs', 'pi_durable_note_apply_count', + $l2s_pi_durable_applied_before{$target}, 15); + ok($applied, + "L2S target node$target DATA worker consumed its checkpoint status-3 PI durable note") + or diag("L2S node$target pi_durable_note_apply_count before=" + . $l2s_pi_durable_applied_before{$target} . ' after=' + . (defined($after) ? $after : '')); +} +for my $i (0 .. 3) +{ + my ($checkpoint_probe_rc, $checkpoint_probe_out, $checkpoint_probe_err) + = $quad->node($i)->psql('postgres', 'SELECT 1', timeout => 5); + ok(defined($checkpoint_probe_rc) && $checkpoint_probe_rc == 0 + && defined($checkpoint_probe_out) && $checkpoint_probe_out eq '1', + "L2S node$i survived checkpoint PI durable-note routing") + or diag("L2S node$i checkpoint probe rc=" + . (defined($checkpoint_probe_rc) ? $checkpoint_probe_rc : '') + . " stdout=[" . ($checkpoint_probe_out // '') + . "] stderr=[" . ($checkpoint_probe_err // '') . "]"); + my $checkpoint_log = substr(slurp_file($quad->node($i)->logfile), + $l2s_checkpoint_log_offsets[$i]); + unlike($checkpoint_log, + qr/gcs block invalidate-ack misrouted|failed Assert\("tag_shard == recv_worker"\)/, + "L2S node$i checkpoint emitted no invalidate-ack shard violation"); + is(state_int($quad->node($i), 'gcs', 'dedup_misroute_failclosed_count'), + $l2s_pi_route_before_by_node[$i]{misroute}, + "L2S node$i checkpoint misroute counter stayed exact"); + is(state_int($quad->node($i), 'lms', 'lms_outbound_requeue_drop_count'), + $l2s_pi_route_before_by_node[$i]{requeue_drop}, + "L2S node$i checkpoint outbound requeue-drop counter stayed exact"); +} # Build a deterministic sole-requester S source before the four-writer leg. # The cache-off INSERT above releases node0's X to N at content-lock unlock. @@ -330,6 +434,14 @@ sub write_file 'pcm_x_self_handoff_count'); my $self_handoff_drain_before = state_int($quad->node0, 'gcs', 'pcm_x_self_handoff_drain_count'); +my @self_runtime_before_by_node = map { + { + runtime_state => state_int($quad->node($_), 'pcm', + 'pcm_x_runtime_state'), + recovery_blocked => state_int($quad->node($_), 'pcm', + 'pcm_x_queue_recovery_blocked_count'), + } +} (0 .. 3); my ($self_write_rc, $self_write_out, $self_write_err) = $quad->node0->psql( 'postgres', q{UPDATE pcm_xq_self SET v = v + 1 WHERE id = 1}, timeout => 30); @@ -356,10 +468,116 @@ sub write_file 'L2S sole-requester S source exercised the fused revoke-to-grant handoff'); cmp_ok($self_handoff_drain_after - $self_handoff_drain_before, '>', 0, 'L2S sole-requester S source released its immutable record at DRAIN'); +for my $i (0 .. 3) +{ + is(state_int($quad->node($i), 'pcm', 'pcm_x_runtime_state'), 1, + "L2S node$i PCM-X runtime remained ACTIVE across DRAIN/RETIRE"); + is(state_int($quad->node($i), 'pcm', + 'pcm_x_queue_recovery_blocked_count'), + $self_runtime_before_by_node[$i]{recovery_blocked}, + "L2S node$i recovery-blocked count did not advance at RETIRE"); +} is($quad->node0->safe_psql('postgres', q{SELECT v FROM pcm_xq_self WHERE id = 1}), '1', 'L2S sole-requester conversion preserved exact page contents'); +# Seed and vacuum one target page so node1's later INSERT obtains that exact +# heap page from the FSM and asks for X directly. An UPDATE would first read +# the row into a requester-local S mirror, making the authority selector take +# the intentional self-source path instead of the remote finish path tested +# here. The node0 UPDATE after this checkpoint is the sole dirty source. +ok(write_retry($quad->node0, + q{INSERT INTO pcm_xq_dirty_retain(id, v) VALUES (1, 0)}), + 'L2F seeded the direct-X target page'); +ok(write_retry($quad->node0, 'VACUUM pcm_xq_dirty_retain'), + 'L2F published target-page free space before the direct-X request'); +ok(write_retry($quad->node0, 'CHECKPOINT'), + 'L2F baseline checkpointed before the dirty retain leg'); + +# The source INSERT leaves an uncheckpointed dirty X image on node0. A +# different node must materialize that exact image and retain it only after +# FlushBuffer's caller-owned pin contract is satisfied. This is a behavioral +# gate: the immutable staging counter proves the DATA worker reached the dirty +# remote-source leg, while h_pi_write_note_count increments only after +# FlushBuffer's smgrwrite returned, proving the real flush completed. +my $dirty_stage_before = state_int($quad->node0, 'gcs', + 'dedup_pcm_x_stage_count'); +my $dirty_flush_before = state_int($quad->node0, 'xnode_lever', + 'h_pi_write_note_count'); +my $dirty_flush_log_offset = (-s $quad->node0->logfile) // 0; +my $dirty_requester_log_offset = (-s $quad->node1->logfile) // 0; +my $dirty_lms_workers = $quad->node0->safe_psql('postgres', + 'SHOW cluster.lms_workers') + 0; +my $dirty_requester_lms_workers = $quad->node1->safe_psql('postgres', + 'SHOW cluster.lms_workers') + 0; +$quad->node0->safe_psql('postgres', q{ + ALTER SYSTEM SET cluster.injection_points = 'cluster-pcm-x-retain-flush-error'; + SELECT pg_reload_conf() +}); +$quad->node1->safe_psql('postgres', q{ + ALTER SYSTEM SET cluster.injection_points = 'cluster-pcm-x-retain-flush-error'; + SELECT pg_reload_conf() +}); +my ($dirty_reload_ready, $dirty_reload_log) = wait_for_lms_finish_flush_reload( + $quad->node0, $dirty_flush_log_offset, $dirty_lms_workers, 'true', + 'cluster-pcm-x-retain-flush-error', 15); +ok($dirty_reload_ready, + 'L2F every node0 DATA worker applied the finish-Flush injection arm') + or diag("L2F DATA-worker reload log=[$dirty_reload_log]"); +my ($dirty_requester_reload_ready, $dirty_requester_reload_log) + = wait_for_lms_finish_flush_reload( + $quad->node1, $dirty_requester_log_offset, $dirty_requester_lms_workers, 'true', + 'cluster-pcm-x-retain-flush-error', 15); +ok($dirty_requester_reload_ready, + 'L2F every node1 DATA worker applied the transfer-boundary diagnostic arm') + or diag("L2F requester DATA-worker reload log=[$dirty_requester_reload_log]"); +my ($dirty_seed_rc, $dirty_seed_out, $dirty_seed_err) = $quad->node0->psql( + 'postgres', q{UPDATE pcm_xq_dirty_retain SET v = 1 WHERE id = 1}, timeout => 30); +is($dirty_seed_rc, 0, 'L2F node0 created an uncheckpointed dirty X source') + or diag("L2F dirty seed stdout=[$dirty_seed_out] stderr=[$dirty_seed_err]"); +my ($dirty_retain_rc, $dirty_retain_out, $dirty_retain_err) = $quad->node1->psql( + 'postgres', q{INSERT INTO pcm_xq_dirty_retain(id, v) VALUES (2, 1)}, timeout => 30); +is($dirty_retain_rc, 0, + 'L2F remote writer completed the dirty retain/flush lifecycle') + or diag("L2F dirty retain stdout=[$dirty_retain_out] stderr=[$dirty_retain_err]"); +cmp_ok(state_int($quad->node0, 'gcs', 'dedup_pcm_x_stage_count') + - $dirty_stage_before, '>', 0, + 'L2F source DATA worker materialized the immutable dirty image'); +cmp_ok(state_int($quad->node0, 'xnode_lever', 'h_pi_write_note_count') + - $dirty_flush_before, '>', 0, + 'L2F source FlushBuffer completed smgrwrite before ownership commit'); +my $dirty_flush_log = substr(slurp_file($quad->node0->logfile), + $dirty_flush_log_offset); +like($dirty_flush_log, + qr/cluster injection point "cluster-pcm-x-retain-flush-error" armed with WARNING/, + 'L2F DATA worker reached the finish-exclusive FlushBuffer injection seam'); +like($dirty_flush_log, + qr/cluster PCM-X retained-image finish FlushBuffer succeeded:/, + 'L2F finish-exclusive FlushBuffer returned after the physical write'); +is($quad->node1->safe_psql('postgres', + q{SELECT string_agg(id::text || ':' || v::text, ',' ORDER BY id) + FROM pcm_xq_dirty_retain}), '1:1,2:1', + 'L2F dirty retain preserved the exact page contents'); +$quad->node0->safe_psql('postgres', q{ + ALTER SYSTEM RESET cluster.injection_points; + SELECT pg_reload_conf() +}); +$quad->node1->safe_psql('postgres', q{ + ALTER SYSTEM RESET cluster.injection_points; + SELECT pg_reload_conf() +}); +my ($dirty_reset_ready, $dirty_reset_log) = wait_for_lms_finish_flush_reload( + $quad->node0, $dirty_flush_log_offset, $dirty_lms_workers, 'false', '', 15); +ok($dirty_reset_ready, + 'L2F every node0 DATA worker applied the finish-Flush injection disarm') + or diag("L2F DATA-worker reset log=[$dirty_reset_log]"); +my ($dirty_requester_reset_ready, $dirty_requester_reset_log) + = wait_for_lms_finish_flush_reload( + $quad->node1, $dirty_requester_log_offset, $dirty_requester_lms_workers, 'false', '', 15); +ok($dirty_requester_reset_ready, + 'L2F every node1 DATA worker applied the transfer-boundary diagnostic disarm') + or diag("L2F requester DATA-worker reset log=[$dirty_requester_reset_log]"); + for my $i (0 .. 3) { ok(write_retry($quad->node($i), @@ -382,6 +600,7 @@ sub write_file }); diag("L2 fixed hot-block tuple map: rel=$paths[0] tuples=$tuple_map"); +my @workload_log_offsets = map { (-s $quad->node($_)->logfile) // 0 } (0 .. 3); my @pcm_before_by_node = map { state_snapshot($quad->node($_), 'pcm', \@pcm_x_pcm_keys) } (0 .. 3); @@ -397,10 +616,14 @@ sub write_file my $passive_s_before = 0; $passive_s_before += state_int($quad->node($_), 'gcs', 'invalidate_passive_s_release_count') for (0 .. 3); +my @holder_evicted_before_by_node = map { + state_int($quad->node($_), 'gcs', 'block_forward_holder_evicted_count') +} (0 .. 3); my $start_at = $quad->node0->safe_psql('postgres', q{SELECT (clock_timestamp() + interval '5 seconds')::text}); my $script_dir = PostgreSQL::Test::Utils::tempdir(); my @runs; +my $data_worker_buffer_content_wait_samples = 0; for my $i (0 .. 3) { @@ -422,6 +645,100 @@ sub write_file push @runs, \%run; } +# Mid-leg probe: while the four writers are (potentially) stalled, capture +# each node's writer wait state and the live queue gauges. A post-mortem +# probe cannot see this — a wedged writer holds no cluster state after +# kill_kill. Diagnostic only; every query is bounded and failure-tolerant. +{ + my @samples; + + for my $offset (1, 3, 8) + { + my $probe_at = $quad->node0->safe_psql('postgres', + "SELECT GREATEST(0.0, EXTRACT(EPOCH FROM " + . "(TIMESTAMPTZ '$start_at' + interval '$offset seconds' - clock_timestamp())))"); + sleep($probe_at) if $probe_at > 0; + for my $i (0 .. 3) + { + my $waits = eval { + $quad->node($i)->safe_psql('postgres', + q{SELECT pid || ':' || state || ':' || coalesce(wait_event_type, '-') + || '/' || coalesce(wait_event, '-') + FROM pg_stat_activity + WHERE query LIKE 'UPDATE pcm_xq_hot%'}, + timeout => 10); + } // 'probe-failed'; + $waits =~ s/\n/ | /g; + my $aux = eval { + $quad->node($i)->safe_psql('postgres', + q{SELECT backend_type || ':' || coalesce(wait_event, '-') + FROM pg_stat_activity + WHERE backend_type IN ('lmon', 'lms', 'lms worker', 'lck', 'lmd', + 'cssd', 'diag', 'cluster stats', 'qvotec', 'interconnect listener') + ORDER BY backend_type}, + timeout => 10); + } // 'probe-failed'; + $aux =~ s/\n/ | /g; + my $data_content_waits = eval { + $quad->node($i)->safe_psql('postgres', + q{SELECT count(*) FROM pg_stat_activity + WHERE backend_type = 'lms worker' + AND wait_event = 'BufferContent'}, + timeout => 10); + } // 0; + $data_worker_buffer_content_wait_samples += $data_content_waits + if $data_content_waits =~ /\A\d+\z/; + my $wire = eval { + $quad->node($i)->safe_psql('postgres', + q{SELECT string_agg('peer' || node_id || ':s' || msg_send_count + || ':r' || msg_recv_count, ' ' ORDER BY node_id) + FROM pg_cluster_ic_peers}, + timeout => 10); + } // 'probe-failed'; + my $slots = eval { + $quad->node($i)->safe_psql('postgres', + q{SELECT string_agg(key || '=[' || value || ']', ' ' ORDER BY key) + FROM pg_cluster_state + WHERE category = 'pcm' + AND (key LIKE 'pcm_x_tag_%' OR key LIKE 'pcm_x_ticket_%' + OR key LIKE 'pcm_x_ltag_%' OR key = 'pcm_x_terminal_last_note')}, + timeout => 10); + } // 'probe-failed'; + $slots =~ s/\n/ | /g; + $samples[$offset][$i] = eval { + state_snapshot($quad->node($i), 'pcm', \@pcm_x_pcm_keys); + }; + diag("L3 mid-leg t+$offset node$i waits=[$waits]" + . ($samples[$offset][$i] ? '' : ' snapshot-failed')); + diag("L3 mid-leg t+$offset node$i aux=[$aux]"); + diag("L3 mid-leg t+$offset node$i DATA BufferContent waits=" + . $data_content_waits); + diag("L3 mid-leg t+$offset node$i wire=[$wire]"); + diag("L3 mid-leg t+$offset node$i slots=[$slots]"); + } + } + + # Early samples catch the terminal handoff mid-flight; the late pair + # separates the live retry loop (still ticking) from the wedged stages + # (frozen). + for my $pair ([1, 3], [3, 8]) + { + my ($from, $to) = @{$pair}; + for my $i (0 .. 3) + { + next unless $samples[$from][$i] && $samples[$to][$i]; + my @moved; + for my $key (@pcm_x_pcm_keys) + { + my $d = $samples[$to][$i]{$key} - $samples[$from][$i]{$key}; + push @moved, "$key:+$d" if $d != 0; + } + diag("L3 mid-leg node$i t+$from..t+$to moved: " + . (@moved ? join(' ', @moved) : '(all frozen)')); + } + } +} + for my $i (0 .. 3) { my $run = $runs[$i]; @@ -460,12 +777,54 @@ sub write_file my $passive_s_after = 0; $passive_s_after += state_int($quad->node($_), 'gcs', 'invalidate_passive_s_release_count') for (0 .. 3); +my @holder_evicted_after_by_node = map { + state_int($quad->node($_), 'gcs', 'block_forward_holder_evicted_count') +} (0 .. 3); diag('L3 path probes: pcm_x_queue_enqueue_delta=' . ($queue_after - $queue_before) . ' legacy_denied_pending_x_delta=' . ($denied_after - $denied_before) . ' passive_s_release_delta=' . ($passive_s_after - $passive_s_before)); +for my $i (0 .. 3) +{ + diag("L3 node$i holder copy refusal baseline=$holder_evicted_before_by_node[$i]" + . " final=$holder_evicted_after_by_node[$i] delta=" + . ($holder_evicted_after_by_node[$i] - $holder_evicted_before_by_node[$i])); +} + +# Per-node runtime probe: the aggregate sums above cannot distinguish which +# node fused. Name the fused node and its fail-closed arm (file:line). +for my $i (0 .. 3) +{ + my $site = $quad->node($i)->safe_psql('postgres', + q{SELECT value FROM pg_cluster_state + WHERE category = 'pcm' AND key = 'pcm_x_runtime_fail_closed_site'}); + diag("L3 node$i runtime probe:" + . " state=$pcm_after_by_node[$i]{pcm_x_runtime_state}" + . " generation=$pcm_after_by_node[$i]{pcm_x_runtime_generation}" + . ' generation_delta=' + . ($pcm_after_by_node[$i]{pcm_x_runtime_generation} + - $pcm_before_by_node[$i]{pcm_x_runtime_generation}) + . ' recovery_blocked_delta=' + . ($pcm_after_by_node[$i]{pcm_x_queue_recovery_blocked_count} + - $pcm_before_by_node[$i]{pcm_x_queue_recovery_blocked_count}) + . " fail_closed_site=[$site]"); + is($pcm_after_by_node[$i]{pcm_x_runtime_state}, 1, + "L3 node$i PCM-X runtime remained ACTIVE"); + is($pcm_after_by_node[$i]{pcm_x_runtime_generation}, + $pcm_before_by_node[$i]{pcm_x_runtime_generation}, + "L3 node$i PCM-X runtime generation stayed exact"); + is($site, '(none)', "L3 node$i PCM-X fail-closed site stayed empty"); + is($lmd_after_by_node[$i]{wait_edge_count}, + $lmd_before_by_node[$i]{wait_edge_count}, + "L3 node$i WFG live edge count returned to baseline"); + for my $key (@pcm_x_final_gauge_keys) + { + is($pcm_after_by_node[$i]{$key}, 0, + "L3 node$i final PCM-X gauge $key is zero"); + } +} for my $key (@pcm_x_pcm_keys) { diag("L3 PCM-X state $key=$pcm_after{$key} delta=" @@ -484,14 +843,30 @@ sub write_file WHERE category = 'gcs' AND key IN ( 'block_master_not_holder_count', + 'block_forward_holder_evicted_count', 'block_x_self_ship_count', 'drop_pinned_deny_count', 'x_vs_s_no_carrier_denied_count', 'block_invalidate_broadcast_count', 'block_invalidate_ack_received_count', + 'invalidate_send_queued_count', 'invalidate_passive_s_release_count', + 'invalidate_parked_count', + 'invalidate_busy_sent_count', + 'invalidate_busy_received_count', + 'invalidate_park_expired_count', + 'invalidate_park_overflow_count', 'pcm_x_self_handoff_count', 'pcm_x_self_handoff_drain_count', + 'dedup_pcm_x_stage_count', + 'dedup_pcm_x_replay_count', + 'dedup_pcm_x_release_count', + 'dedup_pcm_x_failclosed_count', + 'stale_reply_drop_count', + 'block_checksum_fail_count', + 'block_forward_received_count', + 'block_from_holder_ship_count', + 'cf_xheld_read_ship_count', 'invalidate_send_not_admitted_count', 'forward_send_not_admitted_count', 'reply_send_not_admitted_count') @@ -502,11 +877,37 @@ sub write_file ', ' ORDER BY category, key) FROM pg_cluster_state WHERE (category = 'cr' AND key IN ( + 'rtvis_undo_fetch_wire_count', + 'rtvis_undo_fetch_cache_hit_count', + 'rtvis_undo_fetch_failclosed_count', + 'rtvis_resolve_committed_count', + 'rtvis_resolve_aborted_count', + 'rtvis_resolve_failclosed_count', 'rtvis_verdict_wire_count', 'rtvis_verdict_failclosed_count', 'rtvis_verdict_exact_count', 'rtvis_verdict_below_horizon_count', - 'rtvis_verdict_inadmissible_count')) + 'rtvis_verdict_inadmissible_count', + 'rtvis_underivable_failclosed_count', + 'cr_server_verdict_served_count', + 'cr_server_verdict_denied_count', + 'cr_server_fence_refused_count', + 'undo_authority_serve_hit_count', + 'undo_authority_fail_closed_count', + 'undo_authority_epoch_stale_reject_count', + 'undo_authority_scan_incomplete_reject_count', + 'undo_authority_multi_match_reject_count', + 'vis53r97_leg_invalid_scn_refuse_count', + 'vis53r97_leg_zero_match_refuse_count', + 'vis53r97_leg_srv_other_refuse_count', + 'vis53r97_leg_covers_refuse_count', + 'vis53r97_leg_multi_unresolvable_count', + 'vis53r97_leg_xmax_unprovable_count', + 'vis53r97_leg_xmin_overlay_verdict_ask_count', + 'vis53r97_leg_xmin_overlay_verdict_hit_count', + 'vis53r97_leg_multi_member_serve_ask_count', + 'vis53r97_leg_multi_member_serve_hit_count', + 'vis53r97_leg_live_upgrade_hit_count')) OR (category = 'xnode' AND key IN ( 'c_resolve_count', 'c_tt_lookup_count', 'c_memo_hit_count', 'c_memo_install_count')) @@ -521,16 +922,19 @@ sub write_file is($runs[$i]->{errors}, 0, "L3 node$i writer surfaced zero client errors"); cmp_ok($runs[$i]->{transactions}, '>', 0, "L3 node$i writer made progress"); - cmp_ok( - $pcm_after_by_node[$i]->{pcm_x_queue_enqueue_count} - - $pcm_before_by_node[$i]->{pcm_x_queue_enqueue_count}, - '>', 0, "L3 node$i PCM-X enqueue delta proves this requester used the queue"); + cmp_ok($pcm_after_by_node[$i]{pcm_x_queue_wait_count} + - $pcm_before_by_node[$i]{pcm_x_queue_wait_count}, '>', 0, + "L3 node$i requester identity entered the queue wait lifecycle"); } +# Queue counters live on the static tag master, not on each requester. The +# aggregate lifecycle plus every writer's committed progress proves that all +# four requesters traversed the protocol; requiring a local enqueue delta on +# non-master nodes is a false topology assumption. cmp_ok($queue_after - $queue_before, '>=', 4, - 'L3 all four node writers entered the PCM-X queue protocol'); -is($denied_after - $denied_before, 0, - 'L3 no writer fell back to legacy reader pending-X denial'); + 'L3 aggregate PCM-X queue admission floor reached four'); +cmp_ok($denied_after - $denied_before, '>', 0, + 'L3 Shape-B arbitration denied an in-flight legacy reader without a client error'); cmp_ok($passive_s_after - $passive_s_before, '>', 0, 'L3 exact queue INVALIDATE exercised passive-pinned S release'); for my $key (@positive_pcm_lifecycle_keys) @@ -543,23 +947,50 @@ sub write_file is($pcm_after{$key} - $pcm_before{$key}, 0, "L3 PCM-X failure counter $key stayed zero"); } +# STALE/NOT_FOUND are retry classifications used by generation-exact role +# refresh and idempotent replay. They are observable churn, not terminal +# failures; zero terminal gauges, zero client errors, and L4 conservation are +# the authoritative safety/liveness verdicts. for my $key (@positive_wfg_keys) { cmp_ok($lmd_after{$key} - $lmd_before{$key}, '>', 0, "L3 PCM-X WFG lifecycle $key advanced"); } +# Atomic replacement with a zero-blocker set removes the old waiter edges but +# intentionally counts as a replace, so an explicit remove call is not +# required on every successful run. is($lmd_after{pcm_convert_wfg_replace_fail_count} - $lmd_before{pcm_convert_wfg_replace_fail_count}, 0, 'L3 PCM-X WFG atomic replace failures stayed zero'); is($lmd_after{pcm_convert_wfg_exact_remove_stale_count} - $lmd_before{pcm_convert_wfg_exact_remove_stale_count}, 0, 'L3 PCM-X WFG exact-remove stale identities stayed zero'); +is($data_worker_buffer_content_wait_samples, 0, + 'L3 DATA workers never waited on BufferContent while receive progress depended on them'); ok($gauges_drained, 'L3 PCM-X terminal gauges drained within 30 seconds'); for my $key (@pcm_x_final_gauge_keys) { is($pcm_after{$key}, 0, "L3 final aggregate PCM-X gauge $key is zero"); } +my $successful_transactions = 0; +$successful_transactions += $runs[$_]->{transactions} for (0 .. 3); +my $stale_miss_delta + = ($pcm_after{pcm_x_queue_stale_count} - $pcm_before{pcm_x_queue_stale_count}) + + ($pcm_after{pcm_x_queue_miss_count} - $pcm_before{pcm_x_queue_miss_count}); +cmp_ok($stale_miss_delta, '<=', 4 * $successful_transactions + 16, + 'L3 retryable stale/miss churn stayed within the per-transaction budget'); + +for my $i (0 .. 3) +{ + my $log = slurp_file($quad->node($i)->logfile); + my $workload_log = substr($log, $workload_log_offsets[$i]); + unlike($workload_log, qr/\b(?:FATAL|PANIC):/, + "L3 node$i log contains no FATAL or PANIC"); + unlike($workload_log, qr/owner-plane[^\n]*(?:violation|corrupt|fail(?:ed|ure)?)/i, + "L3 node$i log contains no owner-plane violation"); +} + my ($advanced_rc, $advanced, $advanced_err) = $quad->node0->psql('postgres', q{ SELECT string_agg(id::text || ':' || row_count::text || ':' || total_v::text, ',' ORDER BY id) @@ -587,5 +1018,94 @@ sub write_file 'L4 aggregate value equals total committed pgbench transactions') or diag("L4 expected_sum=$expected_sum stderr=[$sum_err]"); +# The destructive leg is deliberately last: its expected outcome is a +# fail-closed runtime, so no later assertion may depend on ACTIVE or drained +# gauges. GUC+reload is required because injection state is process-local; +# the DATA worker, not this SQL backend, executes the finish boundary. +ok(write_retry($quad->node0, + q{INSERT INTO pcm_xq_flush_error(id, v) VALUES (1, 0)}), + 'L5F seeded the direct-X target page'); +ok(write_retry($quad->node0, 'VACUUM pcm_xq_flush_error'), + 'L5F published target-page free space before the direct-X request'); +ok(write_retry($quad->node0, 'CHECKPOINT'), + 'L5F checkpointed before the destructive finish-Flush test'); +my $flush_error_relfilenode = $quad->node0->safe_psql('postgres', + q{SELECT pg_relation_filenode('pcm_xq_flush_error'::regclass)}) + 0; +my $flush_error_stage_before = state_int($quad->node0, 'gcs', + 'dedup_pcm_x_stage_count'); +my $flush_error_release_before = state_int($quad->node0, 'gcs', + 'dedup_pcm_x_release_count'); +my $flush_error_blocked_before = state_int($quad->node0, 'pcm', + 'pcm_x_queue_recovery_blocked_count'); +my $flush_error_log_offset = (-s $quad->node0->logfile) // 0; + +$quad->node0->safe_psql('postgres', q{ + ALTER SYSTEM SET cluster.injection_points = + 'cluster-pcm-x-retain-flush-error:skipn:1'; + SELECT pg_reload_conf() +}); +my ($flush_error_reload_ready, $flush_error_reload_log) + = wait_for_lms_finish_flush_reload( + $quad->node0, $flush_error_log_offset, $dirty_lms_workers, 'true', + 'cluster-pcm-x-retain-flush-error:skipn:1', 15); +ok($flush_error_reload_ready, + 'L5F every node0 DATA worker applied the one-shot finish-Flush fault arm') + or diag("L5F DATA-worker reload log=[$flush_error_reload_log]"); + +my ($flush_seed_rc, $flush_seed_out, $flush_seed_err) = $quad->node0->psql( + 'postgres', q{UPDATE pcm_xq_flush_error SET v = 1 WHERE id = 1}, timeout => 30); +is($flush_seed_rc, 0, 'L5F node0 created the destructive-leg X source') + or diag("L5F seed stdout=[$flush_seed_out] stderr=[$flush_seed_err]"); +my ($flush_error_rc, $flush_error_out, $flush_error_err) = $quad->node1->psql( + 'postgres', q{ + SET statement_timeout = '5s'; + INSERT INTO pcm_xq_flush_error(id, v) VALUES (2, 1) + }, timeout => 30); +isnt($flush_error_rc, 0, + 'L5F remote writer failed when finish FlushBuffer raised the injected ERROR') + or diag("L5F unexpected success stdout=[$flush_error_out] stderr=[$flush_error_err]"); + +my ($flush_error_runtime_after, $flush_error_blocked_after); +my $flush_error_deadline = time() + 15; +while (1) +{ + $flush_error_runtime_after = state_int($quad->node0, 'pcm', + 'pcm_x_runtime_state'); + $flush_error_blocked_after = state_int($quad->node0, 'pcm', + 'pcm_x_queue_recovery_blocked_count'); + last if $flush_error_runtime_after == 0 + && $flush_error_blocked_after > $flush_error_blocked_before; + last if time() >= $flush_error_deadline; + usleep(100_000); +} + +is($quad->node0->safe_psql('postgres', 'SELECT 1'), '1', + 'L5F node0 postmaster remained alive after the DATA-worker ERROR'); +is($flush_error_runtime_after, 0, + 'L5F node0 PCM-X runtime moved to RECOVERY_BLOCKED'); +cmp_ok($flush_error_blocked_after - $flush_error_blocked_before, '>', 0, + 'L5F recovery-blocked counter recorded the finish error'); +cmp_ok(state_int($quad->node0, 'gcs', 'dedup_pcm_x_stage_count') + - $flush_error_stage_before, '>', 0, + 'L5F immutable A-record reached MATERIALIZED_UNCOMMITTED before ERROR'); +my $flush_error_log = substr(slurp_file($quad->node0->logfile), + $flush_error_log_offset); +like($flush_error_log, + qr/PCM-X finish-error evidence exact.*?preserve_result=12 retained=true worker=\d+ tag=\d+\/\d+\/\Q$flush_error_relfilenode\E\/0\/0 requester=1 backend=\d+ request_id=\d+ ticket=\d+ queue_generation=\d+ grant_generation=\d+ image_id=\d+ reservation_token=\d+ source_state=\d+/s, + 'L5F exact immutable A-record remains retained after ERROR'); +is(state_int($quad->node0, 'gcs', 'dedup_pcm_x_release_count'), + $flush_error_release_before, + 'L5F ERROR did not release immutable evidence or its REVOKING fence'); +like($flush_error_log, + qr/injected PCM-X retained-image FlushBuffer failure/, + 'L5F exact pre-smgrwrite finish FlushBuffer ERROR reached the DATA worker'); +like($flush_error_log, + qr/cluster PCM-X runtime fail-closed \(recovery blocked\)/, + 'L5F GCS finish catch preserved evidence and fused the runtime'); + $quad->stop_quad; +my $flush_error_shutdown_log = substr(slurp_file($quad->node0->logfile), + $flush_error_log_offset); +unlike($flush_error_shutdown_log, qr/lost track of buffer IO/, + 'L5F absorbed FlushBuffer ERROR left no ResourceOwner BufferIO at shutdown'); done_testing(); diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 5ffd59af30..b5a42a1762 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -1353,7 +1353,9 @@ test_cluster_advisory: test_cluster_advisory.c unit_test.h \ # acquire/release/upgrade/downgrade/query paths are exercised standalone. test_cluster_pcm_lock: test_cluster_pcm_lock.c unit_test.h \ $(CLUSTER_VERSION_O) $(CLUSTER_PCM_LOCK_O) - $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CC) $(CFLAGS) $(CPPFLAGS) \ + -DGCS_BLOCK_SOURCE_PATH='"$(top_srcdir)/src/backend/cluster/cluster_gcs_block.c"' \ + $< \ $(CLUSTER_VERSION_O) $(CLUSTER_PCM_LOCK_O) \ $(top_builddir)/src/common/libpgcommon_srv.a \ $(top_builddir)/src/port/libpgport_srv.a -o $@ @@ -1378,6 +1380,8 @@ test_cluster_pcm_direct_init: test_cluster_pcm_direct_init.c unit_test.h \ -DBUFMGR_SOURCE_PATH='"$(top_srcdir)/src/backend/storage/buffer/bufmgr.c"' \ -DVM_SOURCE_PATH='"$(top_srcdir)/src/backend/access/heap/visibilitymap.c"' \ -DFSM_SOURCE_PATH='"$(top_srcdir)/src/backend/storage/freespace/freespace.c"' \ + -DHEAPAM_SOURCE_PATH='"$(top_srcdir)/src/backend/access/heap/heapam.c"' \ + -DHIO_SOURCE_PATH='"$(top_srcdir)/src/backend/access/heap/hio.c"' \ $< \ $(CLUSTER_PCM_DIRECT_INIT_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ @@ -1449,7 +1453,10 @@ test_cluster_gcs_dispatch: test_cluster_gcs_dispatch.c unit_test.h \ test_cluster_gcs_block: test_cluster_gcs_block.c unit_test.h $(CLUSTER_VERSION_O) $(CC) $(CFLAGS) $(CPPFLAGS) \ -DGCS_BLOCK_SOURCE_PATH='"$(top_srcdir)/src/backend/cluster/cluster_gcs_block.c"' \ + -DBUFMGR_SOURCE_PATH='"$(top_srcdir)/src/backend/storage/buffer/bufmgr.c"' \ -DLMS_SOURCE_PATH='"$(top_srcdir)/src/backend/cluster/cluster_lms.c"' \ + -DLMS_OUTBOUND_SOURCE_PATH='"$(top_srcdir)/src/backend/cluster/cluster_lms_outbound.c"' \ + -DT400_SOURCE_PATH='"$(top_srcdir)/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl"' \ $< \ $(CLUSTER_VERSION_O) \ $(top_builddir)/src/common/libpgcommon_srv.a \ @@ -1612,9 +1619,12 @@ test_cluster_tt_status_hint: test_cluster_tt_status_hint.c unit_test.h \ # HeapTupleSatisfiesMVCC behavioral testing (requires real PG backend; # behavioral coverage in cluster_tap t/204). test_cluster_visibility_fork: test_cluster_visibility_fork.c unit_test.h \ - $(CLUSTER_VERSION_O) - $(CC) $(CFLAGS) $(CPPFLAGS) $< \ - $(CLUSTER_VERSION_O) \ + $(CLUSTER_VERSION_O) $(CLUSTER_VIS_VERDICT_O) + $(CC) $(CFLAGS) $(CPPFLAGS) \ + -DHEAPAM_SOURCE_PATH='"$(top_srcdir)/src/backend/access/heap/heapam.c"' \ + -DTT_LOCAL_SOURCE_PATH='"$(top_srcdir)/src/backend/cluster/cluster_tt_local.c"' \ + $< \ + $(CLUSTER_VERSION_O) $(CLUSTER_VIS_VERDICT_O) \ $(top_builddir)/src/common/libpgcommon_srv.a \ $(top_builddir)/src/port/libpgport_srv.a -o $@ diff --git a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c index 24ca2b2573..985770101f 100644 --- a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c +++ b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c @@ -370,7 +370,8 @@ bool cluster_gcs_send_block_request_and_wait(struct BufferDesc *buf pg_attribute_unused(), PcmLockTransition trans pg_attribute_unused(), int master_node pg_attribute_unused(), - bool clean_eligible pg_attribute_unused()) + bool clean_eligible pg_attribute_unused(), + bool *out_retry_denied pg_attribute_unused()) { abort(); } @@ -378,16 +379,19 @@ cluster_gcs_send_block_request_and_wait(struct BufferDesc *buf pg_attribute_unus /* spec-5.2 D2 sub-case B stub: local-master read-image forward unreachable. */ bool cluster_gcs_local_master_read_image_and_wait(struct BufferDesc *buf pg_attribute_unused(), - int32 holder_node pg_attribute_unused()) + const PcmAuthoritySnapshot *expected + pg_attribute_unused(), + bool *out_retry_denied pg_attribute_unused()) { abort(); } /* spec-5.2 D11 stub: local-master X-transfer forward unreachable in this unit. */ bool -cluster_gcs_local_master_x_transfer_and_wait(struct BufferDesc *buf pg_attribute_unused(), - int32 holder_node pg_attribute_unused(), - bool clean_eligible pg_attribute_unused()) +cluster_gcs_local_master_x_transfer_and_wait( + struct BufferDesc *buf pg_attribute_unused(), + const PcmAuthoritySnapshot *expected pg_attribute_unused(), + bool clean_eligible pg_attribute_unused(), bool *out_retry_denied pg_attribute_unused()) { abort(); } diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index be7c737f80..5560c6e735 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -933,6 +933,39 @@ cluster_pcm_x_runtime_snapshot(void) return snapshot; } +bool +cluster_pcm_x_runtime_fail_closed_site(char *buf, Size buflen) +{ + if (buf != NULL && buflen > 0) + buf[0] = '\0'; + return false; +} + +bool +cluster_pcm_x_master_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, Size buflen) +{ + return false; +} + +bool +cluster_pcm_x_master_ticket_debug_next(Size *cursor_io, Size *index_out, char *buf, Size buflen) +{ + return false; +} + +bool +cluster_pcm_x_local_tag_debug_next(Size *cursor_io, Size *index_out, char *buf, Size buflen) +{ + return false; +} + +bool +cluster_pcm_x_terminal_note_read(uint32 *op_out, uint32 *result_out, uint64 *ticket_out, + uint32 *count_out) +{ + return false; +} + /* PGRAC spec-2.30 D9 R10 stub audit — 9 transition counter accessors. */ uint64 cluster_pcm_get_trans_n_to_s_count(void) @@ -1522,6 +1555,11 @@ cluster_gcs_get_pi_watermark_retire_count(void) return 0; } uint64 +cluster_gcs_get_pi_durable_note_apply_count(void) +{ + return 0; +} +uint64 cluster_gcs_get_lost_write_detected_count(void) { return 0; @@ -4238,6 +4276,7 @@ UT_TEST(test_debug_dump_exposes_exact_pcm_x_lmd_and_gcs_key_sets) static const char *const pcm_keys[] = { "pcm_x_runtime_state", "pcm_x_runtime_generation", + "pcm_x_runtime_fail_closed_site", "pcm_x_queue_enqueue_count", "pcm_x_queue_admit_count", "pcm_x_queue_confirm_count", @@ -4281,6 +4320,7 @@ UT_TEST(test_debug_dump_exposes_exact_pcm_x_lmd_and_gcs_key_sets) "invalidate_passive_s_release_count", "pcm_x_self_handoff_count", "pcm_x_self_handoff_drain_count", + "pi_durable_note_apply_count", }; LOCAL_FCINFO(fcinfo, 0); ReturnSetInfo rsinfo; @@ -4294,9 +4334,9 @@ UT_TEST(test_debug_dump_exposes_exact_pcm_x_lmd_and_gcs_key_sets) fcinfo->resultinfo = (fmNodePtr)&rsinfo; (void)cluster_dump_state(fcinfo); - UT_ASSERT_EQ(captured_dump_count("pcm", NULL), 58); + UT_ASSERT_EQ(captured_dump_count("pcm", NULL), 60); UT_ASSERT_EQ(captured_dump_count("lmd", NULL), 51); - UT_ASSERT_EQ(captured_dump_count("gcs", NULL), 119); + UT_ASSERT_EQ(captured_dump_count("gcs", NULL), 120); for (i = 0; i < (int)lengthof(pcm_keys); i++) UT_ASSERT_EQ(captured_dump_count("pcm", pcm_keys[i]), 1); for (i = 0; i < (int)lengthof(lmd_keys); i++) @@ -4503,6 +4543,12 @@ cluster_lms_obs_get_outbound_requeue_drop(int worker_id pg_attribute_unused()) { return 0; } + +uint64 +cluster_lms_obs_get_outbound_cap_guard_drop(int worker_id pg_attribute_unused()) +{ + return 0; +} uint64 cluster_lms_obs_get_serve_hist(int worker_id pg_attribute_unused(), int bucket pg_attribute_unused()) diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 328c70cbb7..7b2c86f6bf 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -130,6 +130,34 @@ read_gcs_block_source(void) } +static void +assert_ordered_in_function(const char *source, const char *function_start, const char *function_end, + const char *const *needles, int needle_count) +{ + const char *cursor = strstr(source, function_start); + const char *end; + int i; + + UT_ASSERT_NOT_NULL(cursor); + if (cursor == NULL) + return; + end = strstr(cursor + strlen(function_start), function_end); + UT_ASSERT_NOT_NULL(end); + if (end == NULL) + return; + + for (i = 0; i < needle_count; i++) { + cursor = strstr(cursor, needles[i]); + UT_ASSERT_NOT_NULL(cursor); + if (cursor == NULL) + return; + UT_ASSERT(cursor < end); + if (cursor >= end) + return; + cursor += strlen(needles[i]); + } +} + static int count_occurrences(const char *source, const char *needle) { @@ -1172,6 +1200,145 @@ UT_TEST(test_pcm_x_formation_transient_or_inconsistent_sample_is_tick_noop) } +/* P0-20: a post-handoff COMMIT_X ticket can remain freshly armed while the + * production LMON path silently exits before retry mutation. Every + * pre-mutation boundary must therefore leave one bounded, stage-specific + * breadcrumb; otherwise t/400 cannot distinguish formation rejection from a + * missed work scan or a drive precheck refusal. */ +UT_TEST(test_pcm_x_periodic_retry_reports_pre_mutation_exit_stage) +{ + char *source = read_gcs_block_source(); + const char *formation; + const char *formation_end; + const char *retry_tick; + const char *retry_tick_end; + const char *drive; + const char *drive_end; + const char *transfer; + const char *transfer_end; + + UT_ASSERT_NOT_NULL(source); + if (source != NULL) { + formation = strstr(source, "\ncluster_gcs_block_pcm_x_formation_tick("); + formation_end = formation != NULL ? strstr(formation, "\nfail_closed:") : NULL; + UT_ASSERT_NOT_NULL(formation); + UT_ASSERT_NOT_NULL(formation_end); + if (formation != NULL && formation_end != NULL) { + UT_ASSERT_NOT_NULL(strstr(formation, "\"collect-before\"")); + UT_ASSERT_NOT_NULL(strstr(formation, "\"collect-after\"")); + UT_ASSERT_NOT_NULL(strstr(formation, "\"stability\"")); + UT_ASSERT_NOT_NULL(strstr(formation, "\"peer-revalidate\"")); + UT_ASSERT_NOT_NULL(strstr(formation, "\"runtime-resample\"")); + } + + retry_tick = strstr(source, "\ngcs_block_pcm_x_master_drive_retry_tick("); + retry_tick_end = retry_tick != NULL ? strstr(retry_tick, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(retry_tick); + UT_ASSERT_NOT_NULL(retry_tick_end); + if (retry_tick != NULL && retry_tick_end != NULL) { + UT_ASSERT_NOT_NULL(strstr(retry_tick, "\"work-next\"")); + UT_ASSERT_NOT_NULL(strstr(retry_tick, "cursor_before")); + } + UT_ASSERT_NOT_NULL(strstr(source, "cluster_pcm_x_stats_snapshot(&stats)")); + UT_ASSERT_NOT_NULL(strstr(source, "stats.live_tickets == 0")); + + drive = strstr(source, "\ngcs_block_pcm_x_master_drive_tag("); + drive_end = drive != NULL ? strstr(drive, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(drive); + UT_ASSERT_NOT_NULL(drive_end); + if (drive != NULL && drive_end != NULL) { + UT_ASSERT_NOT_NULL(strstr(drive, "\"drive-precheck-")); + } + + transfer = strstr(source, "\ngcs_block_pcm_x_master_drive_transfer("); + transfer_end = transfer != NULL ? strstr(transfer, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(transfer); + UT_ASSERT_NOT_NULL(transfer_end); + if (transfer != NULL && transfer_end != NULL) { + UT_ASSERT_NOT_NULL(strstr(transfer, "\"commit-retry\"")); + } + free(source); + } +} + + +/* review R2 P0-1: a SELF-loopback V2 INSTALL_READY (master == requester + * node) has no HELLO capability record; the handler must treat self as + * rebase-capable (the local binary) while still requiring the + * activation-bound coverage flag. The regression silently dropped the + * self V2 frame, froze the PREPARE->INSTALL chain on the master ticket and + * leaked the remote retained source. */ +UT_TEST(test_pcm_x_install_ready_v2_self_loopback_is_admissible) +{ + char *source = read_gcs_block_source(); + const char *handler; + const char *self_arm; + const char *peer_arm; + const char *valid_call; + + UT_ASSERT_NOT_NULL(source); + if (source != NULL) { + handler = strstr(source, "\ncluster_gcs_handle_pcm_x_install_ready_envelope("); + UT_ASSERT_NOT_NULL(handler); + self_arm = handler != NULL ? strstr(handler, "source_node == cluster_node_id") : NULL; + UT_ASSERT_NOT_NULL(self_arm); + peer_arm = self_arm != NULL + ? strstr(self_arm, "cluster_sf_peer_supports_pcm_x_rebase(source_node)") + : NULL; + UT_ASSERT_NOT_NULL(peer_arm); + valid_call = peer_arm != NULL + ? strstr(peer_arm, "cluster_gcs_pcm_x_install_ready_ingress_valid(") + : NULL; + UT_ASSERT_NOT_NULL(valid_call); + free(source); + } +} + + +/* review P0-2: the collector binds capability bits to the peer session only + * through one lock-coherent (bits, generation) sample taken on BOTH sides of + * the authority pass; a reconnect between the samples changes the record + * generation and rejects the tick, so a stale connection's REBASE bit can + * never activate V2 against a fresh session. */ +UT_TEST(test_pcm_x_formation_samples_capability_family_atomically) +{ + char *source = read_gcs_block_source(); + const char *collect; + const char *first_sample; + const char *barrier; + const char *second_sample; + const char *end; + const char *stray; + + UT_ASSERT_NOT_NULL(source); + if (source != NULL) { + collect = strstr(source, "\ngcs_block_pcm_x_collect_formation("); + UT_ASSERT_NOT_NULL(collect); + end = collect != NULL ? strstr(collect, "\n#define PCM_X_MASTER_DRIVE_SCAN_BUDGET") : NULL; + UT_ASSERT_NOT_NULL(end); + first_sample + = collect != NULL ? strstr(collect, "cluster_sf_peer_pcm_x_capability_sample(") : NULL; + UT_ASSERT_NOT_NULL(first_sample); + barrier = first_sample != NULL ? strstr(first_sample, "pg_read_barrier();") : NULL; + UT_ASSERT_NOT_NULL(barrier); + second_sample + = barrier != NULL ? strstr(barrier, "cluster_sf_peer_pcm_x_capability_sample(") : NULL; + UT_ASSERT_NOT_NULL(second_sample); + if (second_sample != NULL && end != NULL) + UT_ASSERT(second_sample < end); + /* The separate single-shot capability reads are gone from the + * collector: only the atomic family sample may feed it. */ + if (collect != NULL && end != NULL) { + stray = strstr(collect, "cluster_sf_peer_supports_pcm_x_convert("); + UT_ASSERT(stray == NULL || stray > end); + stray = strstr(collect, "cluster_sf_peer_supports_pcm_x_rebase("); + UT_ASSERT(stray == NULL || stray > end); + } + free(source); + } +} + + UT_TEST(test_pcm_x_confirm_publish_then_stale_requires_exact_graph_close) { UT_ASSERT(cluster_gcs_pcm_x_confirm_compensation_required(UINT64_C(7001), PCM_X_QUEUE_STALE)); @@ -1297,21 +1464,70 @@ UT_TEST(test_pcm_x_install_ready_ingress_is_canonical_requester_ack) ready.result = PCM_X_QUEUE_OK; ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; - UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 3, 11, 2, 2)); + UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 3, 11, 2, 2, + true, true)); ready.result = PCM_X_QUEUE_DUPLICATE; - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); ready.result = PCM_X_QUEUE_OK; ready.phase = PGRAC_IC_MSG_PCM_X_PREPARE_GRANT; - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; ready.flags = 1; - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); ready.flags = 0; ready.image_id = UINT64CONST(0xf000000000000025); - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); UT_ASSERT(cluster_pcm_x_image_id_encode(3, 37, &ready.image_id)); - UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); + + /* A' rebase: both exact frame lengths are legal, nothing in between; a + * V1-length frame must carry a zero rebase and a V2 rebase must be + * strictly newer than the immutable identity base and not the exhausted + * sentinel. */ + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 37, &ready.image_id)); + UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, PCM_X_INSTALL_READY_V1_LEN, 1, + 11, 2, 2, true, true)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, PCM_X_INSTALL_READY_V1_LEN + 4, + 1, 11, 2, 2, true, true)); + ready.ref.identity.base_own_generation = 5; + ready.rebased_own_generation = 8; + UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, PCM_X_INSTALL_READY_V1_LEN, 1, + 11, 2, 2, true, true)); + ready.rebased_own_generation = 5; + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); + ready.rebased_own_generation = 4; + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); + ready.rebased_own_generation = UINT64_MAX; + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, true)); + + /* Receiver-side V2 admission (review P1-3): a 112-byte frame is refused + * unless BOTH this node's activated formation has full V2 coverage and + * the source's current connection advertised the REBASE capability. The + * sender-side coverage gate is not a receiver invariant. */ + ready.rebased_own_generation = 8; + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + false, true)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + true, false)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2, + false, false)); + ready.rebased_own_generation = 0; + ready.ref.identity.base_own_generation = 0; + /* V1 frames stay independent of the rebase capability pair. */ + UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, PCM_X_INSTALL_READY_V1_LEN, 1, + 11, 2, 2, false, false)); } @@ -1334,7 +1550,7 @@ UT_TEST(test_pcm_x_commit_x_ingress_is_canonical_master_phase) } -UT_TEST(test_pcm_x_final_ack_ingress_binds_exact_committed_generation) +UT_TEST(test_pcm_x_final_ack_ingress_binds_monotonic_committed_floor) { PcmXFinalAckPayload ack; uint64 image_id; @@ -1348,8 +1564,19 @@ UT_TEST(test_pcm_x_final_ack_ingress_binds_exact_committed_generation) UT_ASSERT(cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); UT_ASSERT(!cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 3, 11, 2, 2)); + /* A' rebase: the wire check is only a monotonic floor (committed > base); + * a published rebase legally lifts committed past base+1 and the exact + * "+1" proof runs under the master ticket lock against the effective + * grant base. At or below the base stays refused. */ ack.committed_own_generation = 2; + UT_ASSERT(cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); + ack.ref.identity.base_own_generation = 5; + ack.committed_own_generation = 5; + UT_ASSERT(!cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); + ack.committed_own_generation = 4; UT_ASSERT(!cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); + ack.committed_own_generation = 9; + UT_ASSERT(cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); ack.ref.identity.base_own_generation = UINT64_MAX; ack.committed_own_generation = 0; UT_ASSERT(!cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); @@ -1784,6 +2011,311 @@ UT_TEST(test_pcm_x_invalidate_ack_matches_only_exact_unacked_holder) } +UT_TEST(test_pcm_x_invalidate_busy_routes_to_exact_ticket_backoff) +{ + char *source = read_gcs_block_source(); + const char *handler; + const char *queue_busy; + const char *backoff; + const char *received; + const char *queue_return; + const char *legacy_busy; + const char *transfer; + const char *deadline; + const char *revoke; + const char *delay_helper; + const char *delay_end; + const char *leg_retry_delay; + const char *execute; + const char *self_capable_first; + const char *self_capable_second; + const char *execute_end; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + handler = strstr(source, "\ncluster_gcs_handle_block_invalidate_ack_envelope("); + queue_busy = handler != NULL ? strstr(handler, "queue_result == PCM_X_QUEUE_BUSY") : NULL; + backoff = queue_busy != NULL + ? strstr(queue_busy, "cluster_pcm_x_master_invalidate_busy_backoff_exact(") + : NULL; + received = backoff != NULL ? strstr(backoff, "invalidate_busy_received_count") : NULL; + queue_return = received != NULL ? strstr(received, "if (!queue_positive)") : NULL; + legacy_busy = queue_return != NULL + ? strstr(queue_return, + "ack->ack_status == GCS_BLOCK_INVALIDATE_ACK_STATUS_RETRYABLE_BUSY") + : NULL; + UT_ASSERT_NOT_NULL(handler); + UT_ASSERT_NOT_NULL(queue_busy); + UT_ASSERT_NOT_NULL(backoff); + UT_ASSERT_NOT_NULL(received); + UT_ASSERT_NOT_NULL(queue_return); + UT_ASSERT_NOT_NULL(legacy_busy); + if (handler != NULL && queue_busy != NULL && backoff != NULL && received != NULL + && queue_return != NULL && legacy_busy != NULL) + UT_ASSERT(handler < queue_busy && queue_busy < backoff && backoff < received + && received < queue_return && queue_return < legacy_busy); + + transfer = strstr(source, "\ngcs_block_pcm_x_master_drive_transfer("); + deadline = transfer != NULL ? strstr(transfer, "snapshot->retry_deadline_ms") : NULL; + revoke = transfer != NULL ? strstr(transfer, "cluster_pcm_x_master_revoke_arm_exact(") : NULL; + UT_ASSERT_NOT_NULL(transfer); + UT_ASSERT_NOT_NULL(deadline); + UT_ASSERT_NOT_NULL(revoke); + if (transfer != NULL && deadline != NULL && revoke != NULL) + UT_ASSERT(transfer < deadline && deadline < revoke); + + /* reliable.retry_count counts repeated REVOKE arming while an earlier + * holder INVALIDATE is still missing; it is not an INVALIDATE-BUSY retry + * counter. Feeding it into this delay can saturate the very first BUSY + * retry at 25s and phase-lock the denied reader's GRANT_PENDING window. */ + delay_helper = strstr(source, "\ngcs_block_pcm_x_invalidate_busy_retry_delay_ms("); + delay_end = delay_helper != NULL ? strstr(delay_helper, "\n}") : NULL; + leg_retry_delay = delay_helper != NULL ? strstr(delay_helper, "snapshot->retry_count") : NULL; + UT_ASSERT_NOT_NULL(delay_helper); + UT_ASSERT_NOT_NULL(delay_end); + if (delay_helper != NULL && delay_end != NULL) + UT_ASSERT(leg_retry_delay == NULL || leg_retry_delay > delay_end); + + /* Self has no HELLO capability record. The local binary is nevertheless + * BUSY-capable on both GRANT_PENDING and pinned-S refusal arms. */ + execute = strstr(source, "\ngcs_block_invalidate_execute("); + execute_end = execute != NULL ? strstr(execute, "\n}\n") : NULL; + self_capable_first + = execute != NULL ? strstr(execute, "inv->master_node == cluster_node_id") : NULL; + self_capable_second = self_capable_first != NULL ? strstr(self_capable_first + 1, + "inv->master_node == cluster_node_id") + : NULL; + UT_ASSERT_NOT_NULL(execute); + UT_ASSERT_NOT_NULL(execute_end); + UT_ASSERT_NOT_NULL(self_capable_first); + UT_ASSERT_NOT_NULL(self_capable_second); + if (execute != NULL && self_capable_first != NULL && self_capable_second != NULL + && execute_end != NULL) + UT_ASSERT(execute < self_capable_first && self_capable_first < self_capable_second + && self_capable_second < execute_end); + free(source); +} + + +UT_TEST(test_pcm_x_local_pending_s_denial_match_is_attempt_exact) +{ + BufferTag slot_tag = { 0 }; + BufferTag inv_tag; + + slot_tag.spcOid = 31; + slot_tag.dbOid = 32; + slot_tag.relNumber = 33; + slot_tag.forkNum = MAIN_FORKNUM; + slot_tag.blockNum = 34; + inv_tag = slot_tag; + + UT_ASSERT(GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + UT_ASSERT(GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_ABORTED, false, &inv_tag, UINT64_C(41), 2)); + + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + false, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, true, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, false, true, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_X, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + inv_tag.blockNum++; + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + inv_tag = slot_tag; + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(40), 2, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 1, + GCS_BLOCK_DIRECT_UNARMED, false, &inv_tag, UINT64_C(41), 2)); + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_ARMED, false, &inv_tag, UINT64_C(41), 2)); + UT_ASSERT(!GcsBlockLocalPendingSDenialMatches( + true, false, false, (uint8)PCM_TRANS_N_TO_S, &slot_tag, UINT64_C(41), 2, + GCS_BLOCK_DIRECT_UNARMED, true, &inv_tag, UINT64_C(41), 2)); +} + + +UT_TEST(test_pcm_x_grant_pending_invalidate_wakes_local_s_before_busy) +{ + char *source = read_gcs_block_source(); + const char *direct_prepare; + const char *direct_prepare_end; + const char *direct_first_reply_check; + const char *direct_target_prepare; + const char *direct_second_reply_check; + const char *direct_target_cleanup; + const char *helper; + const char *execute; + const char *pending; + const char *wake; + const char *busy; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + direct_prepare = strstr(source, "\ngcs_block_direct_prepare_attempt("); + direct_prepare_end = direct_prepare != NULL ? strstr(direct_prepare, "\n}\n\n\n") : NULL; + direct_first_reply_check + = direct_prepare != NULL ? strstr(direct_prepare, "slot->reply_received") : NULL; + direct_target_prepare = direct_first_reply_check != NULL + ? strstr(direct_first_reply_check, + "cluster_bufmgr_prepare_direct_land_target_for_gcs(") + : NULL; + direct_second_reply_check = direct_target_prepare != NULL + ? strstr(direct_target_prepare, "slot->reply_received") + : NULL; + direct_target_cleanup + = direct_second_reply_check != NULL + ? strstr(direct_second_reply_check, "gcs_block_direct_finish_target(buf, true, false") + : NULL; + helper = strstr(source, "\ngcs_block_wake_local_pending_s_request("); + execute = strstr(source, "\ngcs_block_invalidate_execute("); + pending + = execute != NULL ? strstr(execute, "cluster_bufmgr_block_grant_pending(inv->tag)") : NULL; + wake = pending != NULL ? strstr(pending, "gcs_block_wake_local_pending_s_request(inv)") : NULL; + busy = wake != NULL ? strstr(wake, "invalidate_busy_sent_count") : NULL; + UT_ASSERT_NOT_NULL(helper); + UT_ASSERT_NOT_NULL(direct_prepare); + UT_ASSERT_NOT_NULL(direct_prepare_end); + UT_ASSERT_NOT_NULL(direct_first_reply_check); + UT_ASSERT_NOT_NULL(direct_target_prepare); + UT_ASSERT_NOT_NULL(direct_second_reply_check); + UT_ASSERT_NOT_NULL(direct_target_cleanup); + UT_ASSERT_NOT_NULL(execute); + UT_ASSERT_NOT_NULL(pending); + UT_ASSERT_NOT_NULL(wake); + UT_ASSERT_NOT_NULL(busy); + if (direct_prepare != NULL && direct_prepare_end != NULL && direct_first_reply_check != NULL + && direct_target_prepare != NULL && direct_second_reply_check != NULL + && direct_target_cleanup != NULL) + UT_ASSERT(direct_prepare < direct_first_reply_check + && direct_first_reply_check < direct_target_prepare + && direct_target_prepare < direct_second_reply_check + && direct_second_reply_check < direct_target_cleanup + && direct_target_cleanup < direct_prepare_end); + if (helper != NULL && execute != NULL && pending != NULL && wake != NULL && busy != NULL) + UT_ASSERT(helper < execute && execute < pending && pending < wake && wake < busy); + free(source); +} + + +UT_TEST(test_pcm_x_grant_pending_orphan_observation_is_identity_exact) +{ + char *gcs_source = read_gcs_block_source(); + char *bufmgr_source = read_source_path(BUFMGR_SOURCE_PATH); + const char *execute; + const char *pending; + const char *wake; + const char *observe; + const char *busy; + const char *retry; + const char *abort; + const char *abort_observe; + const char *release; + const char *release_end; + const char *release_live; + const char *direct_fail; + const char *direct_fail_end; + const char *direct_finish; + const char *direct_abort_observe; + + UT_ASSERT_NOT_NULL(gcs_source); + UT_ASSERT_NOT_NULL(bufmgr_source); + if (gcs_source == NULL || bufmgr_source == NULL) { + free(gcs_source); + free(bufmgr_source); + return; + } + + execute = strstr(gcs_source, "\ngcs_block_invalidate_execute("); + pending + = execute != NULL ? strstr(execute, "cluster_bufmgr_block_grant_pending(inv->tag)") : NULL; + wake = pending != NULL + ? strstr(pending, "woke_local = gcs_block_wake_local_pending_s_request(inv)") + : NULL; + observe = wake != NULL + ? strstr(wake, "gcs_block_observe_grant_pending_invalidate(inv, woke_local)") + : NULL; + busy = observe != NULL ? strstr(observe, "invalidate_busy_sent_count") : NULL; + UT_ASSERT_NOT_NULL(execute); + UT_ASSERT_NOT_NULL(pending); + UT_ASSERT_NOT_NULL(wake); + UT_ASSERT_NOT_NULL(observe); + UT_ASSERT_NOT_NULL(busy); + if (execute != NULL && pending != NULL && wake != NULL && observe != NULL && busy != NULL) + UT_ASSERT(execute < pending && pending < wake && wake < observe && observe < busy); + + /* The diagnostic must report why a candidate was rejected without replacing + * or weakening the established attempt-exact protocol predicate. */ + UT_ASSERT_NOT_NULL(strstr(gcs_source, "GcsBlockLocalPendingSDenialMatches(")); + UT_ASSERT_NOT_NULL(strstr(gcs_source, "grant-pending invalidate observation:")); + UT_ASSERT_NOT_NULL(strstr(gcs_source, "slot_backend=%d slot_index=%d slot_request_id=")); + UT_ASSERT_NOT_NULL(strstr(gcs_source, "direct_state=%d direct_prepared=%d reject_mask=0x%x")); + + retry = strstr(bufmgr_source, "\ncluster_bufmgr_pcm_retry_denied_rearm("); + abort = retry != NULL + ? strstr(retry, + "cluster_pcm_own_abort_grant_reservation(buf, base, *reservation_token)") + : NULL; + abort_observe + = abort != NULL ? strstr(abort, "cluster PCM pending-X exact abort observation:") : NULL; + UT_ASSERT_NOT_NULL(retry); + UT_ASSERT_NOT_NULL(abort); + UT_ASSERT_NOT_NULL(abort_observe); + if (retry != NULL && abort != NULL && abort_observe != NULL) + UT_ASSERT(retry < abort && abort < abort_observe); + + release = strstr(gcs_source, "\ngcs_block_release_slot("); + release_end = release != NULL ? strstr(release, "\n}\n") : NULL; + release_live + = release != NULL + ? strstr(release, + "cluster GCS block slot released with live direct target observation:") + : NULL; + UT_ASSERT_NOT_NULL(release); + UT_ASSERT_NOT_NULL(release_end); + UT_ASSERT_NOT_NULL(release_live); + if (release != NULL && release_end != NULL && release_live != NULL) + UT_ASSERT(release < release_live && release_live < release_end); + + direct_fail = strstr(gcs_source, "\ngcs_block_direct_fail_slot("); + direct_fail_end = direct_fail != NULL ? strstr(direct_fail, "\n}\n") : NULL; + direct_finish + = direct_fail != NULL + ? strstr(direct_fail, "gcs_block_direct_finish_target(target_buf, prepared, false") + : NULL; + direct_abort_observe + = direct_finish != NULL + ? strstr(direct_finish, "cluster GCS block direct abort observation:") + : NULL; + UT_ASSERT_NOT_NULL(direct_fail); + UT_ASSERT_NOT_NULL(direct_fail_end); + UT_ASSERT_NOT_NULL(direct_finish); + UT_ASSERT_NOT_NULL(direct_abort_observe); + if (direct_fail != NULL && direct_fail_end != NULL && direct_finish != NULL + && direct_abort_observe != NULL) + UT_ASSERT(direct_fail < direct_finish && direct_finish < direct_abort_observe + && direct_abort_observe < direct_fail_end); + + free(gcs_source); + free(bufmgr_source); +} + + UT_TEST(test_pcm_x_final_ack_builds_exact_grd_handoff_token) { PcmAuthoritySnapshot authority; @@ -1829,6 +2361,51 @@ UT_TEST(test_pcm_x_final_ack_builds_exact_grd_handoff_token) } +UT_TEST(test_pcm_x_final_ack_fail_closed_names_exact_handoff_stage) +{ + char *source = read_gcs_block_source(); + const char *handler + = source != NULL ? strstr(source, "\ncluster_gcs_handle_pcm_x_final_ack_envelope(") : NULL; + const char *handler_end + = handler != NULL ? strstr(handler, "\ncluster_gcs_handle_pcm_x_final_commit_ack_envelope(") + : NULL; + const char *stage_log + = handler != NULL ? strstr(handler, "PCM-X FINAL_ACK fail-closed at %s") : NULL; + const char *canonical + = handler != NULL ? strstr(handler, "cluster_pcm_x_runtime_fail_closed()") : NULL; + const char *direct + = handler != NULL ? strstr(handler, "cluster_pcm_x_runtime_transition(") : NULL; + const char *master_holder = handler != NULL ? strstr(handler, "master_holder=%u") : NULL; + const char *image_page_scn = handler != NULL ? strstr(handler, "image_page_scn=%llu") : NULL; + const char *watermark_scn = handler != NULL ? strstr(handler, "watermark_scn=%llu") : NULL; + const char *watermark_source = handler != NULL ? strstr(handler, "wm_src=%s") : NULL; + const char *watermark_sender = handler != NULL ? strstr(handler, "wm_sender=%d") : NULL; + const char *watermark_request = handler != NULL ? strstr(handler, "wm_request_id=%llu") : NULL; + const char *watermark_old = handler != NULL ? strstr(handler, "wm_old_scn=%llu") : NULL; + const char *watermark_new = handler != NULL ? strstr(handler, "wm_new_scn=%llu") : NULL; + + UT_ASSERT_NOT_NULL(handler); + UT_ASSERT_NOT_NULL(handler_end); + UT_ASSERT_NOT_NULL(stage_log); + UT_ASSERT_NOT_NULL(canonical); + UT_ASSERT_NOT_NULL(master_holder); + UT_ASSERT_NOT_NULL(image_page_scn); + UT_ASSERT_NOT_NULL(watermark_scn); + UT_ASSERT_NOT_NULL(watermark_source); + UT_ASSERT_NOT_NULL(watermark_sender); + UT_ASSERT_NOT_NULL(watermark_request); + UT_ASSERT_NOT_NULL(watermark_old); + UT_ASSERT_NOT_NULL(watermark_new); + UT_ASSERT_NULL(source != NULL ? strstr(source, "cluster_pcm_x_runtime_transition(") : NULL); + /* A blocked runtime must always publish the canonical counter and file:line arm. */ + if (handler != NULL && handler_end != NULL && stage_log != NULL && canonical != NULL) + UT_ASSERT(handler < stage_log && stage_log < canonical && canonical < handler_end); + if (handler != NULL && handler_end != NULL) + UT_ASSERT(direct == NULL || direct >= handler_end); + free(source); +} + + UT_TEST(test_pcm_x_holder_image_evidence_never_uses_generation_as_presence) { PcmXLocalHolderProgress progress; @@ -1882,6 +2459,15 @@ UT_TEST(test_pcm_x_ready_publication_follows_exact_retained_commit) const char *publish; const char *send; const char *end; + const char *wrapper; + const char *wrapper_end; + const char *wrapper_catch; + const char *copy_error; + const char *flush_error; + const char *preserve; + const char *fail_closed; + const char *return_corrupt; + const char *rollback; if (source == NULL) return; @@ -1893,7 +2479,7 @@ UT_TEST(test_pcm_x_ready_publication_follows_exact_retained_commit) } end = strstr(begin + 1, "\n}\n\n\n"); materialize = strstr(begin, "cluster_gcs_block_dedup_pcm_x_materialize("); - finish = strstr(begin, "cluster_bufmgr_pcm_own_finish_revoke_retain("); + finish = strstr(begin, "gcs_block_pcm_x_finish_revoke_retain("); publish = strstr(begin, "cluster_gcs_block_dedup_pcm_x_publish_ready_exact("); send = strstr(begin, "gcs_block_pcm_x_stage_ready_work("); UT_ASSERT_NOT_NULL(end); @@ -1903,6 +2489,45 @@ UT_TEST(test_pcm_x_ready_publication_follows_exact_retained_commit) UT_ASSERT_NOT_NULL(send); if (end != NULL && materialize != NULL && finish != NULL && publish != NULL && send != NULL) UT_ASSERT(materialize < finish && finish < publish && publish < send && send < end); + wrapper = strstr(source, "\ngcs_block_pcm_x_finish_revoke_retain("); + wrapper_end = wrapper != NULL ? strstr(wrapper, "\n}\n") : NULL; + wrapper_catch = wrapper != NULL ? strstr(wrapper, "PG_CATCH();") : NULL; + copy_error = wrapper_catch != NULL ? strstr(wrapper_catch, "CopyErrorData();") : NULL; + flush_error = copy_error != NULL ? strstr(copy_error, "FlushErrorState();") : NULL; + preserve + = flush_error != NULL + ? strstr(flush_error, "cluster_gcs_block_dedup_pcm_x_preserve_finish_error_exact(") + : NULL; + fail_closed + = preserve != NULL ? strstr(preserve, "cluster_pcm_x_runtime_fail_closed();") : NULL; + return_corrupt + = fail_closed != NULL ? strstr(fail_closed, "result = CLUSTER_PCM_OWN_CORRUPT;") : NULL; + rollback = wrapper_catch != NULL + ? strstr(wrapper_catch, "gcs_block_pcm_x_abort_image_before_finish(") + : NULL; + UT_ASSERT_NOT_NULL(wrapper); + UT_ASSERT_NOT_NULL(wrapper_end); + UT_ASSERT_NOT_NULL(wrapper_catch); + UT_ASSERT_NOT_NULL(copy_error); + UT_ASSERT_NOT_NULL(flush_error); + UT_ASSERT_NOT_NULL(preserve); + UT_ASSERT_NOT_NULL(strstr(wrapper_catch, "PCM-X finish-error evidence exact")); + UT_ASSERT_NOT_NULL(strstr(wrapper_catch, "preserve_result")); + UT_ASSERT_NOT_NULL(strstr(wrapper_catch, "work->key")); + UT_ASSERT_NOT_NULL(strstr(wrapper_catch, "work->binding.identity.ref")); + UT_ASSERT_NOT_NULL(strstr(wrapper_catch, "revoking->reservation_token")); + UT_ASSERT_NOT_NULL(fail_closed); + UT_ASSERT_NOT_NULL(return_corrupt); + if (wrapper != NULL && wrapper_end != NULL && wrapper_catch != NULL && copy_error != NULL + && flush_error != NULL && preserve != NULL && fail_closed != NULL + && return_corrupt != NULL) { + UT_ASSERT(wrapper < wrapper_catch && wrapper_catch < copy_error && copy_error < flush_error + && flush_error < preserve && preserve < fail_closed + && fail_closed < return_corrupt && return_corrupt < wrapper_end); + UT_ASSERT(rollback == NULL || rollback >= wrapper_end); + UT_ASSERT(strstr(wrapper_catch, "PG_RE_THROW();") == NULL + || strstr(wrapper_catch, "PG_RE_THROW();") >= wrapper_end); + } free(source); } @@ -1943,13 +2568,19 @@ UT_TEST(test_pcm_x_ready_materializes_exact_n_s_or_x_source_without_wire_change) UT_ASSERT_NOT_NULL(strstr(begin, "current.pcm_state == (uint8)PCM_STATE_S")); UT_ASSERT_NOT_NULL(strstr(begin, "current.pcm_state == (uint8)PCM_STATE_X")); UT_ASSERT_NOT_NULL(strstr(begin, "cluster_bufmgr_pcm_own_prepare_n_source_image(")); - UT_ASSERT_NOT_NULL(strstr(begin, "cluster_bufmgr_pcm_own_begin_s_revoke(")); + UT_ASSERT_NOT_NULL(strstr(begin, "cluster_bufmgr_pcm_own_prepare_s_source_image(")); + UT_ASSERT_NOT_NULL(strstr(begin, "binding.required_page_scn")); + UT_ASSERT_NOT_NULL(strstr(begin, "&source_prepare_refusal")); + UT_ASSERT_NOT_NULL(strstr(source, "materialize-begin-s-content-lock")); + UT_ASSERT_NOT_NULL(strstr(source, "materialize-begin-s-dirty-flushed")); + UT_ASSERT_NOT_NULL(strstr(source, "materialize-begin-s-dirty-raced")); + UT_ASSERT_NOT_NULL(strstr(source, "materialize-begin-s-io-in-progress")); UT_ASSERT_NOT_NULL(strstr(begin, "cluster_bufmgr_pcm_own_begin_x_revoke(")); UT_ASSERT_NOT_NULL(strstr(begin, "cluster_pcm_x_revoke_finish_mode(")); UT_ASSERT_NOT_NULL(strstr(begin, "CLUSTER_PCM_X_REVOKE_FINISH_DROP")); copy = strstr(begin, "cluster_bufmgr_copy_block_for_gcs("); materialize = strstr(begin, "cluster_gcs_block_dedup_pcm_x_materialize("); - finish = strstr(begin, "cluster_bufmgr_pcm_own_finish_revoke_retain("); + finish = strstr(begin, "gcs_block_pcm_x_finish_revoke_retain("); publish = strstr(begin, "cluster_gcs_block_dedup_pcm_x_publish_ready_exact("); UT_ASSERT_NOT_NULL(copy); UT_ASSERT_NOT_NULL(materialize); @@ -1964,9 +2595,26 @@ UT_TEST(test_pcm_x_ready_materializes_exact_n_s_or_x_source_without_wire_change) UT_ASSERT_NOT_NULL(strstr(abort, "cluster_bufmgr_pcm_own_abort_x_revoke(")); } if (drain != NULL) { - const char *local_drain = strstr(drain, "cluster_pcm_x_local_drain_poll_exact("); + const char *local_drain + = strstr(drain, "cluster_pcm_x_local_drain_poll_certificate_exact("); const char *duplicate_guard - = local_drain != NULL ? strstr(local_drain, "if (result != PCM_X_QUEUE_OK)") : NULL; + = local_drain != NULL + ? strstr(local_drain, + "if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE)") + : NULL; + const char *drain_status + = duplicate_guard != NULL + ? strstr(duplicate_guard, "cluster_gcs_block_dedup_pcm_x_drain_status_exact(") + : NULL; + const char *drained_replay + = drain_status != NULL ? strstr(drain_status, "GCS_BLOCK_PCM_X_IMAGE_DUPLICATE") : NULL; + /* The self-source release authority is the exact completion + * certificate. Once local DRAIN is durable, an inexact ledger cannot + * be retried from a fabricated certificate and must preserve evidence + * under the runtime fuse . */ + const char *certificate_policy = strstr(drain, "source_own_generation + 1"); + const char *certificate_refusal + = certificate_policy != NULL ? strstr(certificate_policy, "PCM_X_QUEUE_CORRUPT") : NULL; const char *release_record = strstr(drain, "cluster_gcs_block_dedup_pcm_x_release_exact("); const char *finish_mode_gate = strstr(drain, "cluster_pcm_x_revoke_finish_mode("); const char *drop_arm = strstr(drain, "CLUSTER_PCM_X_REVOKE_FINISH_DROP"); @@ -1975,15 +2623,24 @@ UT_TEST(test_pcm_x_ready_materializes_exact_n_s_or_x_source_without_wire_change) UT_ASSERT_NOT_NULL(local_drain); UT_ASSERT_NOT_NULL(duplicate_guard); + UT_ASSERT_NOT_NULL(drain_status); + UT_ASSERT_NOT_NULL(drained_replay); + UT_ASSERT_NOT_NULL(certificate_policy); + UT_ASSERT_NOT_NULL(certificate_refusal); UT_ASSERT_NOT_NULL(release_record); UT_ASSERT_NOT_NULL(finish_mode_gate); UT_ASSERT_NOT_NULL(drop_arm); UT_ASSERT_NOT_NULL(release_retained); - if (local_drain != NULL && duplicate_guard != NULL && release_record != NULL - && finish_mode_gate != NULL && drop_arm != NULL && release_retained != NULL) - UT_ASSERT(local_drain < duplicate_guard && duplicate_guard < release_record - && release_record < finish_mode_gate && finish_mode_gate < drop_arm - && drop_arm < release_retained); + if (local_drain != NULL && duplicate_guard != NULL && drain_status != NULL + && drained_replay != NULL && certificate_policy != NULL && certificate_refusal != NULL + && release_record != NULL && finish_mode_gate != NULL && drop_arm != NULL + && release_retained != NULL) + UT_ASSERT(local_drain < duplicate_guard && duplicate_guard < drain_status + && drain_status < drained_replay && drained_replay < certificate_policy + && certificate_policy < certificate_refusal + && certificate_refusal < finish_mode_gate + && finish_mode_gate < release_retained && release_retained < drop_arm + && drop_arm < release_record); } if (generic_install != NULL) { const char *content = strstr(generic_install, "LWLockAcquire(content_lock, LW_EXCLUSIVE)"); @@ -2001,6 +2658,61 @@ UT_TEST(test_pcm_x_ready_materializes_exact_n_s_or_x_source_without_wire_change) } +UT_TEST(test_pcm_x_s_source_hard_failure_observation_is_reason_exact) +{ + char *source = read_source_path(BUFMGR_SOURCE_PATH); + const char *observe; + const char *prepare; + const char *prepare_end; + static const char *const reasons[] = { "CLUSTER_PCM_S_SOURCE_HARD_INITIAL_CURRENT_IMAGE", + "CLUSTER_PCM_S_SOURCE_HARD_INITIAL_IO_ERROR", + "CLUSTER_PCM_S_SOURCE_HARD_BEGIN_REVOKE_CORRUPT", + "CLUSTER_PCM_S_SOURCE_HARD_STORAGE_VERIFY", + "CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_CURRENT_IMAGE", + "CLUSTER_PCM_S_SOURCE_HARD_POST_LOCK_IO_ERROR", + "CLUSTER_PCM_S_SOURCE_HARD_NO_COVER", + "CLUSTER_PCM_S_SOURCE_HARD_ABORT_FAILURE" }; + int i; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + observe = strstr(source, "\ncluster_bufmgr_pcm_own_observe_s_source_hard_failure("); + prepare = strstr(source, "\ncluster_bufmgr_pcm_own_prepare_s_source_image("); + prepare_end + = prepare != NULL + ? strstr(prepare, "\n/* Abort only the matching S-source staging reservation. */") + : NULL; + UT_ASSERT_NOT_NULL(observe); + UT_ASSERT_NOT_NULL(prepare); + UT_ASSERT_NOT_NULL(prepare_end); + for (i = 0; i < lengthof(reasons); i++) + UT_ASSERT_NOT_NULL(strstr(source, reasons[i])); + + /* Hard-failure evidence is local-only and state-change suppressed. It + * identifies the exact ownership tuple and records both sides of the SCN + * floor comparison without changing the prepare function's public API. */ + UT_ASSERT_NOT_NULL(strstr(source, "ClusterPcmOwnSSourceHardFailureObservation cache[8]")); + UT_ASSERT_NOT_NULL(strstr(source, "memcmp(&cache[cache_idx], &obs, sizeof(obs)) == 0")); + UT_ASSERT_NOT_NULL(strstr(source, "cluster PCM S-source hard failure observation:")); + UT_ASSERT_NOT_NULL(strstr(source, "reason=%s cause=%s result=%d abort_result=%d")); + UT_ASSERT_NOT_NULL(strstr(source, "buffer=%d spc=%u db=%u rel=%u fork=%d blk=%u")); + UT_ASSERT_NOT_NULL(strstr(source, "state=%u generation=%llu token=%llu flags=0x%x")); + UT_ASSERT_NOT_NULL(strstr(source, "required_scn=%llu local_scn=%llu storage_scn=%llu")); + UT_ASSERT_NOT_NULL(strstr(source, "buffer_state=0x%x buffer_type=%u")); + if (observe != NULL && prepare != NULL) + UT_ASSERT(observe < prepare); + if (prepare != NULL && prepare_end != NULL) { + UT_ASSERT_NOT_NULL(strstr(prepare, "hard_failure_reason =")); + UT_ASSERT_NOT_NULL( + strstr(prepare, "cluster_bufmgr_pcm_own_observe_s_source_hard_failure(")); + UT_ASSERT(strstr(prepare, "cluster_bufmgr_pcm_own_observe_s_source_hard_failure(") + < prepare_end); + } + free(source); +} + + UT_TEST(test_pcm_x_self_and_remote_drain_share_full_image_release_wrapper) { char *source = read_gcs_block_source(); @@ -2055,14 +2767,24 @@ UT_TEST(test_pcm_x_self_and_remote_drain_share_full_image_release_wrapper) UT_TEST(test_pcm_x_ready_admission_marks_before_send_and_rolls_back_refusal) { char *source = read_gcs_block_source(); + char *outbound_source = read_source_path(LMS_OUTBOUND_SOURCE_PATH); const char *begin; const char *end; + const char *arm; + const char *arm_refusal; const char *mark; const char *send; const char *rollback; + const char *handler; + const char *handler_end; + const char *prepare_handler; + const char *prepare_handler_end; - if (source == NULL) + if (source == NULL || outbound_source == NULL) { + free(source); + free(outbound_source); return; + } begin = strstr(source, "\ngcs_block_pcm_x_stage_ready_work("); UT_ASSERT_NOT_NULL(begin); if (begin == NULL) { @@ -2070,16 +2792,59 @@ UT_TEST(test_pcm_x_ready_admission_marks_before_send_and_rolls_back_refusal) return; } end = strstr(begin + 1, "\n}\n\n\n"); + arm = strstr(begin, "cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed("); + arm_refusal = strstr(begin, "gcs_block_pcm_x_image_ready_arm_refusal_note_work("); mark = strstr(begin, "cluster_gcs_block_dedup_pcm_x_mark_staged_exact("); send = strstr(begin, "cluster_gcs_pcm_x_stage_frame("); rollback = strstr(begin, "cluster_gcs_block_dedup_pcm_x_unmark_staged_exact("); UT_ASSERT_NOT_NULL(end); + UT_ASSERT_NOT_NULL(arm); + UT_ASSERT_NOT_NULL(arm_refusal); UT_ASSERT_NOT_NULL(mark); UT_ASSERT_NOT_NULL(send); UT_ASSERT_NOT_NULL(rollback); - if (end != NULL && mark != NULL && send != NULL && rollback != NULL) - UT_ASSERT(mark < send && send < rollback && rollback < end); + if (end != NULL && arm != NULL && arm_refusal != NULL && mark != NULL && send != NULL + && rollback != NULL) + UT_ASSERT(arm < arm_refusal && arm_refusal < mark && mark < send && send < rollback + && rollback < end); + UT_ASSERT_NOT_NULL(strstr(begin, "PCM_X_LOCAL_IMAGE_READY_REFUSAL_ACTIVE_HOLDER")); + UT_ASSERT_NOT_NULL(strstr(begin, "PCM_X_LOCAL_IMAGE_READY_REFUSAL_RELIABLE_LEG")); + UT_ASSERT_NOT_NULL(strstr(begin, "PCM-X IMAGE_READY stage boundary")); + UT_ASSERT_NOT_NULL(strstr(begin, "mark_result")); + UT_ASSERT_NOT_NULL(strstr(begin, "stage_result")); + UT_ASSERT_NOT_NULL(strstr(outbound_source, "cluster_lms_note_pcm_x_image_ready_boundary(")); + handler = strstr(source, "\ncluster_gcs_handle_pcm_x_image_ready_envelope("); + UT_ASSERT_NOT_NULL(handler); + if (handler != NULL) { + handler_end = strstr(handler + 1, "\n}\n\n\n"); + UT_ASSERT_NOT_NULL(handler_end); + UT_ASSERT_NOT_NULL(strstr(handler, "image-ready-master-consume")); + UT_ASSERT_NOT_NULL(strstr(handler, "PCM-X IMAGE_READY master boundary: ingress")); + UT_ASSERT_NOT_NULL(strstr(handler, "PCM-X IMAGE_READY master boundary: consume")); + UT_ASSERT_NOT_NULL(strstr(handler, "wire_valid")); + UT_ASSERT_NOT_NULL(strstr(handler, "authorized")); + UT_ASSERT_NOT_NULL(strstr(handler, "prepare_stage_result")); + if (handler_end != NULL) + UT_ASSERT(strstr(handler, "image-ready-master-consume") < handler_end); + } + prepare_handler = strstr(source, "\ncluster_gcs_handle_pcm_x_prepare_grant_envelope("); + UT_ASSERT_NOT_NULL(prepare_handler); + if (prepare_handler != NULL) { + prepare_handler_end = strstr(prepare_handler + 1, "\n}\n\n\n"); + UT_ASSERT_NOT_NULL(prepare_handler_end); + UT_ASSERT_NOT_NULL( + strstr(prepare_handler, "PCM-X PREPARE_GRANT requester boundary: ingress")); + UT_ASSERT_NOT_NULL( + strstr(prepare_handler, "PCM-X PREPARE_GRANT requester boundary: apply")); + UT_ASSERT_NOT_NULL(strstr(prepare_handler, "wire_valid")); + UT_ASSERT_NOT_NULL(strstr(prepare_handler, "authorized")); + UT_ASSERT_NOT_NULL(strstr(prepare_handler, "lookup_result")); + if (prepare_handler_end != NULL) + UT_ASSERT(strstr(prepare_handler, "PCM-X PREPARE_GRANT requester boundary: apply") + < prepare_handler_end); + } free(source); + free(outbound_source); } @@ -2117,6 +2882,7 @@ UT_TEST(test_pcm_x_lms_owner_death_and_restart_audit_fail_closed) worker_start = strstr(lms_source, "\nLmsWorkerMain(int worker_id)"); UT_ASSERT_NOT_NULL(main_start); UT_ASSERT_NOT_NULL(worker_start); + UT_ASSERT_NOT_NULL(strstr(lms_source, "PCM-X IMAGE_READY transport boundary")); main_call = main_start != NULL ? strstr(main_start, "cluster_gcs_block_pcm_x_owner_start(0)") : NULL; worker_call = worker_start != NULL @@ -2134,6 +2900,98 @@ UT_TEST(test_pcm_x_lms_owner_death_and_restart_audit_fail_closed) } +UT_TEST(test_pcm_x_lms_reload_acknowledges_finish_flush_injection) +{ + char *lms_source = read_source_path(LMS_SOURCE_PATH); + const char *helper; + const char *main_start; + const char *main_reload; + const char *main_ack; + const char *worker_start; + const char *worker_reload; + const char *worker_ack; + + if (lms_source == NULL) + return; + helper = strstr(lms_source, "\nlms_note_pcm_x_finish_flush_injection_reload("); + main_start = strstr(lms_source, "\nLmsMain(void)"); + worker_start = strstr(lms_source, "\nLmsWorkerMain(int worker_id)"); + UT_ASSERT_NOT_NULL(helper); + UT_ASSERT_NOT_NULL(main_start); + UT_ASSERT_NOT_NULL(worker_start); + if (helper != NULL) { + UT_ASSERT_NOT_NULL( + strstr(helper, "cluster_injection_is_armed(\"cluster-pcm-x-retain-flush-error\")")); + UT_ASSERT_NOT_NULL(strstr(helper, "MyProcPid")); + UT_ASSERT_NOT_NULL(strstr(helper, "cluster_injection_points")); + } + main_reload = main_start != NULL ? strstr(main_start, "ProcessConfigFile(PGC_SIGHUP);") : NULL; + main_ack = main_reload != NULL + ? strstr(main_reload, "lms_note_pcm_x_finish_flush_injection_reload(0);") + : NULL; + worker_reload + = worker_start != NULL ? strstr(worker_start, "ProcessConfigFile(PGC_SIGHUP);") : NULL; + worker_ack + = worker_reload != NULL + ? strstr(worker_reload, "lms_note_pcm_x_finish_flush_injection_reload(worker_id);") + : NULL; + UT_ASSERT_NOT_NULL(main_reload); + UT_ASSERT_NOT_NULL(main_ack); + UT_ASSERT_NOT_NULL(worker_reload); + UT_ASSERT_NOT_NULL(worker_ack); + if (main_reload != NULL && main_ack != NULL && worker_start != NULL) + UT_ASSERT(main_reload < main_ack && main_ack < worker_start); + if (worker_reload != NULL && worker_ack != NULL) + UT_ASSERT(worker_reload < worker_ack); + free(lms_source); +} + + +UT_TEST(test_pcm_x_destructive_finish_fault_times_out_in_sql_before_harness) +{ + char *source = read_source_path(T400_SOURCE_PATH); + const char *leg; + const char *statement_timeout; + const char *insert; + const char *harness_timeout; + const char *assertion; + const char *destructive_leg; + const char *exact_oracle; + const char *global_occupancy; + const char *relfilenode; + + if (source == NULL) + return; + leg = strstr(source, "my ($flush_error_rc, $flush_error_out, $flush_error_err)"); + destructive_leg = strstr(source, "# The destructive leg is deliberately last"); + UT_ASSERT_NOT_NULL(leg); + UT_ASSERT_NOT_NULL(destructive_leg); + statement_timeout = leg != NULL ? strstr(leg, "statement_timeout") : NULL; + insert + = leg != NULL ? strstr(leg, "INSERT INTO pcm_xq_flush_error(id, v) VALUES (2, 1)") : NULL; + harness_timeout = leg != NULL ? strstr(leg, "timeout => 30") : NULL; + assertion = leg != NULL ? strstr(leg, "L5F remote writer failed") : NULL; + relfilenode + = destructive_leg != NULL ? strstr(destructive_leg, "flush_error_relfilenode") : NULL; + exact_oracle = destructive_leg != NULL + ? strstr(destructive_leg, "PCM-X finish-error evidence exact") + : NULL; + global_occupancy + = destructive_leg != NULL ? strstr(destructive_leg, "dedup_entry_count") : NULL; + UT_ASSERT_NOT_NULL(statement_timeout); + UT_ASSERT_NOT_NULL(insert); + UT_ASSERT_NOT_NULL(harness_timeout); + UT_ASSERT_NOT_NULL(assertion); + UT_ASSERT_NOT_NULL(relfilenode); + UT_ASSERT_NOT_NULL(exact_oracle); + UT_ASSERT_NULL(global_occupancy); + if (statement_timeout != NULL && insert != NULL && harness_timeout != NULL && assertion != NULL) + UT_ASSERT(statement_timeout < insert && insert < harness_timeout + && harness_timeout < assertion); + free(source); +} + + UT_TEST(test_pcm_x_image_fetch_intercepts_canonical_id_before_generic_dedup) { char *source = read_gcs_block_source(); @@ -2158,6 +3016,10 @@ UT_TEST(test_pcm_x_image_fetch_intercepts_canonical_id_before_generic_dedup) UT_ASSERT(intercept < generic_lookup); } if (serve != NULL) { + UT_ASSERT_NOT_NULL(strstr(serve, "PCM-X image fetch source boundary: ingress")); + UT_ASSERT_NOT_NULL(strstr(serve, "PCM-X image fetch source boundary: validate")); + UT_ASSERT_NOT_NULL(strstr(serve, "PCM-X image fetch source boundary: lookup")); + UT_ASSERT_NOT_NULL(strstr(serve, "PCM-X image fetch source boundary: reply")); UT_ASSERT_NOT_NULL(strstr(serve, "cluster_pcm_x_image_fetch_request_exact(")); UT_ASSERT(count_occurrences(serve, "cluster_pcm_x_local_holder_progress_exact(") >= 2); UT_ASSERT_NOT_NULL(strstr(serve, "gcs_block_pcm_x_authenticated_session(")); @@ -2201,6 +3063,15 @@ UT_TEST(test_pcm_x_requester_fetch_revalidates_queue_and_reservation_before_inst UT_ASSERT_NOT_NULL(fetch); UT_ASSERT_NOT_NULL(fetch_end); if (fetch != NULL && fetch_end != NULL) { + UT_ASSERT_NOT_NULL(strstr(fetch, "PCM-X image fetch requester boundary: entry")); + UT_ASSERT_NOT_NULL(strstr(fetch, "PCM-X image fetch requester boundary: pre-slot reject")); + UT_ASSERT_NOT_NULL(strstr(fetch, "branch=identity-base")); + UT_ASSERT_NOT_NULL(strstr(fetch, "branch=reservation-live")); + UT_ASSERT_NOT_NULL(strstr(fetch, "progress_grant_base=%llu effective_grant_base=%llu")); + UT_ASSERT_NOT_NULL(strstr(fetch, "live_writer_activation_token=%llu")); + UT_ASSERT_NOT_NULL(strstr(fetch, "PCM-X image fetch requester boundary: send")); + UT_ASSERT_NOT_NULL(strstr(fetch, "PCM-X image fetch requester boundary: receive")); + UT_ASSERT_NOT_NULL(strstr(fetch, "PCM-X image fetch requester boundary: complete")); UT_ASSERT_NOT_NULL(strstr(fetch, "gcs_block_try_reserve_exact_slot(")); UT_ASSERT_NOT_NULL(strstr(fetch, "cluster_pcm_x_image_fetch_build_request(")); UT_ASSERT_NOT_NULL(strstr(fetch, "const PcmXRuntimeSnapshot *request_runtime")); @@ -2399,13 +3270,13 @@ UT_TEST(test_pcm_x_self_source_handoff_is_no_copy_and_drain_preserves_x) && finish_catch_rethrow < finish_end_try && finish_end_try < finish_end); } self_arm = strstr(materialize, "if (self_source_handoff)"); - remote_finish = strstr(materialize, "cluster_bufmgr_pcm_own_finish_revoke_retain("); + remote_finish = strstr(materialize, "gcs_block_pcm_x_finish_revoke_retain("); UT_ASSERT_NOT_NULL(self_arm); UT_ASSERT_NOT_NULL(remote_finish); if (self_arm != NULL && remote_finish != NULL && materialize_end != NULL) UT_ASSERT(self_arm < remote_finish && remote_finish < materialize_end); - verify_x = strstr(drain, "cluster_bufmgr_pcm_own_self_handoff_x_exact("); + verify_x = strstr(drain, "cluster_bufmgr_pcm_own_self_handoff_probe("); release_record = strstr(drain, "cluster_gcs_block_dedup_pcm_x_release_exact("); self_return = release_record != NULL ? strstr(release_record, "if (self_source_handoff)") : NULL; @@ -2416,8 +3287,8 @@ UT_TEST(test_pcm_x_self_source_handoff_is_no_copy_and_drain_preserves_x) UT_ASSERT_NOT_NULL(release_retained); if (verify_x != NULL && release_record != NULL && self_return != NULL && release_retained != NULL && drain_end != NULL) - UT_ASSERT(verify_x < release_record && release_record < self_return - && self_return < release_retained && release_retained < drain_end); + UT_ASSERT(verify_x < release_retained && release_retained < release_record + && release_record < self_return && self_return < drain_end); free(source); } @@ -2548,7 +3419,11 @@ UT_TEST(test_pcm_x_requester_driver_owns_fifo_and_transfer_lifecycles) const char *wait_exact_end; const char *wait_exact_first_authority; const char *wait_exact_physical; - const char *wait_exact_second_authority; + const char *wait_exact_second_revalidate; + const char *wait_revalidate; + const char *wait_revalidate_end; + const char *wait_revalidate_authority; + const char *wait_revalidate_nested_guard; const char *wait_scan; int raw_wait_count = 0; int exact_wait_count = 0; @@ -2571,6 +3446,8 @@ UT_TEST(test_pcm_x_requester_driver_owns_fifo_and_transfer_lifecycles) authority_helper_end = authority_helper != NULL ? strstr(authority_helper, "\n}\n") : NULL; wait_exact = strstr(source, "\ngcs_block_pcm_x_requester_wait_exact("); wait_exact_end = wait_exact != NULL ? strstr(wait_exact, "\n}\n") : NULL; + wait_revalidate = strstr(source, "\ngcs_block_pcm_x_requester_pre_sleep_revalidate("); + wait_revalidate_end = wait_revalidate != NULL ? strstr(wait_revalidate, "\n}\n") : NULL; UT_ASSERT_NOT_NULL(driver); UT_ASSERT_NOT_NULL(driver_end); UT_ASSERT_NOT_NULL(rekey_helper); @@ -2585,6 +3462,8 @@ UT_TEST(test_pcm_x_requester_driver_owns_fifo_and_transfer_lifecycles) UT_ASSERT_NOT_NULL(authority_helper_end); UT_ASSERT_NOT_NULL(wait_exact); UT_ASSERT_NOT_NULL(wait_exact_end); + UT_ASSERT_NOT_NULL(wait_revalidate); + UT_ASSERT_NOT_NULL(wait_revalidate_end); if (driver == NULL || driver_end == NULL || wrapper == NULL || wrapper_end == NULL) { free(source); return; @@ -2682,28 +3561,43 @@ UT_TEST(test_pcm_x_requester_driver_owns_fifo_and_transfer_lifecycles) : NULL; wait_exact_physical = wait_exact_first_authority != NULL - ? strstr(wait_exact_first_authority, "gcs_block_pcm_x_requester_wait(wait_index)") + ? strstr(wait_exact_first_authority, "cluster_pcm_x_requester_wait_once_result(") : NULL; - wait_exact_second_authority + wait_exact_second_revalidate = wait_exact_physical != NULL - ? strstr(wait_exact_physical + 1, "gcs_block_pcm_x_requester_authority_exact(") + ? strstr(wait_exact_physical + 1, + "gcs_block_pcm_x_requester_pre_sleep_revalidate(&context)") + : NULL; + wait_revalidate_authority + = wait_revalidate != NULL + ? strstr(wait_revalidate, "gcs_block_pcm_x_requester_authority_exact(") + : NULL; + wait_revalidate_nested_guard + = wait_revalidate_authority != NULL + ? strstr(wait_revalidate_authority, "cluster_pcm_x_nested_wait_guard_before_block()") : NULL; UT_ASSERT_NOT_NULL(authority_runtime); UT_ASSERT_NOT_NULL(authority_runtime_exact); UT_ASSERT_NOT_NULL(authority_peer); UT_ASSERT_NOT_NULL(wait_exact_first_authority); UT_ASSERT_NOT_NULL(wait_exact_physical); - UT_ASSERT_NOT_NULL(wait_exact_second_authority); + UT_ASSERT_NOT_NULL(wait_exact_second_revalidate); + UT_ASSERT_NOT_NULL(wait_revalidate_authority); + UT_ASSERT_NOT_NULL(wait_revalidate_nested_guard); if (authority_runtime != NULL && authority_runtime_exact != NULL && authority_peer != NULL && authority_helper_end != NULL) UT_ASSERT(authority_runtime < authority_runtime_exact && authority_runtime_exact < authority_peer && authority_peer < authority_helper_end); if (wait_exact_first_authority != NULL && wait_exact_physical != NULL - && wait_exact_second_authority != NULL && wait_exact_end != NULL) + && wait_exact_second_revalidate != NULL && wait_exact_end != NULL) UT_ASSERT(wait_exact_first_authority < wait_exact_physical - && wait_exact_physical < wait_exact_second_authority - && wait_exact_second_authority < wait_exact_end); + && wait_exact_physical < wait_exact_second_revalidate + && wait_exact_second_revalidate < wait_exact_end); + if (wait_revalidate_authority != NULL && wait_revalidate_nested_guard != NULL + && wait_revalidate_end != NULL) + UT_ASSERT(wait_revalidate_authority < wait_revalidate_nested_guard + && wait_revalidate_nested_guard < wait_revalidate_end); formation = strstr(driver, "cluster_gcs_pcm_x_requester_formation_action("); formation_wait = formation != NULL ? strstr(formation, "gcs_block_pcm_x_requester_wait(&wait_index)") @@ -2821,6 +3715,22 @@ UT_TEST(test_pcm_x_requester_driver_owns_fifo_and_transfer_lifecycles) UT_ASSERT_NOT_NULL( strstr(driver, "cluster_gcs_pcm_x_role_refresh_exact(&handle, &fresh_handle)")); UT_ASSERT_NOT_NULL(strstr(driver, "initial_own.pcm_state != (uint8)PCM_STATE_X")); + { + const char *preflight; + const char *preflight_retry; + + preflight = strstr(driver, "cluster_gcs_pcm_x_remote_reservation_preflight("); + preflight_retry + = preflight != NULL + ? strstr(preflight, "GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT") + : NULL; + UT_ASSERT_NOT_NULL(preflight); + UT_ASSERT_NOT_NULL(preflight_retry); + /* The transient-BUSY wait dispatch must sit between the preflight and + * the image fetch; a non-wait verdict is the only fail-closed exit. */ + if (preflight != NULL && preflight_retry != NULL && remote_fetch != NULL) + UT_ASSERT(preflight < preflight_retry && preflight_retry < remote_fetch); + } if (follower_snapshot != NULL && graph_replace != NULL && graph_clear != NULL) UT_ASSERT(graph_clear < follower_snapshot && follower_snapshot < graph_replace && graph_replace < driver_end); @@ -2895,6 +3805,20 @@ UT_TEST(test_pcm_x_requester_retry_policy_is_operation_exact) GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS }, { GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_REPLAY_ARM, PCM_X_QUEUE_STALE, GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + /* Remote-reservation preflight: BUSY is a transient own-slot lifecycle + * (a revoke/grant flag mid-flight on this node) and must wait, never + * close the runtime. STALE stays fail-closed for now: it means the + * enqueue-time base_own_generation was consumed by a revoke while the + * request was queued, which the grant/final-ack chain cannot absorb + * without a master-visible rebase (pending protocol amendment). */ + { GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, PCM_X_QUEUE_BUSY, + GCS_BLOCK_PCM_X_RETRY_WAIT }, + { GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, PCM_X_QUEUE_STALE, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, PCM_X_QUEUE_CORRUPT, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, PCM_X_QUEUE_COUNTER_EXHAUSTED, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, }; size_t i; @@ -3181,6 +4105,7 @@ UT_TEST(test_pcm_x_retire_commit_wakes_exact_waiters_before_ack_or_resolve) const char *allocate; const char *allocate_after; const char *collect; + const char *dedup_retire; const char *exact_wake; const char *wake; const char *wake_end; @@ -3285,14 +4210,19 @@ UT_TEST(test_pcm_x_retire_commit_wakes_exact_waiters_before_ack_or_resolve) collect = allocate != NULL ? strstr(allocate, "cluster_pcm_x_local_retire_up_to_collect_exact(") : NULL; + dedup_retire = collect != NULL + ? strstr(collect, "cluster_gcs_block_dedup_pcm_x_retire_up_to(") + : NULL; exact_wake = collect != NULL ? strstr(collect, "gcs_block_pcm_x_wake_requester_exact(") : NULL; allocate_after = allocate != NULL ? strstr(allocate + 1, "palloc0(") : NULL; UT_ASSERT_NOT_NULL(allocate); UT_ASSERT_NOT_NULL(collect); + UT_ASSERT_NOT_NULL(dedup_retire); UT_ASSERT_NOT_NULL(exact_wake); - if (allocate != NULL && collect != NULL && exact_wake != NULL) - UT_ASSERT(allocate < collect && collect < exact_wake && exact_wake < common_end); + if (allocate != NULL && collect != NULL && dedup_retire != NULL && exact_wake != NULL) + UT_ASSERT(allocate < collect && collect < dedup_retire && dedup_retire < exact_wake + && exact_wake < common_end); UT_ASSERT(allocate_after == NULL || allocate_after > common_end); } remote = strstr(source, "\ncluster_gcs_handle_pcm_x_retire_up_to_envelope("); @@ -3405,11 +4335,906 @@ UT_TEST(test_pcm_x_role_refresh_accepts_only_same_member_promotion) UT_ASSERT(!cluster_gcs_pcm_x_role_refresh_exact(&follower, &promoted)); } +UT_TEST(test_legacy_byte_proof_sites_republish_kept_pi_mirror) +{ + static const char *const install_contract[] + = { "cluster_bufmgr_pcm_x_content_write_permitted", "memcpy", "PageSetLSN", + "cluster_bufmgr_pcm_own_republish_grant_pending_image", "LWLockRelease" }; + static const char *const fallback_contract[] + = { "GCS_LOST_WRITE_PASS", "cluster_bufmgr_pcm_own_republish_grant_pending_image", + "cluster_bufmgr_refresh_block_from_storage_for_gcs", "GCS_LOST_WRITE_PASS", + "cluster_bufmgr_pcm_own_republish_grant_pending_image" }; + char *source = read_gcs_block_source(); + + /* A retained-image release now keeps a frozen PI+BM_VALID N mirror for + * pre-existing pins. Its bytes regain CURRENT only where a legacy grant + * just proved them: the shipped-image install memcpy (still under the + * same content EXCLUSIVE hold) and both storage-fallback PASS proofs + * (direct SCN PASS, and refresh-then-PASS). The SKIP arms must leave PI + * frozen so the finish valid-image gate keeps failing closed on unproven + * bytes. */ + assert_ordered_in_function(source, "\ngcs_block_install_block(", "\nstatic ", install_contract, + lengthof(install_contract)); + assert_ordered_in_function(source, "\ncluster_gcs_block_fallback_verify_refresh(", "\nstatic ", + fallback_contract, lengthof(fallback_contract)); + free(source); +} + + +UT_TEST(test_revoke_handler_silent_refusal_arms_all_note) +{ + char *source = read_gcs_block_source(); + const char *handler = strstr(source, "\ncluster_gcs_handle_pcm_x_revoke_envelope("); + const char *materialize = strstr(source, "\ngcs_block_pcm_x_materialize_reserved_work("); + const char *publish; + const char *reset; + + /* The master re-sends REVOKE forever, so every refusal arm in the source + * handler must name itself through the log-once streak note (t/400 form-B + * wedges previously refused silently at un-instrumented arms): ingress + * auth, holder-ledger stale, holder-progress error, image reserve, apply, + * and ingress snapshot. Merely accepting/re-arming work is not progress; + * the streak resets only after exact READY publication. */ + UT_ASSERT_NOT_NULL(handler); + if (handler != NULL) + UT_ASSERT(count_occurrences(handler, "gcs_block_pcm_x_revoke_refusal_note_exact(") >= 6); + UT_ASSERT_NOT_NULL(materialize); + if (materialize != NULL) { + UT_ASSERT_NOT_NULL(strstr(materialize, "\"materialize-copy\"")); + UT_ASSERT_NOT_NULL(strstr(materialize, "\"materialize-finish\"")); + } + publish = materialize != NULL + ? strstr(materialize, "cluster_gcs_block_dedup_pcm_x_publish_ready_exact(") + : NULL; + reset = publish != NULL ? strstr(publish, "gcs_block_pcm_x_revoke_refusal_note(NULL, 0, NULL)") + : NULL; + if (reset == NULL && publish != NULL) + reset = strstr(publish, + "gcs_block_pcm_x_revoke_refusal_note_exact(NULL, 0, NULL, NULL, 0, 0)"); + UT_ASSERT_NOT_NULL(publish); + UT_ASSERT_NOT_NULL(reset); + UT_ASSERT_NOT_NULL(strstr(source, "request_id=%llu wait_seq=%llu")); + UT_ASSERT_NOT_NULL(strstr(source, "image_id=%llu source_generation=%llu")); + UT_ASSERT_NOT_NULL(strstr(source, "own_generation=%llu token=%llu")); + UT_ASSERT_NOT_NULL(strstr(source, "materialized-finish-vm-fsm-pinned")); + UT_ASSERT_NOT_NULL(strstr(source, "materialized-finish-io-in-progress")); + UT_ASSERT_NOT_NULL(strstr(source, "materialized-finish-live-flags")); + UT_ASSERT_NOT_NULL(strstr(source, "finish_refusal=%u shared_refcount=%u")); + UT_ASSERT_NOT_NULL(strstr(source, "bm_io_in_progress=%s live_flags=%u live_token=%llu")); + free(source); +} + + +UT_TEST(test_local_master_read_image_retries_holder_busy_with_fresh_identity) +{ + char *source = read_gcs_block_source(); + const char *read_image + = source != NULL ? strstr(source, "\ncluster_gcs_local_master_read_image_and_wait(") : NULL; + const char *read_image_end = read_image != NULL ? strstr(read_image, "\n}\n") : NULL; + const char *budget; + const char *loop; + const char *backoff; + const char *fresh_id; + const char *slot_id; + const char *forward_id; + const char *send; + const char *retryable_deny; + const char *retry; + const char *terminal_error; + + /* P0-21 completion: a DATA worker must never wait behind BufferContent. + * Its conditional-copy refusal is HC105 transient, so the local-master + * requester retries with bounded backoff. Each re-arm mints a new request + * id, preventing a delayed denial from an earlier attempt from winning the + * next attempt's slot. */ + UT_ASSERT_NOT_NULL(read_image); + UT_ASSERT_NOT_NULL(read_image_end); + if (read_image == NULL || read_image_end == NULL) { + free(source); + return; + } + budget = strstr(read_image, "cluster_gcs_block_retransmit_max_retries"); + loop = budget != NULL ? strstr(budget, "for (retry_attempt = 0;") : NULL; + backoff = loop != NULL ? strstr(loop, "gcs_block_backoff_ms_for_retry(retry_attempt)") : NULL; + fresh_id + = backoff != NULL ? strstr(backoff, "gcs_block_pcm_x_next_request_id(&request_id)") : NULL; + slot_id = fresh_id != NULL ? strstr(fresh_id, "slot->request_id = request_id") : NULL; + forward_id = slot_id != NULL ? strstr(slot_id, "fwd.request_id = request_id") : NULL; + send = forward_id != NULL ? strstr(forward_id, "cluster_grd_outbound_enqueue_backend_msg(") + : NULL; + retryable_deny = send != NULL ? strstr(send, "GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER") : NULL; + retry = retryable_deny != NULL ? strstr(retryable_deny, "continue;") : NULL; + terminal_error + = retry != NULL ? strstr(retry, "could not obtain read image from X holder") : NULL; + UT_ASSERT_NOT_NULL(budget); + UT_ASSERT_NOT_NULL(loop); + UT_ASSERT_NOT_NULL(backoff); + UT_ASSERT_NOT_NULL(fresh_id); + UT_ASSERT_NOT_NULL(slot_id); + UT_ASSERT_NOT_NULL(forward_id); + UT_ASSERT_NOT_NULL(send); + UT_ASSERT_NOT_NULL(retryable_deny); + UT_ASSERT_NOT_NULL(retry); + UT_ASSERT_NOT_NULL(terminal_error); + if (budget != NULL && loop != NULL && backoff != NULL && fresh_id != NULL && slot_id != NULL + && forward_id != NULL && send != NULL && retryable_deny != NULL && retry != NULL + && terminal_error != NULL) + UT_ASSERT(read_image < budget && budget < loop && loop < backoff && backoff < fresh_id + && fresh_id < slot_id && slot_id < forward_id && forward_id < send + && send < retryable_deny && retryable_deny < retry && retry < terminal_error + && terminal_error < read_image_end); + free(source); +} + +UT_TEST(test_local_master_read_image_stops_retrying_displaced_holder_exactly) +{ + char *source = read_gcs_block_source(); + const char *read_image + = source != NULL ? strstr(source, "\ncluster_gcs_local_master_read_image_and_wait(") : NULL; + const char *read_image_end = read_image != NULL ? strstr(read_image, "\n}\n") : NULL; + const char *expected_arg; + const char *retry_arg; + const char *precheck; + const char *reserve; + const char *backoff; + const char *backoff_check; + const char *fresh_id; + const char *denial; + const char *denial_check; + const char *drift_retry; + const char *release; + const char *drift_return; + const char *terminal_error; + + /* P0-21 residual: a conditional refusal is retryable only while the + * complete remote-X authority token still names the same holder. Once a + * queue handoff displaces that token, this helper must stop spending its + * bounded budget on the old node and return the retry boundary to bufmgr; + * the outer GRANT_PENDING abort/rearm owns fresh authority selection. */ + UT_ASSERT_NOT_NULL(read_image); + UT_ASSERT_NOT_NULL(read_image_end); + if (read_image == NULL || read_image_end == NULL) { + free(source); + return; + } + expected_arg = strstr(read_image, "const PcmAuthoritySnapshot *expected"); + retry_arg = expected_arg != NULL ? strstr(expected_arg, "bool *out_retry_denied") : NULL; + precheck = retry_arg != NULL + ? strstr(retry_arg, "if (!cluster_pcm_lock_authority_matches(tag, expected))") + : NULL; + reserve = precheck != NULL ? strstr(precheck, "gcs_block_reserve_slot(") : NULL; + backoff + = reserve != NULL ? strstr(reserve, "gcs_block_backoff_ms_for_retry(retry_attempt)") : NULL; + backoff_check = backoff != NULL + ? strstr(backoff, "if (!cluster_pcm_lock_authority_matches(tag, expected))") + : NULL; + fresh_id = backoff_check != NULL + ? strstr(backoff_check, "gcs_block_pcm_x_next_request_id(&request_id)") + : NULL; + denial = fresh_id != NULL ? strstr(fresh_id, "GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER") : NULL; + denial_check = denial != NULL + ? strstr(denial, "if (!cluster_pcm_lock_authority_matches(tag, expected))") + : NULL; + drift_retry = denial_check != NULL ? strstr(denial_check, "*out_retry_denied = true") : NULL; + release = drift_retry != NULL ? strstr(drift_retry, "gcs_block_release_slot(slot)") : NULL; + drift_return = release != NULL ? strstr(release, "if (*out_retry_denied)") : NULL; + terminal_error = drift_return != NULL + ? strstr(drift_return, "could not obtain read image from X holder") + : NULL; + + UT_ASSERT_NOT_NULL(expected_arg); + UT_ASSERT_NOT_NULL(retry_arg); + UT_ASSERT_NOT_NULL(precheck); + UT_ASSERT_NOT_NULL(reserve); + UT_ASSERT_NOT_NULL(backoff); + UT_ASSERT_NOT_NULL(backoff_check); + UT_ASSERT_NOT_NULL(fresh_id); + UT_ASSERT_NOT_NULL(denial); + UT_ASSERT_NOT_NULL(denial_check); + UT_ASSERT_NOT_NULL(drift_retry); + UT_ASSERT_NOT_NULL(release); + UT_ASSERT_NOT_NULL(drift_return); + UT_ASSERT_NOT_NULL(terminal_error); + if (expected_arg != NULL && retry_arg != NULL && precheck != NULL && reserve != NULL + && backoff != NULL && backoff_check != NULL && fresh_id != NULL && denial != NULL + && denial_check != NULL && drift_retry != NULL && release != NULL && drift_return != NULL + && terminal_error != NULL) + UT_ASSERT(read_image < expected_arg && expected_arg < retry_arg && retry_arg < precheck + && precheck < reserve && reserve < backoff && backoff < backoff_check + && backoff_check < fresh_id && fresh_id < denial && denial < denial_check + && denial_check < drift_retry && drift_retry < release && release < drift_return + && drift_return < terminal_error && terminal_error < read_image_end); + free(source); +} + +UT_TEST(test_local_master_read_image_refusal_evidence_is_attempt_exact) +{ + char *source = read_gcs_block_source(); + char *t400 = read_source_path(T400_SOURCE_PATH); + const char *read_image + = source != NULL ? strstr(source, "\ncluster_gcs_local_master_read_image_and_wait(") : NULL; + const char *read_image_end = read_image != NULL ? strstr(read_image, "\n}\n") : NULL; + const char *attempts = read_image != NULL ? strstr(read_image, "attempts=%d") : NULL; + const char *last_status = attempts != NULL ? strstr(attempts, "last_status=%d") : NULL; + const char *forward + = source != NULL ? strstr(source, "\ncluster_gcs_handle_block_forward_envelope(") : NULL; + const char *forward_end = forward != NULL ? strstr(forward, "\n}\n") : NULL; + const char *refusal_name + = forward != NULL ? strstr(forward, "cluster_bufmgr_gcs_copy_refusal_name") : NULL; + const char *refusal_log + = refusal_name != NULL ? strstr(refusal_name, "holder ship image refused") : NULL; + const char *request_id = refusal_log != NULL ? strstr(refusal_log, "request_id=") : NULL; + + /* P0-21 observation only: the requester terminal error reports its exact + * bounded-attempt result, while the holder process log binds each copy + * refusal reason to the forwarded request identity. */ + UT_ASSERT_NOT_NULL(read_image); + UT_ASSERT_NOT_NULL(read_image_end); + UT_ASSERT_NOT_NULL(attempts); + UT_ASSERT_NOT_NULL(last_status); + if (read_image_end != NULL && attempts != NULL && last_status != NULL) + UT_ASSERT(attempts < last_status && last_status < read_image_end); + UT_ASSERT_NOT_NULL(forward); + UT_ASSERT_NOT_NULL(forward_end); + UT_ASSERT_NOT_NULL(refusal_name); + UT_ASSERT_NOT_NULL(refusal_log); + UT_ASSERT_NOT_NULL(request_id); + if (forward_end != NULL && refusal_name != NULL && refusal_log != NULL && request_id != NULL) + UT_ASSERT(refusal_name < refusal_log && refusal_log < request_id + && request_id < forward_end); + UT_ASSERT_NOT_NULL(t400); + if (t400 != NULL) { + const char *holder_before = strstr(t400, "holder_evicted_before_by_node"); + const char *holder_after + = holder_before != NULL ? strstr(holder_before, "holder_evicted_after_by_node") : NULL; + const char *holder_diag + = holder_after != NULL ? strstr(holder_after, "holder copy refusal baseline") : NULL; + const char *holder_final = holder_diag != NULL + ? strstr(holder_diag, "'block_forward_holder_evicted_count'") + : NULL; + + UT_ASSERT_NOT_NULL(holder_before); + UT_ASSERT_NOT_NULL(holder_after); + UT_ASSERT_NOT_NULL(holder_diag); + UT_ASSERT_NOT_NULL(holder_final); + if (holder_before != NULL && holder_after != NULL && holder_diag != NULL + && holder_final != NULL) + UT_ASSERT(holder_before < holder_after && holder_after < holder_diag + && holder_diag < holder_final); + } + free(source); + free(t400); +} + +UT_TEST(test_local_master_x_transfer_revalidates_exact_authority_and_retries_stale) +{ + char *source = read_gcs_block_source(); + const char *transfer + = source != NULL ? strstr(source, "\ncluster_gcs_local_master_x_transfer_and_wait(") : NULL; + const char *transfer_end = transfer != NULL ? strstr(transfer, "\n}\n") : NULL; + const char *precheck; + const char *reserve; + const char *forward_id; + const char *send; + const char *installed; + const char *commit; + const char *stale; + const char *not_found; + const char *commit_retry; + const char *postcheck; + const char *post_retry; + const char *legacy_terminal; + + /* P0-26: carry one complete authority token from the local-master + * decision through the wire request. Reject drift before sending; after + * an installed reply, atomically commit only that token. A displaced + * token or a denial/timeout after authority drift returns retryable so + * bufmgr aborts GRANT_PENDING and mints a fresh token/request identity. */ + UT_ASSERT_NOT_NULL(transfer); + UT_ASSERT_NOT_NULL(transfer_end); + if (transfer == NULL || transfer_end == NULL) { + free(source); + return; + } + precheck = strstr(transfer, "if (!cluster_pcm_lock_authority_matches(tag, expected))"); + reserve = precheck != NULL ? strstr(precheck, "gcs_block_reserve_slot(") : NULL; + forward_id = reserve != NULL ? strstr(reserve, "fwd.request_id = request_id") : NULL; + send = forward_id != NULL ? strstr(forward_id, "cluster_grd_outbound_enqueue_backend_msg(") + : NULL; + installed = send != NULL ? strstr(send, "if (installed)") : NULL; + commit = installed != NULL ? strstr(installed, "cluster_pcm_lock_master_take_x_after_transfer(") + : NULL; + stale = commit != NULL ? strstr(commit, "PCM_X_TRANSFER_COMMIT_STALE") : NULL; + not_found = stale != NULL ? strstr(stale, "PCM_X_TRANSFER_COMMIT_NOT_FOUND") : NULL; + commit_retry = not_found != NULL ? strstr(not_found, "*out_retry_denied = true") : NULL; + postcheck + = commit_retry != NULL + ? strstr(commit_retry, "if (!cluster_pcm_lock_authority_matches(tag, expected))") + : NULL; + post_retry = postcheck != NULL ? strstr(postcheck, "*out_retry_denied = true") : NULL; + legacy_terminal = post_retry != NULL ? strstr(post_retry, "if (read_image)") : NULL; + + UT_ASSERT_NOT_NULL(precheck); + UT_ASSERT_NOT_NULL(reserve); + UT_ASSERT_NOT_NULL(forward_id); + UT_ASSERT_NOT_NULL(send); + UT_ASSERT_NOT_NULL(installed); + UT_ASSERT_NOT_NULL(commit); + UT_ASSERT_NOT_NULL(stale); + UT_ASSERT_NOT_NULL(not_found); + UT_ASSERT_NOT_NULL(commit_retry); + UT_ASSERT_NOT_NULL(postcheck); + UT_ASSERT_NOT_NULL(post_retry); + UT_ASSERT_NOT_NULL(legacy_terminal); + if (precheck != NULL && reserve != NULL && forward_id != NULL && send != NULL + && installed != NULL && commit != NULL && stale != NULL && not_found != NULL + && commit_retry != NULL && postcheck != NULL && post_retry != NULL + && legacy_terminal != NULL) + UT_ASSERT(transfer < precheck && precheck < reserve && reserve < forward_id + && forward_id < send && send < installed && installed < commit && commit < stale + && stale < not_found && not_found < commit_retry && commit_retry < postcheck + && postcheck < post_retry && post_retry < legacy_terminal + && legacy_terminal < transfer_end); + free(source); +} + +UT_TEST(test_remote_downgrade_prepares_exact_image_before_notify_and_reply) +{ + char *gcs_source = read_gcs_block_source(); + char *bufmgr_source = read_source_path("../../backend/storage/buffer/bufmgr.c"); + const char *forward = gcs_source != NULL + ? strstr(gcs_source, "\ncluster_gcs_handle_block_forward_envelope(") + : NULL; + const char *forward_end = forward != NULL ? strstr(forward, "\n}\n") : NULL; + const char *inject + = forward != NULL ? strstr(forward, "cluster-gcs-block-evict-holder-before-ship") : NULL; + const char *downgrade + = inject != NULL + ? strstr(inject, "cluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image(") + : NULL; + const char *reuse + = downgrade != NULL ? strstr(downgrade, "holder_ship_ok = remote_downgraded") : NULL; + const char *post_notify_guard + = downgrade != NULL + ? strstr(downgrade, "remote_downgrade_outcome == " + "CLUSTER_BUFMGR_GCS_DOWNGRADE_FAILCLOSED_POST_NOTIFY") + : NULL; + const char *post_notify_return + = post_notify_guard != NULL ? strstr(post_notify_guard, "return;") : NULL; + const char *copy = reuse != NULL ? strstr(reuse, "gcs_block_get_ship_image(") : NULL; + const char *helper + = bufmgr_source != NULL + ? strstr(bufmgr_source, + "\ncluster_bufmgr_downgrade_x_to_s_remote_for_gcs_prepare_image(") + : NULL; + const char *helper_end = helper != NULL ? strstr(helper, "\n}\n") : NULL; + const char *reserve + = helper != NULL ? strstr(helper, "cluster_bufmgr_pcm_own_begin_x_revoke(") : NULL; + const char *precopy = reserve != NULL ? strstr(reserve, "memcpy(dst,") : NULL; + const char *notify + = precopy != NULL ? strstr(precopy, "cluster_gcs_send_transition_nowait(") : NULL; + const char *commit + = notify != NULL ? strstr(notify, "cluster_bufmgr_pcm_own_finish_x_to_s_downgrade(") : NULL; + const char *abort + = helper != NULL ? strstr(helper, "cluster_bufmgr_pcm_own_abort_x_revoke(") : NULL; + const char *local_helper + = bufmgr_source != NULL + ? strstr(bufmgr_source, "\ncluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image(") + : NULL; + const char *local_helper_end = local_helper != NULL ? strstr(local_helper, "\n}\n") : NULL; + const char *local_reserve = local_helper != NULL + ? strstr(local_helper, "cluster_bufmgr_pcm_own_begin_x_revoke(") + : NULL; + const char *local_copy = local_reserve != NULL ? strstr(local_reserve, "memcpy(dst,") : NULL; + const char *local_master + = local_copy != NULL ? strstr(local_copy, "cluster_pcm_lock_apply_gcs_transition(") : NULL; + const char *local_commit + = local_master != NULL + ? strstr(local_master, "cluster_bufmgr_pcm_own_finish_x_to_s_downgrade(") + : NULL; + const char *local_call + = gcs_source != NULL + ? strstr(gcs_source, "cluster_bufmgr_downgrade_x_to_s_for_gcs_prepare_image(") + : NULL; + const char *prepared + = local_call != NULL ? strstr(local_call, "scache_image_prepared = true") : NULL; + const char *produce + = prepared != NULL + ? strstr(prepared, "gcs_block_produce_reply(req, block_buf, scache_image_prepared") + : NULL; + const char *produce_def + = gcs_source != NULL ? strstr(gcs_source, "\ngcs_block_produce_reply(") : NULL; + const char *skip_second_copy + = produce_def != NULL + ? strstr(produce_def, "if (!preprepared_image\n\t\t&& !gcs_block_get_ship_image(") + : NULL; + + /* P0-32: injection is decided before any irreversible downgrade. The + * successful arm reuses bytes proven under the same content EXCLUSIVE + * interval; it must not take a second conditional copy. */ + UT_ASSERT_NOT_NULL(forward); + UT_ASSERT_NOT_NULL(forward_end); + UT_ASSERT_NOT_NULL(inject); + UT_ASSERT_NOT_NULL(downgrade); + UT_ASSERT_NOT_NULL(reuse); + UT_ASSERT_NOT_NULL(post_notify_guard); + UT_ASSERT_NOT_NULL(post_notify_return); + UT_ASSERT_NOT_NULL(copy); + if (forward_end != NULL && inject != NULL && downgrade != NULL && reuse != NULL && copy != NULL) + UT_ASSERT(inject < downgrade && downgrade < reuse && reuse < copy && copy < forward_end); + if (post_notify_guard != NULL && post_notify_return != NULL && copy != NULL) + UT_ASSERT(downgrade < post_notify_guard && post_notify_guard < post_notify_return + && post_notify_return < copy); + + /* The remote holder's only irreversible order is exact reserve, flush + + * pre-copy/LSN proof, nowait notify, then exact X+REVOKING -> S commit. + * Pre-notify refusal has an exact abort; post-notify commit failure is + * fail-closed and therefore cannot produce S_GRANTED. */ + UT_ASSERT_NOT_NULL(helper); + UT_ASSERT_NOT_NULL(helper_end); + UT_ASSERT_NOT_NULL(reserve); + UT_ASSERT_NOT_NULL(precopy); + UT_ASSERT_NOT_NULL(notify); + UT_ASSERT_NOT_NULL(commit); + UT_ASSERT_NOT_NULL(abort); + if (helper_end != NULL && reserve != NULL && precopy != NULL && notify != NULL + && commit != NULL) + UT_ASSERT(reserve < precopy && precopy < notify && notify < commit && commit < helper_end); + if (helper_end != NULL && abort != NULL) + UT_ASSERT(abort < notify && abort < helper_end); + UT_ASSERT_NOT_NULL(helper != NULL ? strstr(helper, "cluster_pcm_x_runtime_fail_closed()") + : NULL); + UT_ASSERT_NOT_NULL(local_helper); + UT_ASSERT_NOT_NULL(local_helper_end); + UT_ASSERT_NOT_NULL(local_reserve); + UT_ASSERT_NOT_NULL(local_copy); + UT_ASSERT_NOT_NULL(local_master); + UT_ASSERT_NOT_NULL(local_commit); + if (local_helper_end != NULL && local_reserve != NULL && local_copy != NULL + && local_master != NULL && local_commit != NULL) + UT_ASSERT(local_reserve < local_copy && local_copy < local_master + && local_master < local_commit && local_commit < local_helper_end); + UT_ASSERT_NOT_NULL(local_call); + UT_ASSERT_NOT_NULL(prepared); + UT_ASSERT_NOT_NULL(produce); + UT_ASSERT_NOT_NULL(skip_second_copy); + if (local_call != NULL && prepared != NULL && produce != NULL) + UT_ASSERT(local_call < prepared && prepared < produce); + free(gcs_source); + free(bufmgr_source); +} + + +UT_TEST(test_pending_x_apply_race_maps_to_retryable_block_denial) +{ + char *source = read_gcs_block_source(); + const char *produce = source != NULL ? strstr(source, "\ngcs_block_produce_reply(") : NULL; + const char *produce_end + = produce != NULL + ? strstr(produce, "\n}\n\nstatic bool\ngcs_block_queue_pending_x_authoritative(") + : NULL; + const char *apply1; + const char *deny1; + const char *apply2; + const char *deny2; + const char *requester_retry; + + /* P0-25: the entry-lock final admission distinguishes a raced pending-X + * from structural incompatibility. Both direct-storage and image-present + * N->S apply sites must return DENIED_PENDING_X, whose requester path exits + * through the existing fresh-identity retry boundary instead of a client + * terminal ERROR. */ + UT_ASSERT_NOT_NULL(produce); + UT_ASSERT_NOT_NULL(produce_end); + if (produce == NULL || produce_end == NULL) { + free(source); + return; + } + apply1 = strstr(produce, "cluster_pcm_lock_apply_gcs_transition_result("); + deny1 = apply1 != NULL ? strstr(apply1, "GcsBlockApplyRefusalStatus(") : NULL; + apply2 = deny1 != NULL ? strstr(deny1, "cluster_pcm_lock_apply_gcs_transition_result(") : NULL; + deny2 = apply2 != NULL ? strstr(apply2, "GcsBlockApplyRefusalStatus(") : NULL; + requester_retry = strstr(source, "if (final_status == GCS_BLOCK_REPLY_DENIED_PENDING_X)"); + UT_ASSERT_NOT_NULL(apply1); + UT_ASSERT_NOT_NULL(deny1); + UT_ASSERT_NOT_NULL(apply2); + UT_ASSERT_NOT_NULL(deny2); + UT_ASSERT_NOT_NULL(requester_retry); + if (apply1 != NULL && deny1 != NULL && apply2 != NULL && deny2 != NULL) + UT_ASSERT(produce < apply1 && apply1 < deny1 && deny1 < apply2 && apply2 < deny2 + && deny2 < produce_end); + free(source); +} + + +UT_TEST(test_gcs_apply_state_drift_restarts_with_fresh_request_identity) +{ + char *source = read_gcs_block_source(); + const char *produce = source != NULL ? strstr(source, "\ngcs_block_produce_reply(") : NULL; + const char *produce_end + = produce != NULL + ? strstr(produce, "\n}\n\nstatic bool\ngcs_block_queue_pending_x_authoritative(") + : NULL; + const char *apply1; + const char *map1; + const char *apply2; + const char *map2; + + /* The request transition was already wire-validated. If authority moves + * after the master's outer S/X decision but before its entry-lock apply, + * N->S/N->X must abandon this attempt and re-enter bufmgr with a fresh + * request/token identity. Other transition incompatibilities remain + * structural terminal denials. */ + UT_ASSERT_EQ((int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_PENDING_X, PCM_TRANS_N_TO_S), + (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ((int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_INCOMPATIBLE, PCM_TRANS_N_TO_S), + (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ((int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_INCOMPATIBLE, PCM_TRANS_N_TO_X), + (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ( + (int)GcsBlockApplyRefusalStatus(PCM_GCS_TRANSITION_INCOMPATIBLE, PCM_TRANS_S_TO_X_UPGRADE), + (int)GCS_BLOCK_REPLY_DENIED_INCOMPATIBLE); + + UT_ASSERT_NOT_NULL(produce); + UT_ASSERT_NOT_NULL(produce_end); + if (produce == NULL || produce_end == NULL) { + free(source); + return; + } + apply1 = strstr(produce, "cluster_pcm_lock_apply_gcs_transition_result("); + map1 = apply1 != NULL ? strstr(apply1, "GcsBlockApplyRefusalStatus(") : NULL; + apply2 = map1 != NULL ? strstr(map1, "cluster_pcm_lock_apply_gcs_transition_result(") : NULL; + map2 = apply2 != NULL ? strstr(apply2, "GcsBlockApplyRefusalStatus(") : NULL; + UT_ASSERT_NOT_NULL(apply1); + UT_ASSERT_NOT_NULL(map1); + UT_ASSERT_NOT_NULL(apply2); + UT_ASSERT_NOT_NULL(map2); + if (apply1 != NULL && map1 != NULL && apply2 != NULL && map2 != NULL) + UT_ASSERT(produce < apply1 && apply1 < map1 && map1 < apply2 && apply2 < map2 + && map2 < produce_end); + free(source); +} + + +UT_TEST(test_master_direct_copy_busy_uses_only_fresh_identity_retry_boundary) +{ + struct CopyRefusalCase { + ClusterBufmgrGcsCopyRefusal refusal; + GcsBlockReplyStatus expected; + }; + static const struct CopyRefusalCase cases[] = { + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NONE, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INVALID_ARGUMENT, + GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID, + GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST, GCS_BLOCK_REPLY_DENIED_PENDING_X }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_SECOND, GCS_BLOCK_REPLY_DENIED_PENDING_X }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_OWNERSHIP_REVOKE_BUSY, + GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_HC89_LSN_DRIFT, + GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_SMART_FUSION_UNCLASSIFIED, + GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + { CLUSTER_BUFMGR_GCS_COPY_REFUSAL_INJECTED_EVICT, + GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER }, + }; + char *source = read_gcs_block_source(); + const char *produce = source != NULL ? strstr(source, "\ngcs_block_produce_reply(") : NULL; + const char *produce_end + = produce != NULL + ? strstr(produce, "\n}\n\nstatic bool\ngcs_block_queue_pending_x_authoritative(") + : NULL; + const char *copy_refusal; + const char *copy_refusal_end; + const char *mapping; + const char *terminal_literal; + const char *requester_retry; + const char *retry_flag; + const char *retry_break; + const char *same_id_continue; + size_t i; + + /* P0-21 residual: a DATA worker's conditional BufferContent refusal is + * transient by construction (the lock owner may itself be waiting for that + * worker), so only the two conditional stages enter the existing + * DENIED_PENDING_X fresh-request/token retry boundary. Structural + * residency/current-image failures and HC89's bounded second LSN drift stay + * terminal status 6; no authority or safety gate is weakened. */ + for (i = 0; i < lengthof(cases); i++) + UT_ASSERT_EQ((int)GcsBlockMasterDirectCopyRefusalStatus(cases[i].refusal), + (int)cases[i].expected); + + UT_ASSERT_NOT_NULL(produce); + UT_ASSERT_NOT_NULL(produce_end); + if (produce == NULL || produce_end == NULL) { + free(source); + return; + } + copy_refusal = strstr(produce, "produce-copy-refused:%s"); + copy_refusal_end = copy_refusal != NULL ? strstr(copy_refusal, "return true;") : NULL; + mapping = copy_refusal != NULL + ? strstr(copy_refusal, "GcsBlockMasterDirectCopyRefusalStatus(copy_refusal)") + : NULL; + terminal_literal + = copy_refusal != NULL + ? strstr(copy_refusal, "*out_status = GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER") + : NULL; + UT_ASSERT_NOT_NULL(copy_refusal); + UT_ASSERT_NOT_NULL(copy_refusal_end); + UT_ASSERT_NOT_NULL(mapping); + if (copy_refusal != NULL && copy_refusal_end != NULL && mapping != NULL) + UT_ASSERT(copy_refusal < mapping && mapping < copy_refusal_end); + UT_ASSERT(terminal_literal == NULL || copy_refusal_end == NULL + || terminal_literal > copy_refusal_end); + + /* Status 10 must leave the per-call retransmit loop rather than retrying + * the same cached request id. Bufmgr then aborts the exact old reservation, + * waits, and rearms a fresh token/request identity (covered by + * test_pending_x_denied_retry_leaves_master_invalidate_gap). */ + requester_retry = strstr(source, "if (final_status == GCS_BLOCK_REPLY_DENIED_PENDING_X)"); + retry_flag = requester_retry != NULL ? strstr(requester_retry, "retry_denied = true;") : NULL; + retry_break = retry_flag != NULL ? strstr(retry_flag, "break;") : NULL; + same_id_continue = retry_flag != NULL ? strstr(retry_flag, "continue;") : NULL; + UT_ASSERT_NOT_NULL(requester_retry); + UT_ASSERT_NOT_NULL(retry_flag); + UT_ASSERT_NOT_NULL(retry_break); + if (requester_retry != NULL && retry_flag != NULL && retry_break != NULL) + UT_ASSERT(requester_retry < retry_flag && retry_flag < retry_break + && retry_break < produce); + UT_ASSERT(same_id_continue == NULL || retry_break == NULL || same_id_continue > retry_break); + free(source); +} + + +UT_TEST(test_master_not_holder_producers_log_one_coherent_authority_snapshot) +{ + char *source = read_gcs_block_source(); + const char *helper + = source != NULL ? strstr(source, "\ngcs_block_log_master_not_holder_producer(") : NULL; + const char *helper_end = helper != NULL ? strstr(helper, "\n}\n") : NULL; + const char *produce = source != NULL ? strstr(source, "\ngcs_block_produce_reply(") : NULL; + const char *produce_end + = produce != NULL + ? strstr(produce, "\n}\n\nstatic bool\ngcs_block_queue_pending_x_authoritative(") + : NULL; + static const char *const required_fields[] = { + "cluster_pcm_lock_authority_snapshot(tag, &authority)", + "authority.state", + "authority.x_holder_node", + "authority.s_holders_bitmap", + "authority.master_holder.node_id", + "authority.master_holder.procno", + "authority.master_holder.cluster_epoch", + "authority.master_holder.request_id", + "authority.pending_x_requester_node", + "authority.pending_x_since_lsn", + "authority.transition_count", + }; + static const char *const required_reasons[] = { + "direct-land-nonsendable", "direct-land-forward-rearm", + "pcm-x-image-not-ready", "produce-no-resident-authority", + "produce-copy-refused", "clean-third-party-master", + "live-x-other-holder", "pending-x-reserve-failed", + "x-forward-send-failed", "x-state-holder-unroutable", + "holder-immediate-deny", "holder-drop-pinned", + "holder-drop-stale", "holder-copy-refused", + }; + size_t i; + + /* A status=6 requester error is not enough to identify its producer: the + * aggregate counter is also incremented at requester consumption. Every + * producer therefore logs a stable reason plus one entry-lock-coherent + * authority snapshot, correlated by the unchanged request identity. */ + UT_ASSERT_NOT_NULL(helper); + UT_ASSERT_NOT_NULL(helper_end); + if (helper != NULL && helper_end != NULL) { + const char *snapshot; + const char *duplicate_snapshot; + const char *forbidden; + + for (i = 0; i < lengthof(required_fields); i++) { + const char *field = strstr(helper, required_fields[i]); + + UT_ASSERT_NOT_NULL(field); + if (field != NULL) + UT_ASSERT(field < helper_end); + } + snapshot = strstr(helper, "cluster_pcm_lock_authority_snapshot("); + duplicate_snapshot = snapshot != NULL + ? strstr(snapshot + 1, "cluster_pcm_lock_authority_snapshot(") + : NULL; + UT_ASSERT_NOT_NULL(snapshot); + UT_ASSERT(snapshot != NULL && snapshot < helper_end); + UT_ASSERT(duplicate_snapshot == NULL || duplicate_snapshot >= helper_end); + forbidden = strstr(helper, "cluster_pcm_lock_query("); + UT_ASSERT(forbidden == NULL || forbidden >= helper_end); + forbidden = strstr(helper, "cluster_pcm_master_holder_node_by_tag("); + UT_ASSERT(forbidden == NULL || forbidden >= helper_end); + forbidden = strstr(helper, "cluster_pcm_lock_query_s_holders_bitmap("); + UT_ASSERT(forbidden == NULL || forbidden >= helper_end); + } + for (i = 0; i < lengthof(required_reasons); i++) + UT_ASSERT_NOT_NULL(source != NULL ? strstr(source, required_reasons[i]) : NULL); + /* The final master-direct copy refusal must retain the bufmgr's exact + * nonblocking stage in the producer reason; no status/retry change here. */ + UT_ASSERT_NOT_NULL(produce); + UT_ASSERT_NOT_NULL(produce_end); + if (produce != NULL && produce_end != NULL) { + const char *capture = strstr(produce, "ClusterBufmgrGcsCopyRefusal copy_refusal"); + const char *pass = strstr(produce, "©_refusal"); + const char *name = strstr(produce, "cluster_bufmgr_gcs_copy_refusal_name(copy_refusal)"); + + UT_ASSERT_NOT_NULL(capture); + UT_ASSERT_NOT_NULL(pass); + UT_ASSERT_NOT_NULL(name); + if (capture != NULL && pass != NULL && name != NULL) + UT_ASSERT(produce < capture && capture < pass && pass < name && name < produce_end); + } + free(source); +} + + +UT_TEST(test_pi_durable_note_drain_stages_before_consuming_on_data_plane) +{ + char *source = read_gcs_block_source(); + const char *drain = strstr(source, "\ncluster_gcs_block_pi_discard_drain("); + const char *drain_end = drain != NULL ? strstr(drain, "\n}\n") : NULL; + const char *data_plane; + const char *status3; + const char *shard; + const char *enqueue; + const char *enqueue_refusal; + const char *advance; + const char *data_continue; + const char *control_apply; + const char *control_send; + + /* + * Checkpoint-confirmed PI durable notes are produced in shared memory, + * but the DATA-plane drain runs only in LMS worker 0. It must therefore + * stage both local- and remote-master status-3 ACKs onto tag->worker, + * and may consume the source note only after that staging succeeds. The + * old CONTROL-plane direct apply/send path remains available. + */ + UT_ASSERT_NOT_NULL(drain); + UT_ASSERT_NOT_NULL(drain_end); + if (drain == NULL || drain_end == NULL) { + free(source); + return; + } + data_plane = strstr(drain, "cluster_gcs_block_family_on_data_plane()"); + status3 = strstr(drain, "GCS_BLOCK_INVALIDATE_ACK_STATUS_PI_DURABLE_NOTE"); + shard = strstr(drain, "cluster_lms_shard_for_tag(&tag, cluster_lms_workers)"); + enqueue = strstr(drain, "cluster_lms_outbound_enqueue("); + enqueue_refusal = enqueue != NULL ? strstr(enqueue, "break;") : NULL; + advance = enqueue != NULL ? strstr(enqueue, "ClusterGcsBlock->pi_note_drain_seq++") : NULL; + data_continue = advance != NULL ? strstr(advance, "continue;") : NULL; + control_apply = strstr(drain, "gcs_block_pi_discard_master_apply(tag, page_scn)"); + control_send = strstr(drain, "cluster_ic_send_envelope(PGRAC_IC_MSG_GCS_BLOCK_INVALIDATE_ACK"); + UT_ASSERT_NOT_NULL(data_plane); + UT_ASSERT_NOT_NULL(status3); + UT_ASSERT_NOT_NULL(shard); + UT_ASSERT_NOT_NULL(enqueue); + UT_ASSERT_NOT_NULL(enqueue_refusal); + UT_ASSERT_NOT_NULL(advance); + UT_ASSERT_NOT_NULL(data_continue); + UT_ASSERT_NOT_NULL(control_apply); + UT_ASSERT_NOT_NULL(control_send); + if (data_plane != NULL && status3 != NULL && shard != NULL && enqueue != NULL + && enqueue_refusal != NULL && advance != NULL && data_continue != NULL) + UT_ASSERT(data_plane < status3 && status3 < shard && shard < enqueue + && enqueue < enqueue_refusal && enqueue_refusal < advance + && advance < data_continue && data_continue < drain_end); + if (control_apply != NULL) + UT_ASSERT(data_continue < control_apply && control_apply < drain_end); + if (control_send != NULL) + UT_ASSERT(data_continue < control_send && control_send < drain_end); + free(source); +} + + +UT_TEST(test_pi_durable_note_receive_is_observable_before_apply) +{ + char *source = read_gcs_block_source(); + const char *handler + = source != NULL ? strstr(source, "\ncluster_gcs_handle_block_invalidate_ack_envelope(") + : NULL; + const char *status3 = handler != NULL + ? strstr(handler, "GCS_BLOCK_INVALIDATE_ACK_STATUS_PI_DURABLE_NOTE") + : NULL; + const char *epoch + = status3 != NULL ? strstr(status3, "ack->epoch == cluster_epoch_get_current()") : NULL; + const char *accepted = epoch != NULL ? strstr(epoch, "pi_durable_note_apply_count") : NULL; + const char *apply + = accepted != NULL ? strstr(accepted, "gcs_block_pi_discard_master_apply(") : NULL; + + UT_ASSERT_NOT_NULL(handler); + UT_ASSERT_NOT_NULL(status3); + UT_ASSERT_NOT_NULL(epoch); + UT_ASSERT_NOT_NULL(accepted); + UT_ASSERT_NOT_NULL(apply); + if (handler != NULL && status3 != NULL && epoch != NULL && accepted != NULL && apply != NULL) + UT_ASSERT(handler < status3 && status3 < epoch && epoch < accepted && accepted < apply); + free(source); +} + +/* P0-20 source-floor V2 must remain bound to the HELLO connection sampled + * around source authority. The reliable REVOKE leg is armed first; a later + * generation drift returns retryable without ACKing it, while the exact-match + * V2 enters a guarded LMS slot. A current peer without the additive bit gets + * the byte-compatible V1 fallback. */ +UT_TEST(test_pcm_x_source_floor_v2_is_connection_bound_until_lms_drain) +{ + char *source = read_gcs_block_source(); + const char *stage_bound; + const char *stage_bound_end; + const char *shard; + const char *enqueue; + const char *transfer; + const char *transfer_end; + const char *auth; + const char *arm; + const char *sample; + const char *generation; + const char *guarded_stage; + const char *v1_fallback; + const char *stray_simple_query; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + stage_bound = strstr(source, "\ngcs_block_pcm_x_stage_frame_cap_bound("); + stage_bound_end = stage_bound != NULL ? strstr(stage_bound, "\n}\n") : NULL; + shard = stage_bound != NULL ? strstr(stage_bound, "cluster_gcs_block_payload_shard(") : NULL; + enqueue = shard != NULL ? strstr(shard, "cluster_lms_outbound_enqueue_cap_bound(") : NULL; + UT_ASSERT_NOT_NULL(stage_bound); + UT_ASSERT_NOT_NULL(stage_bound_end); + UT_ASSERT_NOT_NULL(shard); + UT_ASSERT_NOT_NULL(enqueue); + if (stage_bound != NULL && stage_bound_end != NULL && shard != NULL && enqueue != NULL) + UT_ASSERT(stage_bound < shard && shard < enqueue && enqueue < stage_bound_end); + + transfer = strstr(source, "\ngcs_block_pcm_x_master_drive_transfer("); + transfer_end = transfer != NULL ? strstr(transfer, "\n}\n") : NULL; + auth = transfer != NULL ? strstr(transfer, "gcs_block_pcm_x_authenticated_session_result(") + : NULL; + arm = auth != NULL ? strstr(auth, "cluster_pcm_x_master_revoke_arm_exact(") : NULL; + sample = arm != NULL ? strstr(arm, "cluster_sf_peer_pcm_x_source_floor_sample(") : NULL; + generation = sample != NULL ? strstr(sample, "auth_sample.connection_generation_before") : NULL; + guarded_stage + = generation != NULL ? strstr(generation, "gcs_block_pcm_x_stage_frame_cap_bound(") : NULL; + v1_fallback + = guarded_stage != NULL + ? strstr(guarded_stage, "cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_REVOKE") + : NULL; + stray_simple_query + = transfer != NULL ? strstr(transfer, "cluster_sf_peer_supports_pcm_x_source_floor(source)") + : NULL; + UT_ASSERT_NOT_NULL(transfer); + UT_ASSERT_NOT_NULL(transfer_end); + UT_ASSERT_NOT_NULL(auth); + UT_ASSERT_NOT_NULL(arm); + UT_ASSERT_NOT_NULL(sample); + UT_ASSERT_NOT_NULL(generation); + UT_ASSERT_NOT_NULL(guarded_stage); + UT_ASSERT_NOT_NULL(v1_fallback); + if (transfer != NULL && transfer_end != NULL && auth != NULL && arm != NULL && sample != NULL + && generation != NULL && guarded_stage != NULL && v1_fallback != NULL) { + UT_ASSERT(transfer < auth && auth < arm && arm < sample && sample < generation + && generation < guarded_stage && guarded_stage < v1_fallback + && v1_fallback < transfer_end); + UT_ASSERT(stray_simple_query == NULL || stray_simple_query > transfer_end); + } + free(source); +} + int main(void) { - UT_PLAN(76); + UT_PLAN(103); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -3448,6 +5273,9 @@ main(void) UT_RUN(test_pcm_x_blocker_ack_carries_full_generation_and_binds_master_source); UT_RUN(test_pcm_x_formation_identical_complete_samples_may_revalidate); UT_RUN(test_pcm_x_formation_transient_or_inconsistent_sample_is_tick_noop); + UT_RUN(test_pcm_x_periodic_retry_reports_pre_mutation_exit_stage); + UT_RUN(test_pcm_x_install_ready_v2_self_loopback_is_admissible); + UT_RUN(test_pcm_x_formation_samples_capability_family_atomically); UT_RUN(test_pcm_x_confirm_publish_then_stale_requires_exact_graph_close); UT_RUN(test_pcm_x_drain_poll_binds_exact_master_and_generation); UT_RUN(test_pcm_x_drain_ack_binds_participant_and_canonical_payload); @@ -3458,7 +5286,7 @@ main(void) UT_RUN(test_pcm_x_prepare_grant_ingress_binds_master_to_requester); UT_RUN(test_pcm_x_install_ready_ingress_is_canonical_requester_ack); UT_RUN(test_pcm_x_commit_x_ingress_is_canonical_master_phase); - UT_RUN(test_pcm_x_final_ack_ingress_binds_exact_committed_generation); + UT_RUN(test_pcm_x_final_ack_ingress_binds_monotonic_committed_floor); UT_RUN(test_pcm_x_final_commit_ack_ingress_is_canonical_master_phase); UT_RUN(test_pcm_x_final_confirm_ingress_is_canonical_requester_phase); UT_RUN(test_pcm_x_master_drive_selects_exact_authority_and_next_holder); @@ -3466,14 +5294,22 @@ main(void) UT_RUN(test_pcm_x_cancel_cleanup_classifies_exact_wfg_and_post_clear_failure); UT_RUN(test_pcm_x_terminal_retry_reclaims_cancel_cleanup_after_owner_death); UT_RUN(test_pcm_x_invalidate_ack_matches_only_exact_unacked_holder); + UT_RUN(test_pcm_x_invalidate_busy_routes_to_exact_ticket_backoff); + UT_RUN(test_pcm_x_local_pending_s_denial_match_is_attempt_exact); + UT_RUN(test_pcm_x_grant_pending_invalidate_wakes_local_s_before_busy); + UT_RUN(test_pcm_x_grant_pending_orphan_observation_is_identity_exact); UT_RUN(test_pcm_x_final_ack_builds_exact_grd_handoff_token); + UT_RUN(test_pcm_x_final_ack_fail_closed_names_exact_handoff_stage); UT_RUN(test_pcm_x_holder_image_evidence_never_uses_generation_as_presence); UT_RUN(test_pcm_x_pending_x_marker_is_only_a_pre_handoff_gate); UT_RUN(test_pcm_x_ready_publication_follows_exact_retained_commit); UT_RUN(test_pcm_x_ready_materializes_exact_n_s_or_x_source_without_wire_change); + UT_RUN(test_pcm_x_s_source_hard_failure_observation_is_reason_exact); UT_RUN(test_pcm_x_self_and_remote_drain_share_full_image_release_wrapper); UT_RUN(test_pcm_x_ready_admission_marks_before_send_and_rolls_back_refusal); UT_RUN(test_pcm_x_lms_owner_death_and_restart_audit_fail_closed); + UT_RUN(test_pcm_x_lms_reload_acknowledges_finish_flush_injection); + UT_RUN(test_pcm_x_destructive_finish_fault_times_out_in_sql_before_harness); UT_RUN(test_pcm_x_image_fetch_intercepts_canonical_id_before_generic_dedup); UT_RUN(test_pcm_x_requester_fetch_revalidates_queue_and_reservation_before_install); UT_RUN(test_pcm_x_self_source_handoff_is_no_copy_and_drain_preserves_x); @@ -3488,6 +5324,20 @@ main(void) UT_RUN(test_pcm_x_retire_commit_wakes_exact_waiters_before_ack_or_resolve); UT_RUN(test_pcm_x_tagless_retire_uses_explicit_data_plane_handoff); UT_RUN(test_pcm_x_role_refresh_accepts_only_same_member_promotion); + UT_RUN(test_legacy_byte_proof_sites_republish_kept_pi_mirror); + UT_RUN(test_revoke_handler_silent_refusal_arms_all_note); + UT_RUN(test_local_master_read_image_retries_holder_busy_with_fresh_identity); + UT_RUN(test_local_master_read_image_stops_retrying_displaced_holder_exactly); + UT_RUN(test_local_master_read_image_refusal_evidence_is_attempt_exact); + UT_RUN(test_remote_downgrade_prepares_exact_image_before_notify_and_reply); + UT_RUN(test_local_master_x_transfer_revalidates_exact_authority_and_retries_stale); + UT_RUN(test_pending_x_apply_race_maps_to_retryable_block_denial); + UT_RUN(test_gcs_apply_state_drift_restarts_with_fresh_request_identity); + UT_RUN(test_master_direct_copy_busy_uses_only_fresh_identity_retry_boundary); + UT_RUN(test_master_not_holder_producers_log_one_coherent_authority_snapshot); + UT_RUN(test_pi_durable_note_drain_stages_before_consuming_on_data_plane); + UT_RUN(test_pi_durable_note_receive_is_observable_before_apply); + UT_RUN(test_pcm_x_source_floor_v2_is_connection_bound_until_lms_drain); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_gcs_block_3way.c b/src/test/cluster_unit/test_cluster_gcs_block_3way.c index b929b77267..47b4e2f7e2 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_3way.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_3way.c @@ -292,8 +292,8 @@ UT_TEST(test_placeholder_tap_114_2node_x_transfer) * S→X upgrade with bitmap → broadcast invalidate / X→X holder * direct ship via HC115 forward / invalidate ack timeout 53R91 / * dirty buffer XLogFlush-before-ack (HC123) / epoch validation / - * S barrier DENIED_PENDING_X reply / reader backoff retry → - * eventual grant / starvation 53R92 budget exhaustion. */ + * S barrier DENIED_PENDING_X reply / exact reservation abort / + * fresh-identity reader backoff retry → eventual grant. */ UT_ASSERT(true); } diff --git a/src/test/cluster_unit/test_cluster_gcs_block_dedup.c b/src/test/cluster_unit/test_cluster_gcs_block_dedup.c index ee48f05f3e..a1255861a1 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_dedup.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_dedup.c @@ -82,7 +82,6 @@ UT_DEFINE_GLOBALS(); - void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), @@ -472,7 +471,8 @@ stage_pcm_x_ready(int worker_id, const GcsBlockDedupKey *key, const BufferTag *t result = cluster_gcs_block_dedup_pcm_x_reserve(worker_id, key, tag, &reserved); if (result != GCS_BLOCK_PCM_X_IMAGE_RESERVED && result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) return result; - result = cluster_gcs_block_dedup_pcm_x_materialize(worker_id, key, tag, binding, hdr, page); + result = cluster_gcs_block_dedup_pcm_x_materialize(worker_id, key, tag, binding, UINT64_C(41), + (uint8)PCM_STATE_X, hdr, page); if (result != GCS_BLOCK_PCM_X_IMAGE_STORED && result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) return result; return cluster_gcs_block_dedup_pcm_x_publish_ready_exact(worker_id, key, tag, binding); @@ -1054,7 +1054,7 @@ UT_TEST(u16_capability_routing_truth_table) * field or increasing the entry size. */ UT_TEST(u17_pcm_x_binding_layout_is_zero_entry_growth) { - UT_ASSERT_EQ((int)sizeof(GcsBlockPcmXImageBinding), 136); + UT_ASSERT_EQ((int)sizeof(GcsBlockPcmXImageBinding), 144); UT_ASSERT_EQ((int)offsetof(GcsBlockDedupEntry, entry_kind), 46); UT_ASSERT_EQ((int)offsetof(GcsBlockDedupEntry, pcm_x_master_session), 48); UT_ASSERT_EQ((int)offsetof(GcsBlockDedupEntry, reply_header), 56); @@ -1099,6 +1099,7 @@ UT_TEST(u18_pcm_x_stage_duplicate_and_generic_overwrite_refused) UT_ASSERT_EQ((int)stage_pcm_x_ready(0, &key, &tag, &binding, &hdr, page), (int)GCS_BLOCK_PCM_X_IMAGE_DUPLICATE); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &conflicting_binding, + UINT64_C(41), (uint8)PCM_STATE_X, &conflicting_hdr, overwrite), (int)GCS_BLOCK_PCM_X_IMAGE_STALE); @@ -1186,14 +1187,14 @@ UT_TEST(u20_pcm_x_entry_survives_generic_gc_and_retires_exactly) UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &wrong, &cached), (int)GCS_BLOCK_PCM_X_IMAGE_STALE); UT_ASSERT_EQ((int)cached.entry_kind, (int)GCS_BLOCK_DEDUP_ENTRY_GENERIC); - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &wrong), + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &wrong, 2), (int)GCS_BLOCK_PCM_X_IMAGE_STALE); - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &binding), + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &binding, 2), (int)GCS_BLOCK_PCM_X_IMAGE_RELEASED); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &binding, &cached), (int)GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND); UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_release_count(), 1); - UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_failclosed_count(), 3); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_failclosed_count(), 2); } @@ -1263,7 +1264,7 @@ UT_TEST(u22_pcm_x_reserved_entry_waits_for_exact_release) cluster_gcs_block_dedup_cleanup_on_node_dead(1); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &reserved, &cached), (int)GCS_BLOCK_PCM_X_IMAGE_NOT_READY); - UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &reserved), + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &reserved, -1), (int)GCS_BLOCK_PCM_X_IMAGE_RELEASED); UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_stage_count(), 0); UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_replay_count(), 0); @@ -1305,53 +1306,53 @@ UT_TEST(u23_pcm_x_materialize_validation_is_fail_closed_and_byte_stable) bad_binding.identity.image.source_node = 1; bad_hdr = hdr; bad_hdr.sender_node = 1; - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &bad_binding, &bad_hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &bad_binding, UINT64_C(41), (uint8)PCM_STATE_X, &bad_hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); bad_hdr = hdr; bad_hdr.sender_node = 1; - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &binding, &bad_hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &binding, UINT64_C(41), (uint8)PCM_STATE_X, &bad_hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); memcpy(bad_page, page, sizeof(bad_page)); bad_page[sizeof(bad_page) - 1] ^= 0x1; - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &binding, &hdr, bad_page), - (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &binding, UINT64_C(41), (uint8)PCM_STATE_X, &hdr, bad_page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); bad_binding = binding; bad_binding.identity.image.page_lsn++; bad_hdr = hdr; bad_hdr.page_lsn++; - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &bad_binding, &bad_hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &bad_binding, UINT64_C(41), (uint8)PCM_STATE_X, &bad_hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); bad_binding = binding; bad_binding.identity.image.page_scn++; - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &bad_binding, &hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &bad_binding, UINT64_C(41), (uint8)PCM_STATE_X, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); bad_binding = binding; bad_binding.identity.image.page_checksum++; bad_hdr = hdr; bad_hdr.checksum++; - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &bad_binding, &bad_hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &bad_binding, UINT64_C(41), (uint8)PCM_STATE_X, &bad_hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); bad_binding = binding; bad_binding.master_session++; - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &bad_binding, &hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_STALE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &bad_binding, UINT64_C(41), (uint8)PCM_STATE_X, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STALE); - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &binding, &hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &binding, UINT64_C(41), (uint8)PCM_STATE_X, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_stage_count(), 1); UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_failclosed_count(), 7); } @@ -1400,6 +1401,7 @@ UT_TEST(u25_pcm_x_work_prefers_reserved_and_marks_ready_staged) GcsBlockDedupKey reserved_key; GcsBlockPcmXImageBinding ready_binding; GcsBlockPcmXImageBinding reserved_binding; + GcsBlockPcmXImageBinding wrong_floor; GcsBlockPcmXImageWork work; GcsBlockReplyHeader hdr; uint64 ready_image_id; @@ -1424,9 +1426,15 @@ UT_TEST(u25_pcm_x_work_prefers_reserved_and_marks_ready_staged) reserved_binding.identity.image.page_scn = 0; reserved_binding.identity.image.page_lsn = 0; reserved_binding.identity.image.page_checksum = 0; + reserved_binding.required_page_scn = UINT64_C(72057594037950810); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &reserved_key, &reserved_tag, &reserved_binding), (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + wrong_floor = reserved_binding; + wrong_floor.required_page_scn++; + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pcm_x_reserve(0, &reserved_key, &reserved_tag, &wrong_floor), + (int)GCS_BLOCK_PCM_X_IMAGE_STALE); memset(&work, 0, sizeof(work)); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), @@ -1434,7 +1442,7 @@ UT_TEST(u25_pcm_x_work_prefers_reserved_and_marks_ready_staged) UT_ASSERT_EQ(memcmp(&work.key, &reserved_key, sizeof(reserved_key)), 0); UT_ASSERT_EQ(memcmp(&work.binding, &reserved_binding, sizeof(reserved_binding)), 0); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &reserved_key, &reserved_tag, - &reserved_binding), + &reserved_binding, -1), (int)GCS_BLOCK_PCM_X_IMAGE_RELEASED); memset(&work, 0, sizeof(work)); @@ -1577,8 +1585,9 @@ UT_TEST(u28_pcm_x_idle_hint_avoids_empty_rescan_and_reserve_rearms) /* Materialized bytes are retained evidence, not a sendable READY image. The * ownership X->N commit must happen between materialize and the explicit - * publication call; a restarted LMS scanning the intermediate state must not - * synthesize type 50 from it. */ + * publication call. A live owner must receive a distinct commit-only work + * token so conditional BufferContent contention can retry without recopying, + * aborting the A-record, or synthesizing type 50. */ UT_TEST(u29_pcm_x_materialized_bytes_require_explicit_ready_publication) { BufferTag tag = make_tag(123); @@ -1605,13 +1614,17 @@ UT_TEST(u29_pcm_x_materialized_bytes_require_explicit_ready_publication) UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key, &tag, &reserved), (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &binding, &hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &binding, UINT64_C(41), (uint8)PCM_STATE_X, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &binding, &cached), (int)GCS_BLOCK_PCM_X_IMAGE_NOT_READY); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), - (int)GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND); + (int)GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING); + UT_ASSERT_EQ((int)work.entry_kind, (int)GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED); + UT_ASSERT_EQ(memcmp(&work.binding, &binding, sizeof(binding)), 0); + UT_ASSERT_EQ(work.reservation_token, UINT64_C(41)); + UT_ASSERT_EQ((int)work.source_pcm_state, (int)PCM_STATE_X); UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_stage_count(), 1); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_publish_ready_exact(0, &key, &tag, &binding), @@ -1651,9 +1664,9 @@ UT_TEST(u30_pcm_x_owner_restart_audit_detects_and_retains_evidence) UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key, &tag, &reserved), (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); - UT_ASSERT_EQ( - (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &binding, &hdr, page), - (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &binding, UINT64_C(41), (uint8)PCM_STATE_X, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); UT_ASSERT(cluster_gcs_block_dedup_pcm_x_restart_audit(0)); UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &binding, &cached), (int)GCS_BLOCK_PCM_X_IMAGE_NOT_READY); @@ -1752,10 +1765,356 @@ UT_TEST(u32_pcm_x_staged_marker_rolls_back_exactly) (int)GCS_BLOCK_PCM_X_IMAGE_DUPLICATE); } + +/* Shape-B: a legacy S request may already be registered or forwarded before + * the PCM-X queue publishes its pending-X claim. The queue arm must revoke + * every still-live grant/forward right under the dedup lock, cache an exact + * retry denial for loss/replay, and fence a late producer from restoring a + * GRANTED reply. DONE stops periodic replay; a new request identity remains + * a normal MISS after the X round. */ +UT_TEST(u33_pending_x_arm_terminates_inflight_legacy_s_exactly) +{ + BufferTag tag = make_tag(130); + BufferTag other_tag = make_tag(131); + GcsBlockDedupKey inflight = make_key(1, 3, 3001, 17); + GcsBlockDedupKey forwarded = make_key(2, 4, 3002, 17); + GcsBlockDedupKey unrelated = make_key(3, 5, 3003, 17); + GcsBlockDedupKey writer = make_key(1, 6, 3004, 17); + GcsBlockDedupKey fresh = make_key(1, 3, 4001, 17); + GcsBlockReplyHeader forward_hdr; + GcsBlockReplyHeader late_granted; + GcsBlockDedupEntry cached; + GcsBlockDedupEntry denied; + GcsBlockDedupKey denied_keys[2]; + char page[GCS_BLOCK_DATA_SIZE]; + int result; + int i; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + memset(&forward_hdr, 0, sizeof(forward_hdr)); + memset(&late_granted, 0, sizeof(late_granted)); + memset(page, 0x6d, sizeof(page)); + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &inflight, tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &forwarded, tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); + forward_hdr.request_id = forwarded.request_id; + forward_hdr.epoch = forwarded.cluster_epoch; + forward_hdr.sender_node = 3; + forward_hdr.requester_backend_id = forwarded.requester_backend_id; + forward_hdr.transition_id = (uint8)PCM_TRANS_N_TO_S; + forward_hdr.status = (uint8)GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER; + GcsBlockReplyHeaderSetForwardingMasterNode(&forward_hdr, cluster_node_id); + cluster_gcs_block_dedup_install_reply(0, &forwarded, GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER, + &forward_hdr, NULL); + + /* Different tag and same-tag writer identities are not legacy-S victims. */ + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &unrelated, other_tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register(0, &writer, tag, PCM_TRANS_N_TO_X, + 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); + + for (i = 0; i < 2; i++) { + memset(&denied, 0, sizeof(denied)); + result = cluster_gcs_block_dedup_pending_x_deny_next(0, &tag, &denied); + UT_ASSERT_EQ(result, GCS_BLOCK_PENDING_X_DENY_NEW); + UT_ASSERT_EQ((int)denied.entry_kind, (int)GCS_BLOCK_DEDUP_ENTRY_GENERIC); + UT_ASSERT_EQ((int)denied.transition_id, (int)PCM_TRANS_N_TO_S); + UT_ASSERT_EQ((int)denied.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ((int)denied.reply_header.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ(denied.reply_header.request_id, denied.key.request_id); + UT_ASSERT_EQ(denied.reply_header.epoch, denied.key.cluster_epoch); + UT_ASSERT_EQ(denied.reply_header.requester_backend_id, denied.key.requester_backend_id); + UT_ASSERT_EQ((int)denied.reply_header.transition_id, (int)PCM_TRANS_N_TO_S); + UT_ASSERT_EQ(denied.reply_header.sender_node, cluster_node_id); + UT_ASSERT_EQ(GcsBlockReplyHeaderGetForwardingMasterNode(&denied.reply_header), + GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + denied_keys[i] = denied.key; + } + UT_ASSERT(memcmp(&denied_keys[0], &denied_keys[1], sizeof(denied_keys[0])) != 0); + + /* Initial denial loss is recovered by periodic replay of the same exact + * identity, without minting another request or reviving FORWARD. */ + memset(&denied, 0, sizeof(denied)); + UT_ASSERT_EQ(cluster_gcs_block_dedup_pending_x_deny_next(0, &tag, &denied), + GCS_BLOCK_PENDING_X_DENY_REPLAY); + UT_ASSERT(memcmp(&denied.key, &denied_keys[0], sizeof(denied.key)) == 0 + || memcmp(&denied.key, &denied_keys[1], sizeof(denied.key)) == 0); + + /* An asynchronous old GRANTED producer has lost its right once the deny + * is cached; installing it must be a zero-mutation no-op. */ + late_granted.request_id = denied.key.request_id; + late_granted.epoch = denied.key.cluster_epoch; + late_granted.sender_node = cluster_node_id; + late_granted.requester_backend_id = denied.key.requester_backend_id; + late_granted.transition_id = (uint8)PCM_TRANS_N_TO_S; + late_granted.status = (uint8)GCS_BLOCK_REPLY_GRANTED; + cluster_gcs_block_dedup_install_reply(0, &denied.key, GCS_BLOCK_REPLY_GRANTED, &late_granted, + page); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &denied.key, tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_CACHED_REPLY); + UT_ASSERT_EQ((int)cached.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ((int)cached.reply_header.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + + /* Duplicate DONE is idempotent and removes both old identities from the + * denial replay set; unrelated entries remain in their original states. */ + for (i = 0; i < 2; i++) { + UT_ASSERT(cluster_gcs_block_dedup_mark_done(0, &denied_keys[i], &tag, PCM_TRANS_N_TO_S)); + UT_ASSERT(cluster_gcs_block_dedup_mark_done(0, &denied_keys[i], &tag, PCM_TRANS_N_TO_S)); + } + UT_ASSERT_EQ(cluster_gcs_block_dedup_pending_x_deny_next(0, &tag, &denied), + GCS_BLOCK_PENDING_X_DENY_NOT_FOUND); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &unrelated, other_tag, PCM_TRANS_N_TO_S, 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_IN_FLIGHT_DUPLICATE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register(0, &writer, tag, PCM_TRANS_N_TO_X, + 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_IN_FLIGHT_DUPLICATE); + + /* Once the queue round is over, a reader with a fresh identity is admitted + * normally; the two denied identities cannot poison the new request. */ + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register(0, &fresh, tag, PCM_TRANS_N_TO_S, + 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); +} + +/* A reader arriving after the queue claim is registered only to make its + * denial reliable: exact arbitration replaces even a pre-existing cached + * grant before the handler can take a normal dedup shortcut. The original + * direct-land property remains attached to the cached denial for replay. */ +UT_TEST(u34_pending_x_new_reader_exact_deny_precedes_cached_shortcut) +{ + BufferTag tag = make_tag(132); + GcsBlockDedupKey key = make_key(2, 7, 5001, 19); + GcsBlockDedupKey absent = make_key(2, 7, 5002, 19); + GcsBlockReplyHeader granted; + GcsBlockDedupEntry cached; + GcsBlockDedupEntry denied; + char page[GCS_BLOCK_DATA_SIZE]; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + memset(&granted, 0, sizeof(granted)); + memset(page, 0x71, sizeof(page)); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register(0, &key, tag, PCM_TRANS_N_TO_S, + 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); + UT_ASSERT(cluster_gcs_block_dedup_set_request_flags_exact( + 0, &key, &tag, PCM_TRANS_N_TO_S, GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND)); + + granted.request_id = key.request_id; + granted.epoch = key.cluster_epoch; + granted.sender_node = cluster_node_id; + granted.requester_backend_id = key.requester_backend_id; + granted.transition_id = (uint8)PCM_TRANS_N_TO_S; + granted.status = (uint8)GCS_BLOCK_REPLY_GRANTED; + GcsBlockReplyHeaderSetForwardingMasterNode(&granted, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + cluster_gcs_block_dedup_install_reply(0, &key, GCS_BLOCK_REPLY_GRANTED, &granted, page); + + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pending_x_deny_exact(0, &key, &tag, PCM_TRANS_N_TO_S, &denied), + (int)GCS_BLOCK_PENDING_X_DENY_NEW); + UT_ASSERT_EQ((int)denied.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ((int)denied.request_flags, + (int)(GCS_BLOCK_DEDUP_REQUEST_F_PINNED | GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND)); + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pending_x_deny_exact(0, &key, &tag, PCM_TRANS_N_TO_S, &denied), + (int)GCS_BLOCK_PENDING_X_DENY_REPLAY); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register(0, &key, tag, PCM_TRANS_N_TO_S, + 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_CACHED_REPLY); + UT_ASSERT_EQ((int)cached.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + + /* Missing and mismatched identities are zero-mutation invalid results. */ + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pending_x_deny_exact(0, &absent, &tag, + PCM_TRANS_N_TO_S, &denied), + (int)GCS_BLOCK_PENDING_X_DENY_INVALID); + UT_ASSERT(!cluster_gcs_block_dedup_set_request_flags_exact( + 0, &key, &tag, PCM_TRANS_N_TO_X, GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND)); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register(0, &key, tag, PCM_TRANS_N_TO_S, + 1000, true, &cached), + (int)GCS_BLOCK_DEDUP_CACHED_REPLY); + UT_ASSERT_EQ((int)cached.request_flags, + (int)(GCS_BLOCK_DEDUP_REQUEST_F_PINNED | GCS_BLOCK_DEDUP_REQUEST_F_DIRECT_LAND)); +} + + +/* A contended tag A may remain at the commit-only boundary, but its exact + * retry must not prevent the same DATA worker from advancing independent tag + * B. This exercises the production HTAB scan/cursor, not a scheduler model. */ +UT_TEST(u35_pcm_x_commit_pending_rotates_to_independent_reserved_tag) +{ + BufferTag tag_a = make_tag(130); + BufferTag tag_b = make_tag(131); + GcsBlockDedupKey key_a; + GcsBlockDedupKey key_b; + GcsBlockPcmXImageBinding binding_a; + GcsBlockPcmXImageBinding reserved_a; + GcsBlockPcmXImageBinding reserved_b; + GcsBlockPcmXImageWork work; + GcsBlockReplyHeader hdr_a; + char page_a[GCS_BLOCK_DATA_SIZE]; + uint64 image_a; + uint64 image_b; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 30, &image_a)); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 31, &image_b)); + key_a = make_key(1, 3, image_a, 13); + key_b = make_key(1, 4, image_b, 13); + binding_a = make_pcm_x_binding(tag_a, 1, 5, gcs_reqid_requester(1, 2, 100), 13, image_a, 120); + hdr_a = make_pcm_x_reply_header(&key_a, &binding_a); + memset(page_a, 0x68, sizeof(page_a)); + prepare_pcm_x_page(page_a, &binding_a, &hdr_a); + reserved_a = binding_a; + reserved_a.identity.image.page_scn = 0; + reserved_a.identity.image.page_lsn = 0; + reserved_a.identity.image.page_checksum = 0; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key_a, &tag_a, &reserved_a), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key_a, &tag_a, &binding_a, + UINT64_C(51), (uint8)PCM_STATE_X, + &hdr_a, page_a), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + + reserved_b = make_pcm_x_binding(tag_b, 1, 6, gcs_reqid_requester(1, 3, 101), 13, image_b, 121); + reserved_b.identity.image.page_scn = 0; + reserved_b.identity.image.page_lsn = 0; + reserved_b.identity.image.page_checksum = 0; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key_b, &tag_b, &reserved_b), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING); + UT_ASSERT_EQ(memcmp(&work.key, &key_a, sizeof(key_a)), 0); + UT_ASSERT_EQ(work.reservation_token, UINT64_C(51)); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ(memcmp(&work.key, &key_b, sizeof(key_b)), 0); +} + + +/* A local terminal DRAIN is not an ACK boundary by itself. Exact byte and + * descriptor cleanup publishes a replayable tombstone; duplicate DRAIN stays + * provably complete until the matching RETIRE watermark removes that proof. */ +UT_TEST(u36_pcm_x_drain_cleanup_is_replayable_until_exact_retire) +{ + BufferTag tag = make_tag(132); + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageBinding reserved; + GcsBlockReplyHeader hdr; + GcsBlockDedupEntry cached; + char page[GCS_BLOCK_DATA_SIZE]; + uint64 image_id; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 32, &image_id)); + /* The initial live cluster episode is epoch zero; RETIRE must preserve the + * same exact-match semantics there as after the first reconfiguration. */ + key = make_key(1, 3, image_id, 0); + binding = make_pcm_x_binding(tag, 1, 5, gcs_reqid_requester(1, 2, 102), 0, image_id, 122); + hdr = make_pcm_x_reply_header(&key, &binding); + memset(page, 0x79, sizeof(page)); + prepare_pcm_x_page(page, &binding, &hdr); + UT_ASSERT_EQ((int)stage_pcm_x_ready(0, &key, &tag, &binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_drain_status_exact(0, &key, &tag, &binding), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_READY); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &binding, 2), + (int)GCS_BLOCK_PCM_X_IMAGE_RELEASED); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_drain_status_exact(0, &key, &tag, &binding), + (int)GCS_BLOCK_PCM_X_IMAGE_DUPLICATE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &binding, 2), + (int)GCS_BLOCK_PCM_X_IMAGE_DUPLICATE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &binding, &cached), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND); + reserved = binding; + reserved.identity.image.page_scn = 0; + reserved.identity.image.page_lsn = 0; + reserved.identity.image.page_checksum = 0; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key, &tag, &reserved), + (int)GCS_BLOCK_PCM_X_IMAGE_DUPLICATE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_rearm_exact(0, &key, &tag, &reserved), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_READY); + + cluster_gcs_block_dedup_sweep_expired((TimestampTz)INT64_MAX); + cluster_gcs_block_dedup_cleanup_on_backend_exit(1, 3); + cluster_gcs_block_dedup_cleanup_on_node_dead(1); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); + UT_ASSERT(cluster_gcs_block_dedup_pcm_x_retire_up_to(12, 2, 122, 29)); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); + UT_ASSERT(cluster_gcs_block_dedup_pcm_x_retire_up_to(0, 1, 122, 29)); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); + UT_ASSERT(cluster_gcs_block_dedup_pcm_x_retire_up_to(0, 2, 121, 29)); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); + UT_ASSERT(cluster_gcs_block_dedup_pcm_x_retire_up_to(0, 2, 122, 28)); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); + UT_ASSERT(cluster_gcs_block_dedup_pcm_x_retire_up_to(0, 2, 122, 29)); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 0); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_release_count(), 1); +} + + +/* A FlushBuffer ERROR occurs after materialization. Its catch handler must + * validate, but never delete or rewrite, the immutable A-record and revoke + * token needed by recovery. */ +UT_TEST(u37_pcm_x_finish_error_preserves_exact_materialized_evidence) +{ + BufferTag tag = make_tag(133); + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageBinding reserved; + GcsBlockPcmXImageWork work; + GcsBlockReplyHeader hdr; + char page[GCS_BLOCK_DATA_SIZE]; + uint64 image_id; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 33, &image_id)); + key = make_key(1, 3, image_id, 13); + binding = make_pcm_x_binding(tag, 1, 5, gcs_reqid_requester(1, 2, 103), 13, image_id, 123); + hdr = make_pcm_x_reply_header(&key, &binding); + memset(page, 0x6a, sizeof(page)); + prepare_pcm_x_page(page, &binding, &hdr); + reserved = binding; + reserved.identity.image.page_scn = 0; + reserved.identity.image.page_lsn = 0; + reserved.identity.image.page_checksum = 0; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key, &tag, &reserved), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize( + 0, &key, &tag, &binding, UINT64_C(61), (uint8)PCM_STATE_X, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_preserve_finish_error_exact( + 0, &key, &tag, &binding, UINT64_C(62), (uint8)PCM_STATE_X), + (int)GCS_BLOCK_PCM_X_IMAGE_STALE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_preserve_finish_error_exact( + 0, &key, &tag, &binding, UINT64_C(61), (uint8)PCM_STATE_X), + (int)GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_COMMIT_PENDING); + UT_ASSERT_EQ(memcmp(&work.binding, &binding, sizeof(binding)), 0); + UT_ASSERT_EQ(work.reservation_token, UINT64_C(61)); + UT_ASSERT_EQ((int)work.source_pcm_state, (int)PCM_STATE_X); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &binding, -1), + (int)GCS_BLOCK_PCM_X_IMAGE_RELEASED); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 0); +} + int main(void) { - UT_PLAN(32); + UT_PLAN(37); UT_RUN(u1_per_worker_isolation); UT_RUN(u2_dedup_lifecycle_per_shard); UT_RUN(u3_counters_sum_across_shards); @@ -1788,6 +2147,11 @@ main(void) UT_RUN(u30_pcm_x_owner_restart_audit_detects_and_retains_evidence); UT_RUN(u31_pcm_x_work_classes_alternate_when_both_remain_runnable); UT_RUN(u32_pcm_x_staged_marker_rolls_back_exactly); + UT_RUN(u33_pending_x_arm_terminates_inflight_legacy_s_exactly); + UT_RUN(u34_pending_x_new_reader_exact_deny_precedes_cached_shortcut); + UT_RUN(u35_pcm_x_commit_pending_rotates_to_independent_reserved_tag); + UT_RUN(u36_pcm_x_drain_cleanup_is_replayable_until_exact_retire); + UT_RUN(u37_pcm_x_finish_error_preserves_exact_materialized_evidence); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_gcs_block_shard.c b/src/test/cluster_unit/test_cluster_gcs_block_shard.c index 4277ac8248..ad9c3e3a06 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_shard.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_shard.c @@ -405,6 +405,7 @@ UT_TEST(test_pcm_x_route_truth_table) { PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, sizeof(PcmXBlockerSetHeaderPayload) }, { PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK, sizeof(PcmXPhasePayload) }, { PGRAC_IC_MSG_PCM_X_REVOKE, sizeof(PcmXRevokePayload) }, + { PGRAC_IC_MSG_PCM_X_REVOKE, sizeof(PcmXRevokePayloadV2) }, { PGRAC_IC_MSG_PCM_X_IMAGE_READY, sizeof(PcmXGrantPayload) }, { PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, sizeof(PcmXGrantPayload) }, { PGRAC_IC_MSG_PCM_X_INSTALL_READY, sizeof(PcmXInstallReadyPayload) }, @@ -444,10 +445,39 @@ UT_TEST(test_pcm_x_route_truth_table) -1); } +/* A status-3 PI durable note is an INVALIDATE_ACK frame and must use the + * exact tag shard. Exercise every live worker, including worker 1 (the + * t/400 failure arm), against the real payload router. */ +UT_TEST(test_pi_durable_note_routes_to_exact_tag_worker) +{ + bool seen[CLUSTER_LMS_MAX_WORKERS] = { false }; + int seen_count = 0; + int i; + + for (i = 0; i < 4096 && seen_count < CLUSTER_LMS_MAX_WORKERS; i++) { + BufferTag tag = make_tag(1663, 5, 16384 + (i % 4), MAIN_FORKNUM, (BlockNumber)i); + GcsBlockInvalidateAckPayload note = { 0 }; + int expected = cluster_lms_shard_for_tag(&tag, CLUSTER_LMS_MAX_WORKERS); + int routed; + + note.tag = tag; + note.ack_status = GCS_BLOCK_INVALIDATE_ACK_STATUS_PI_DURABLE_NOTE; + routed = cluster_gcs_block_payload_shard(PGRAC_IC_MSG_GCS_BLOCK_INVALIDATE_ACK, ¬e, + sizeof(note), CLUSTER_LMS_MAX_WORKERS); + UT_ASSERT_EQ(routed, expected); + if (!seen[routed]) { + seen[routed] = true; + seen_count++; + } + } + UT_ASSERT(seen[1]); + UT_ASSERT_EQ(seen_count, CLUSTER_LMS_MAX_WORKERS); +} + int main(void) { - UT_PLAN(8); + UT_PLAN(9); UT_RUN(test_route_matches_shard_for_tag); UT_RUN(test_route_ack_request_interleave_affinity); UT_RUN(test_route_registry_partition); @@ -456,6 +486,7 @@ main(void) UT_RUN(test_route_n1_degenerate_zero); UT_RUN(test_route_ignores_non_tag_fields); UT_RUN(test_pcm_x_route_truth_table); + UT_RUN(test_pi_durable_note_routes_to_exact_tag_worker); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_ic.c b/src/test/cluster_unit/test_cluster_ic.c index 71877a8c7f..5ae45dba80 100644 --- a/src/test/cluster_unit/test_cluster_ic.c +++ b/src/test/cluster_unit/test_cluster_ic.c @@ -597,9 +597,11 @@ UT_TEST(test_hello_wire_reference_bytes) * (0x10) + round-3 P0-1 xid wrap barrier (0x20) + round-4 P0-1 * authority flock (0x40) + ownership-gen ruling② invalidate BUSY * (0x80) + TT-lane undo-horizon idle sentinel (0x100) + PCM-X - * conversion (0x200) (smart-fusion is off in this fixture) */ + * conversion (0x200) + A' rebase V2 INSTALL_READY (0x400) + PCM-X + * source-floor type49 V2 (0x800) + * (smart-fusion is off in this fixture) */ UT_ASSERT_EQ(wire[36], 0xFE); - UT_ASSERT_EQ(wire[37], 0x03); + UT_ASSERT_EQ(wire[37], 0x0F); UT_ASSERT_EQ(wire[38], 0x00); UT_ASSERT_EQ(wire[39], 0x00); /* remaining _pad must be all zero: a CONTROL-plane HELLO with @@ -698,11 +700,12 @@ UT_TEST(test_hello_smart_fusion_capability_gate) | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 - | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1); + | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1 + | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1); /* Keep the aggregate word byte-exact as well as symbolically composed: * parallel protocol lanes have collided while preserving the same symbolic * expectation, so the literal catches accidental bit reuse. */ - UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), (uint32)0x000003FEU); + UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), (uint32)0x00000FFEU); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_2; @@ -716,7 +719,8 @@ UT_TEST(test_hello_smart_fusion_capability_gate) | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 - | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1); + | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1 + | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_3; @@ -730,7 +734,8 @@ UT_TEST(test_hello_smart_fusion_capability_gate) | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1 | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 - | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1); + | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 + | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1 | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1); cluster_smart_fusion = false; cluster_interconnect_tier = CLUSTER_IC_TIER_STUB; @@ -790,6 +795,7 @@ UT_TEST(test_hello_gcs_done_and_wrap_barrier_gates) UT_ASSERT_EQ((int)PGRAC_IC_MSG_XID_NATIVE_DISABLE, 39); UT_ASSERT_EQ((int)PGRAC_IC_MSG_XID_NATIVE_DISABLE_ACK, 40); UT_ASSERT_EQ(PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, (uint32)0x00000200U); + UT_ASSERT_EQ(PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, (uint32)0x00000400U); UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_ENQUEUE, 41); UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_ADMIT_ACK, 42); UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM, 43); diff --git a/src/test/cluster_unit/test_cluster_lms_outbound.c b/src/test/cluster_unit/test_cluster_lms_outbound.c index a94623d562..cec1575537 100644 --- a/src/test/cluster_unit/test_cluster_lms_outbound.c +++ b/src/test/cluster_unit/test_cluster_lms_outbound.c @@ -54,6 +54,7 @@ #include "cluster/cluster_ic_router.h" /* cluster_ic_send_envelope prototype */ #include "cluster/cluster_lms.h" #include "cluster/cluster_shmem.h" +#include "cluster/cluster_sf_dep.h" #include "cluster/cluster_write_fence.h" #include "miscadmin.h" #include "storage/lwlock.h" @@ -75,6 +76,42 @@ int cluster_node_id = 0; static PcmXRuntimeState ut_pcm_x_runtime_state = PCM_X_RUNTIME_ACTIVE; static bool ut_write_fence_enforcing = false; static bool ut_write_fence_allowed = true; +static uint32 ut_peer_capabilities[CLUSTER_MAX_NODES]; +static uint32 ut_peer_cap_generation[CLUSTER_MAX_NODES]; +static int ut_pcm_x_boundary_note_count = 0; +static uint8 ut_pcm_x_boundary_msg_types[8]; + +void +cluster_lms_note_pcm_x_image_ready_boundary(uint8 msg_type, const char *boundary, int result, + int runtime_state, bool fence_enforcing, + bool fence_allowed, uint32 dest_node_id, + uint64 request_id, uint64 ticket_id, + uint64 grant_generation, uint64 image_id) +{ + if (ut_pcm_x_boundary_note_count < (int)lengthof(ut_pcm_x_boundary_msg_types)) + ut_pcm_x_boundary_msg_types[ut_pcm_x_boundary_note_count] = msg_type; + ut_pcm_x_boundary_note_count++; + (void)boundary; + (void)result; + (void)runtime_state; + (void)fence_enforcing; + (void)fence_allowed; + (void)dest_node_id; + (void)request_id; + (void)ticket_id; + (void)grant_generation; + (void)image_id; +} + +bool +cluster_sf_peer_capability_generation_matches(int32 peer_id, uint32 required_capabilities, + uint32 expected_generation) +{ + if (peer_id < 0 || peer_id >= CLUSTER_MAX_NODES || required_capabilities == 0) + return false; + return (ut_peer_capabilities[peer_id] & required_capabilities) == required_capabilities + && ut_peer_cap_generation[peer_id] == expected_generation; +} PcmXRuntimeSnapshot cluster_pcm_x_runtime_snapshot(void) @@ -178,6 +215,7 @@ cluster_lms_wakeup(int worker_id) /* Drain honesty counters (shmem-backed in production): count-only stubs. */ static int ut_not_admitted_count = 0; static int ut_requeue_drop_count = 0; +static int ut_cap_guard_drop_count = 0; void cluster_lms_obs_note_outbound_not_admitted(int worker_id) @@ -193,6 +231,20 @@ cluster_lms_obs_note_outbound_requeue_drop(int worker_id) ut_requeue_drop_count++; } +void +cluster_lms_obs_note_outbound_cap_guard_drop(int worker_id) +{ + (void)worker_id; + ut_cap_guard_drop_count++; +} + +void +cluster_gcs_block_note_send_outcome(GcsBlockSendFamily family, ClusterICSendResult rc) +{ + (void)family; + (void)rc; +} + static int ut_prepare_hook_count = 0; void @@ -215,6 +267,8 @@ typedef struct UtSentRec { uint8 msg_type; int32 dest; uint8 marker; /* first payload byte identifies the frame */ + uint32 payload_len; + GcsBlockReplyHeader reply_header; } UtSentRec; static UtSentRec ut_sent_log[64]; @@ -222,6 +276,8 @@ static int ut_sent_n = 0; static ClusterICSendResult ut_peer_rc[CLUSTER_MAX_NODES]; static int ut_local_dispatch_count = 0; static uint8 ut_local_dispatch_marker = 0; +static int ut_direct_zero_reply_count = 0; +static GcsBlockReplyHeader ut_direct_zero_reply_header; bool cluster_ic_envelope_build(ClusterICEnvelope *out_env, uint8 msg_type, uint32 source_node_id, @@ -256,12 +312,30 @@ cluster_ic_send_envelope(uint8 msg_type, int32 dest_node_id, const void *payload ut_sent_log[ut_sent_n].msg_type = msg_type; ut_sent_log[ut_sent_n].dest = dest_node_id; ut_sent_log[ut_sent_n].marker = payload_len > 0 ? *(const uint8 *)payload : 0; + ut_sent_log[ut_sent_n].payload_len = payload_len; + if (msg_type == PGRAC_IC_MSG_GCS_BLOCK_REPLY && payload_len >= sizeof(GcsBlockReplyHeader)) + memcpy(&ut_sent_log[ut_sent_n].reply_header, payload, sizeof(GcsBlockReplyHeader)); } ut_sent_n++; UT_ASSERT(dest_node_id >= 0 && dest_node_id < CLUSTER_MAX_NODES); return ut_peer_rc[dest_node_id]; } +uint32 +cluster_gcs_block_compute_checksum(const char *block_data) +{ + (void)block_data; + return UINT32_C(0xA55A7E11); +} + +ClusterICSendResult +cluster_gcs_block_send_direct_zero_reply(int32 dest_node, const GcsBlockReplyHeader *header) +{ + ut_direct_zero_reply_count++; + ut_direct_zero_reply_header = *header; + return ut_peer_rc[dest_node]; +} + static int ut_count_marker(uint8 marker) { @@ -280,9 +354,16 @@ ut_reset_log(void) ut_sent_n = 0; ut_local_dispatch_count = 0; ut_local_dispatch_marker = 0; + ut_direct_zero_reply_count = 0; + memset(&ut_direct_zero_reply_header, 0, sizeof(ut_direct_zero_reply_header)); ut_pcm_x_runtime_state = PCM_X_RUNTIME_ACTIVE; ut_write_fence_enforcing = false; ut_write_fence_allowed = true; + ut_cap_guard_drop_count = 0; + ut_pcm_x_boundary_note_count = 0; + memset(ut_pcm_x_boundary_msg_types, 0, sizeof(ut_pcm_x_boundary_msg_types)); + memset(ut_peer_capabilities, 0, sizeof(ut_peer_capabilities)); + memset(ut_peer_cap_generation, 0, sizeof(ut_peer_cap_generation)); memset(ut_sent_log, 0, sizeof(ut_sent_log)); } @@ -498,10 +579,164 @@ UT_TEST(test_pcm_x_grant_frame_waits_behind_write_fence) UT_ASSERT_EQ(cluster_lms_outbound_depth(7), 0); } + +/* Both sides of the first reliable transfer hop share PcmXGrantPayload. The + * injected transport trace must identify type 50 and type 51 independently, + * otherwise a successful master consume is indistinguishable from a lost + * PREPARE_GRANT admission. */ +UT_TEST(test_pcm_x_image_ready_and_prepare_transport_boundaries_are_observable) +{ + PcmXGrantPayload payload; + + ut_reset_log(); + memset(&payload, 0, sizeof(payload)); + UT_ASSERT(cluster_lms_outbound_enqueue(0, PGRAC_IC_MSG_PCM_X_IMAGE_READY, UT_PEER_X, &payload, + sizeof(payload))); + UT_ASSERT(cluster_lms_outbound_enqueue(0, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, UT_PEER_Y, &payload, + sizeof(payload))); + UT_ASSERT_EQ(cluster_lms_outbound_drain_send(0), 2); + UT_ASSERT_EQ(ut_pcm_x_boundary_note_count, 2); + UT_ASSERT_EQ((int)ut_pcm_x_boundary_msg_types[0], (int)PGRAC_IC_MSG_PCM_X_IMAGE_READY); + UT_ASSERT_EQ((int)ut_pcm_x_boundary_msg_types[1], (int)PGRAC_IC_MSG_PCM_X_PREPARE_GRANT); +} + +/* + * Shape-B denial replay is driven by LMON, which owns only plane 0. The + * reply is an ABI-sized header + zero block, so LMON stages its compact + * header on the tag's DATA ring and the owning LMS worker expands and sends + * it. A direct LMON send is a production FATAL under the plane guard. + */ +UT_TEST(test_zero_block_reply_is_expanded_by_data_owner) +{ + GcsBlockReplyHeader hdr; + + ut_reset_log(); + memset(&hdr, 0, sizeof(hdr)); + hdr.request_id = UINT64_C(0x1122334455667788); + hdr.epoch = UINT64_C(41); + hdr.sender_node = 1; + hdr.requester_backend_id = 17; + hdr.transition_id = PCM_TRANS_N_TO_X; + hdr.status = (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X; + GcsBlockReplyHeaderSetForwardingMasterNode(&hdr, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + + UT_ASSERT(cluster_lms_outbound_enqueue_zero_block_reply(2, UT_PEER_X, &hdr, false)); + UT_ASSERT_EQ(cluster_lms_outbound_depth(2), 1); + ut_peer_rc[UT_PEER_X] = CLUSTER_IC_SEND_NOT_ADMITTED; + UT_ASSERT_EQ(cluster_lms_outbound_drain_send(2), 0); + UT_ASSERT_EQ(cluster_lms_outbound_depth(2), 1); + UT_ASSERT_EQ(ut_sent_n, 1); + UT_ASSERT_EQ((int)ut_sent_log[0].payload_len, (int)GCS_BLOCK_REPLY_PAYLOAD_TOTAL_SIZE); + + ut_peer_rc[UT_PEER_X] = CLUSTER_IC_SEND_DONE; + UT_ASSERT_EQ(cluster_lms_outbound_drain_send(2), 1); + UT_ASSERT_EQ(cluster_lms_outbound_depth(2), 0); + UT_ASSERT_EQ(ut_sent_n, 2); + UT_ASSERT_EQ((int)ut_sent_log[1].msg_type, (int)PGRAC_IC_MSG_GCS_BLOCK_REPLY); + UT_ASSERT_EQ((int)ut_sent_log[1].payload_len, (int)GCS_BLOCK_REPLY_PAYLOAD_TOTAL_SIZE); + UT_ASSERT_EQ(ut_sent_log[1].reply_header.request_id, hdr.request_id); + UT_ASSERT_EQ((int)ut_sent_log[1].reply_header.status, (int)GCS_BLOCK_REPLY_DENIED_PENDING_X); + UT_ASSERT_EQ(ut_sent_log[1].reply_header.checksum, UINT32_C(0xA55A7E11)); +} + +UT_TEST(test_direct_zero_block_reply_uses_data_owner_direct_lane) +{ + GcsBlockReplyHeader hdr; + + ut_reset_log(); + memset(&hdr, 0, sizeof(hdr)); + hdr.request_id = UINT64_C(0x8877665544332211); + hdr.status = (uint8)GCS_BLOCK_REPLY_DENIED_PENDING_X; + + UT_ASSERT(cluster_lms_outbound_enqueue_zero_block_reply(3, UT_PEER_Y, &hdr, true)); + ut_peer_rc[UT_PEER_Y] = CLUSTER_IC_SEND_DONE; + UT_ASSERT_EQ(cluster_lms_outbound_drain_send(3), 1); + UT_ASSERT_EQ(ut_direct_zero_reply_count, 1); + UT_ASSERT_EQ(ut_sent_n, 0); + UT_ASSERT_EQ(ut_direct_zero_reply_header.request_id, hdr.request_id); + UT_ASSERT_EQ(ut_direct_zero_reply_header.checksum, UINT32_C(0xA55A7E11)); +} + +/* A producer must receive false when the selected worker ring is full. The + * PI durable-note drain couples this real return contract with its structural + * false->break-before-seq-advance unit, so a full shard retains the source + * note for the next tick instead of losing it. */ +UT_TEST(test_full_worker_ring_refuses_without_overwrite) +{ + int accepted = 0; + int sent = 0; + + ut_reset_log(); + while (accepted < 1024 && ut_enqueue_marker(1, UT_PEER_X, 0xE2)) + accepted++; + UT_ASSERT(accepted > 0); + UT_ASSERT(accepted < 1024); + UT_ASSERT_EQ((int)cluster_lms_outbound_depth(1), accepted); + UT_ASSERT(!ut_enqueue_marker(1, UT_PEER_X, 0xE3)); + UT_ASSERT_EQ((int)cluster_lms_outbound_depth(1), accepted); + + ut_peer_rc[UT_PEER_X] = CLUSTER_IC_SEND_DONE; + while (cluster_lms_outbound_depth(1) > 0) + sent += cluster_lms_outbound_drain_send(1); + UT_ASSERT_EQ(sent, accepted); + UT_ASSERT_EQ(cluster_lms_outbound_depth(1), 0); +} + +/* A V2 wire frame is legal only on the exact HELLO-authenticated connection + * generation sampled by its producer. A reconnect or capability downgrade + * consumes the stale ring copy without transport admission; the reliable + * protocol leg remains armed and the periodic master drive reconstructs it. */ +UT_TEST(test_cap_bound_frame_drops_on_connection_generation_drift) +{ + const uint32 cap = PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1; + uint8 marker = 0x91; + + ut_reset_log(); + ut_peer_capabilities[UT_PEER_X] = cap; + ut_peer_cap_generation[UT_PEER_X] = 18; + UT_ASSERT(cluster_lms_outbound_enqueue_cap_bound(0, PGRAC_IC_MSG_PCM_X_REVOKE, UT_PEER_X, + &marker, sizeof(marker), cap, 17)); + UT_ASSERT_EQ(cluster_lms_outbound_drain_send(0), 0); + UT_ASSERT_EQ(cluster_lms_outbound_depth(0), 0); + UT_ASSERT_EQ(ut_count_marker(marker), 0); + UT_ASSERT_EQ(ut_cap_guard_drop_count, 1); +} + +UT_TEST(test_cap_bound_frame_drops_on_capability_downgrade) +{ + const uint32 cap = PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1; + uint8 marker = 0x92; + + ut_reset_log(); + ut_peer_cap_generation[UT_PEER_X] = 21; + UT_ASSERT(cluster_lms_outbound_enqueue_cap_bound(0, PGRAC_IC_MSG_PCM_X_REVOKE, UT_PEER_X, + &marker, sizeof(marker), cap, 21)); + UT_ASSERT_EQ(cluster_lms_outbound_drain_send(0), 0); + UT_ASSERT_EQ(cluster_lms_outbound_depth(0), 0); + UT_ASSERT_EQ(ut_count_marker(marker), 0); + UT_ASSERT_EQ(ut_cap_guard_drop_count, 1); +} + +UT_TEST(test_cap_bound_frame_sends_on_exact_connection_capability) +{ + const uint32 cap = PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1; + uint8 marker = 0x93; + + ut_reset_log(); + ut_peer_capabilities[UT_PEER_X] = cap; + ut_peer_cap_generation[UT_PEER_X] = 34; + UT_ASSERT(cluster_lms_outbound_enqueue_cap_bound(0, PGRAC_IC_MSG_PCM_X_REVOKE, UT_PEER_X, + &marker, sizeof(marker), cap, 34)); + UT_ASSERT_EQ(cluster_lms_outbound_drain_send(0), 1); + UT_ASSERT_EQ(cluster_lms_outbound_depth(0), 0); + UT_ASSERT_EQ(ut_count_marker(marker), 1); + UT_ASSERT_EQ(ut_cap_guard_drop_count, 0); +} + int main(void) { - UT_PLAN(8); + UT_PLAN(15); UT_RUN(test_ring_shmem_init); UT_RUN(test_admitted_frame_is_never_resubmitted); @@ -511,6 +746,13 @@ main(void) UT_RUN(test_self_frame_dispatches_on_owning_worker); UT_RUN(test_pcm_x_grant_frame_waits_for_active_runtime); UT_RUN(test_pcm_x_grant_frame_waits_behind_write_fence); + UT_RUN(test_pcm_x_image_ready_and_prepare_transport_boundaries_are_observable); + UT_RUN(test_zero_block_reply_is_expanded_by_data_owner); + UT_RUN(test_direct_zero_block_reply_uses_data_owner_direct_lane); + UT_RUN(test_full_worker_ring_refuses_without_overwrite); + UT_RUN(test_cap_bound_frame_drops_on_connection_generation_drift); + UT_RUN(test_cap_bound_frame_drops_on_capability_downgrade); + UT_RUN(test_cap_bound_frame_sends_on_exact_connection_capability); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/cluster_unit/test_cluster_pcm_direct_init.c b/src/test/cluster_unit/test_cluster_pcm_direct_init.c index 2eeb07bec7..b7403f33fd 100644 --- a/src/test/cluster_unit/test_cluster_pcm_direct_init.c +++ b/src/test/cluster_unit/test_cluster_pcm_direct_init.c @@ -23,6 +23,12 @@ UT_DEFINE_GLOBALS(); #ifndef FSM_SOURCE_PATH #error "FSM_SOURCE_PATH must identify production freespace.c" #endif +#ifndef HEAPAM_SOURCE_PATH +#error "HEAPAM_SOURCE_PATH must identify production heapam.c" +#endif +#ifndef HIO_SOURCE_PATH +#error "HIO_SOURCE_PATH must identify production hio.c" +#endif void ExceptionalCondition(const char *conditionName pg_attribute_unused(), @@ -369,9 +375,9 @@ UT_TEST(test_valid_n_s_x_without_proof_enters_queue_before_legacy_wire) { char *source = read_source(BUFMGR_SOURCE_PATH); static const char *const order[] - = { "pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode)", - "cluster_pcm_own_begin_grant_reservation(buf, &pcm_pending_base", - "cluster_pcm_lock_acquire_buffer(buf, pcm_mode)" }; + = { "pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode,", + "cluster_bufmgr_pcm_begin_grant_reservation_wait(", + "cluster_pcm_lock_acquire_buffer(buf, pcm_mode," }; UT_ASSERT(source != NULL); if (source != NULL) { @@ -409,10 +415,224 @@ UT_TEST(test_wire_throw_exact_aborts_reservation_before_rethrow) } } +/* t/400 L3 item 3 — a nested-guard BARRIER_CLOSED at the pre-crit VM lock + * must unwind to the caller that owns the outer heap content lock instead of + * escaping as a client ERROR. bufmgr exposes the refusal (never releasing + * the foreign lock itself); heapam releases its own lock(s), warms the map + * page's node X while holding no content lock, and re-enters a proven + * requalify/reacquire point. */ +UT_TEST(test_precrit_vm_barrier_refusal_unwinds_to_caller) +{ + char *bufmgr = read_source(BUFMGR_SOURCE_PATH); + char *heapam = read_source(HEAPAM_SOURCE_PATH); + + UT_ASSERT(bufmgr != NULL); + UT_ASSERT(heapam != NULL); + if (bufmgr != NULL) { + /* The refusal arm sits between the queue acquire and the ERROR + * report, and only the barrier-aware entry can consume it. */ + static const char *const refusal_order[] + = { "cluster_gcs_pcm_x_acquire_writer(buf, &entry->claim", + "PCM_X_QUEUE_BARRIER_CLOSED && barrier_refused != NULL", + "cluster_bufmgr_pcm_x_writer_report_failure(result, buf, \"queue acquire\")" }; + + UT_ASSERT(strstr(bufmgr, "ClusterLockBufferExclusiveBarrierAware(Buffer buffer)") != NULL); + assert_ordered(bufmgr, refusal_order, lengthof(refusal_order)); + free(bufmgr); + } + if (heapam != NULL) { + static const char *const pretoast_order[] = { "PGRAC: vm barrier unwind (update pre-toast)", + "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", + "cluster_heap_vm_barrier_warm", + "ReleaseBuffer(vmbuffer)", + "vmbuffer = InvalidBuffer", + "cluster_heap_lock_with_vm_repin", + "goto l2;" }; + static const char *const requalify_order[] + = { "PGRAC: vm barrier unwind (update requalify)", + "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", + "cluster_heap_vm_barrier_warm", + "ReleaseBuffer(vmbuffer)", + "vmbuffer = InvalidBuffer", + "cluster_heap_lock_with_vm_repin", + "goto l2;" }; + static const char *const reacquire_order[] + = { "PGRAC: vm barrier unwind (update reacquire)", + "LockBuffer(newbuf, BUFFER_LOCK_UNLOCK)", + "ReleaseBuffer(newbuf)", + "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", + "cluster_heap_vm_barrier_warm", + "goto l_pgrac_reacquire;" }; + static const char *const delete_order[] = { "PGRAC: vm barrier unwind (delete requalify)", + "LockBuffer(buffer, BUFFER_LOCK_UNLOCK)", + "cluster_heap_vm_barrier_warm", + "ReleaseBuffer(vmbuffer)", + "vmbuffer = InvalidBuffer", + "cluster_heap_lock_with_vm_repin", + "goto l1;" }; + + /* The warm helper itself may only run with no content lock held: + * it must take and drop the map-page lock, nothing else. */ + static const char *const warm_order[] = { "cluster_heap_vm_barrier_warm(Buffer vmbuf)", + "LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE)", + "LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK)" }; + + assert_ordered(heapam, warm_order, lengthof(warm_order)); + assert_ordered(heapam, pretoast_order, lengthof(pretoast_order)); + assert_ordered(heapam, requalify_order, lengthof(requalify_order)); + assert_ordered(heapam, reacquire_order, lengthof(reacquire_order)); + assert_ordered(heapam, delete_order, lengthof(delete_order)); + free(heapam); + } +} + +/* P0-20: an UPDATE must not keep its visibility-map pin while it can block + * acquiring heap X. Another writer can already be transferring that same + * VM page, and VM/FSM deliberately cannot become a retained PI while pinned. + * Pre-read the VM page, release the pin before the heap PCM wait, then repin + * only the exact recent descriptor without I/O after heap X is held. */ +UT_TEST(test_heap_update_drops_vm_pin_across_heap_pcm_wait) +{ + char *heapam = read_source(HEAPAM_SOURCE_PATH); + char *visibilitymap = read_source(VM_SOURCE_PATH); + + UT_ASSERT(heapam != NULL); + UT_ASSERT(visibilitymap != NULL); + if (heapam != NULL) { + const char *helper = strstr(heapam, "cluster_heap_lock_with_vm_repin("); + const char *helper_end = helper != NULL ? strstr(helper, "\n}\n") : NULL; + const char *update = strstr(heapam, "\nheap_update("); + const char *update_lock + = update != NULL ? strstr(update, "cluster_heap_lock_with_vm_repin(") : NULL; + const char *repin_branch + = update != NULL + ? strstr(update, "if (vmbuffer == InvalidBuffer && PageIsAllVisible(page))") + : NULL; + const char *repin_branch_end + = repin_branch != NULL ? strstr(repin_branch, "goto l2;") : NULL; + const char *repin_branch_helper + = repin_branch != NULL ? strstr(repin_branch, "cluster_heap_lock_with_vm_repin(") + : NULL; + const char *success_tail + = update != NULL ? strstr(update, "recptr = log_heap_update(") : NULL; + const char *vm_unlock + = success_tail != NULL + ? strstr(success_tail, + "if (vm_locked)\n\t\tLockBuffer(vmbuffer, BUFFER_LOCK_UNLOCK);") + : NULL; + const char *vm_release + = vm_unlock != NULL ? strstr(vm_unlock, "ReleaseBuffer(vmbuffer);") : NULL; + const char *heap_unlock = vm_unlock != NULL + ? strstr(vm_unlock, "LockBuffer(buffer, BUFFER_LOCK_UNLOCK);") + : NULL; + static const char *const helper_order[] + = { "visibilitymap_pin(relation, heap_block, vmbuffer)", + "ReleaseBuffer(*vmbuffer)", + "*vmbuffer = InvalidBuffer", + "LockBuffer(heap_buffer, BUFFER_LOCK_EXCLUSIVE)", + "visibilitymap_pin_recent(relation, heap_block, recent_vm, vmbuffer)", + "LockBuffer(heap_buffer, BUFFER_LOCK_UNLOCK)" }; + + UT_ASSERT(helper != NULL); + UT_ASSERT(helper_end != NULL); + UT_ASSERT(update != NULL); + UT_ASSERT(update_lock != NULL); + UT_ASSERT(repin_branch != NULL); + UT_ASSERT(repin_branch_end != NULL); + UT_ASSERT(repin_branch_helper != NULL); + UT_ASSERT(repin_branch_helper < repin_branch_end); + UT_ASSERT(vm_unlock != NULL); + UT_ASSERT(vm_release != NULL); + UT_ASSERT(heap_unlock != NULL); + UT_ASSERT(vm_release < heap_unlock); + if (helper != NULL && helper_end != NULL) + assert_ordered(helper, helper_order, lengthof(helper_order)); + free(heapam); + } + if (visibilitymap != NULL) { + const char *repin = strstr(visibilitymap, "\nvisibilitymap_pin_recent("); + const char *repin_end = repin != NULL ? strstr(repin, "\n}\n") : NULL; + static const char *const repin_order[] + = { "HEAPBLK_TO_MAPBLOCK(heapBlk)", "ReadRecentBuffer(", "VISIBILITYMAP_FORKNUM", + "recent_buffer" }; + + UT_ASSERT(repin != NULL); + UT_ASSERT(repin_end != NULL); + if (repin != NULL && repin_end != NULL) + assert_ordered(repin, repin_order, lengthof(repin_order)); + free(visibilitymap); + } +} + +/* P0-20: RelationGetBufferForTuple acquires two heap content locks for a + * cross-page UPDATE. If the second queue acquire sees the first page under + * a frozen revoke barrier, it must release the first lock, resolve the second + * conversion with no content lock held, and retry the ordered pair. */ +UT_TEST(test_cross_page_heap_pair_barrier_refusal_retries_unlocked) +{ + char *hio = read_source(HIO_SOURCE_PATH); + const char *helper; + const char *helper_end; + const char *pins; + const char *pins_end; + const char *pins_call; + const char *relation; + const char *relation_end; + const char *lower_branch; + const char *lower_call; + const char *upper_branch; + const char *upper_call; + const char *extension; + const char *extension_call; + static const char *const helper_order[] + = { "cluster_hio_lock_buffer_pair(Buffer first, Buffer second)", + "LockBuffer(first, BUFFER_LOCK_EXCLUSIVE)", + "ClusterLockBufferExclusiveBarrierAware(second)", + "LockBuffer(first, BUFFER_LOCK_UNLOCK)", + "LockBuffer(second, BUFFER_LOCK_EXCLUSIVE)", + "LockBuffer(second, BUFFER_LOCK_UNLOCK)" }; + + UT_ASSERT(hio != NULL); + if (hio == NULL) + return; + helper = strstr(hio, "cluster_hio_lock_buffer_pair(Buffer first, Buffer second)"); + helper_end = helper != NULL ? strstr(helper, "\n}\n") : NULL; + pins = strstr(hio, "GetVisibilityMapPins(Relation relation"); + pins_end = pins != NULL ? strstr(pins, "\n}\n") : NULL; + pins_call = pins != NULL ? strstr(pins, "cluster_hio_lock_buffer_pair(") : NULL; + relation = strstr(hio, "\nRelationGetBufferForTuple("); + relation_end = relation != NULL ? strstr(relation, "\n}\n") : NULL; + lower_branch = relation != NULL ? strstr(relation, "else if (otherBlock < targetBlock)") : NULL; + lower_call + = lower_branch != NULL ? strstr(lower_branch, "cluster_hio_lock_buffer_pair(") : NULL; + upper_branch = lower_branch != NULL ? strstr(lower_branch, "\n\t\telse\n\t\t{") : NULL; + upper_call + = upper_branch != NULL ? strstr(upper_branch, "cluster_hio_lock_buffer_pair(") : NULL; + extension = relation != NULL ? strstr(relation, "Reacquire locks if necessary") : NULL; + extension_call = extension != NULL ? strstr(extension, "cluster_hio_lock_buffer_pair(") : NULL; + + UT_ASSERT(helper != NULL); + UT_ASSERT(helper_end != NULL); + UT_ASSERT(pins != NULL); + UT_ASSERT(pins_end != NULL); + UT_ASSERT(relation != NULL); + UT_ASSERT(relation_end != NULL); + if (helper != NULL && helper_end != NULL) + assert_ordered(helper, helper_order, lengthof(helper_order)); + UT_ASSERT(pins != NULL && pins_end != NULL && pins_call != NULL && pins_call < pins_end); + UT_ASSERT(lower_branch != NULL && lower_call != NULL && upper_branch != NULL + && lower_call < upper_branch); + UT_ASSERT(upper_branch != NULL && upper_call != NULL && relation_end != NULL + && upper_call < relation_end); + UT_ASSERT(extension != NULL && extension_call != NULL && relation_end != NULL + && extension_call < relation_end); + free(hio); +} + int main(void) { - UT_PLAN(17); + UT_PLAN(20); UT_RUN(test_valid_read_miss_proof); UT_RUN(test_valid_extend_proof); UT_RUN(test_valid_vm_and_fsm_proofs); @@ -430,6 +650,9 @@ main(void) UT_RUN(test_valid_n_s_x_without_proof_enters_queue_before_legacy_wire); UT_RUN(test_direct_init_one_shot_image_cannot_return_without_x); UT_RUN(test_wire_throw_exact_aborts_reservation_before_rethrow); + UT_RUN(test_precrit_vm_barrier_refusal_unwinds_to_caller); + UT_RUN(test_heap_update_drops_vm_pin_across_heap_pcm_wait); + UT_RUN(test_cross_page_heap_pair_barrier_refusal_retries_unlocked); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_pcm_lock.c b/src/test/cluster_unit/test_cluster_pcm_lock.c index 1e50902305..8ddf14e20b 100644 --- a/src/test/cluster_unit/test_cluster_pcm_lock.c +++ b/src/test/cluster_unit/test_cluster_pcm_lock.c @@ -123,11 +123,28 @@ static struct { int holder_node; bool armed; } fake_cv_wake_release = { { 0 }, 0, false }; +static struct { + BufferTag tag; + int requester_node; + uint64 ticket_id; + bool armed; +} fake_cv_wake_pending_x_clear = { { 0 }, 0, 0, false }; static sigjmp_buf ut_ereport_jump; static bool ut_ereport_jump_armed = false; static int ut_ereport_fired_count = 0; static bool fake_local_x_upgrade_result = false; +static bool fake_acquire_entry_handoff_armed = false; +static BufferTag fake_acquire_entry_handoff_tag; +static int fake_acquire_entry_handoff_source = -1; +static int fake_acquire_entry_handoff_target = -1; +static PcmLockTransition fake_acquire_entry_handoff_release = PCM_TRANS_X_TO_N_RELEASE; +static int fake_local_read_image_count = 0; +static int fake_local_read_image_holder = -1; +static PcmAuthoritySnapshot fake_local_read_image_expected; +static int fake_local_x_transfer_count = 0; +static int fake_local_x_transfer_holder = -1; +static PcmAuthoritySnapshot fake_local_x_transfer_expected; void ExceptionalCondition(const char *conditionName pg_attribute_unused(), @@ -191,6 +208,12 @@ reset_fake_pcm_runtime(int max_entries) fake_cv_broadcast_count = 0; fake_cv_sleep_wait_event = 0; fake_cv_wake_release.armed = false; + fake_cv_wake_pending_x_clear.armed = false; + fake_acquire_entry_handoff_armed = false; + fake_acquire_entry_handoff_release = PCM_TRANS_X_TO_N_RELEASE; + fake_local_read_image_count = 0; + fake_local_read_image_holder = -1; + memset(&fake_local_read_image_expected, 0, sizeof(fake_local_read_image_expected)); fake_local_x_upgrade_result = false; cluster_node_id = 0; NBuffers = max_entries; @@ -456,6 +479,12 @@ ConditionVariableSleep(ConditionVariable *cv pg_attribute_unused(), uint32 wait_ cluster_pcm_lock_release(fake_cv_wake_release.tag); cluster_node_id = save_node; } + if (fake_cv_wake_pending_x_clear.armed) { + fake_cv_wake_pending_x_clear.armed = false; + UT_ASSERT(cluster_pcm_lock_clear_queue_pending_x_exact( + fake_cv_wake_pending_x_clear.tag, fake_cv_wake_pending_x_clear.requester_node, + fake_cv_wake_pending_x_clear.ticket_id)); + } } bool @@ -500,8 +529,21 @@ cluster_shmem_register_region(const ClusterShmemRegion *region pg_attribute_unus {} void -cluster_injection_run(const char *name pg_attribute_unused()) -{} +cluster_injection_run(const char *name) +{ + if (fake_acquire_entry_handoff_armed && strcmp(name, "cluster-pcm-acquire-entry") == 0) { + fake_acquire_entry_handoff_armed = false; + cluster_injection_armed_count = 0; + UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition(fake_acquire_entry_handoff_tag, + fake_acquire_entry_handoff_release, + fake_acquire_entry_handoff_source), + 1); + UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition(fake_acquire_entry_handoff_tag, + PCM_TRANS_N_TO_X, + fake_acquire_entry_handoff_target), + 1); + } +} /* PGRAC spec-2.32 D5 stubs: cluster_pcm_lock.c now calls cluster_gcs * helpers from each mutation entry point (master lookup branch). Test @@ -528,29 +570,42 @@ bool cluster_gcs_send_block_request_and_wait(struct BufferDesc *buf pg_attribute_unused(), PcmLockTransition trans pg_attribute_unused(), int master_node pg_attribute_unused(), - bool clean_eligible pg_attribute_unused()) + bool clean_eligible pg_attribute_unused(), + bool *out_retry_denied pg_attribute_unused()) { abort(); } /* spec-5.2 D2 sub-case B stub: local-master read-image forward. The pcm_lock - * fixtures force master == self with no remote X holder, so this is never - * reached; abort if it is. */ + * fixture records the selected holder so optimistic-precheck handoff tests can + * prove the buffer-aware S path routes to the existing one-shot image helper + * instead of the tag-only fail-closed terminal. */ bool cluster_gcs_local_master_read_image_and_wait(struct BufferDesc *buf pg_attribute_unused(), - int32 holder_node pg_attribute_unused()) + const PcmAuthoritySnapshot *expected, + bool *out_retry_denied) { - abort(); + *out_retry_denied = false; + fake_local_read_image_count++; + fake_local_read_image_expected = *expected; + fake_local_read_image_holder = expected->x_holder_node; + return false; } -/* spec-5.2 D11 stub: same rationale — the pcm_lock unit fixtures never drive a - * local-master X-transfer, so abort if reached. */ +/* spec-5.2 D11 stub: record the authoritative holder selected by the + * buffer-aware local-master path. The real data-plane behavior is covered by + * the GCS block tests; this fixture only proves PCM routing and authority. */ bool cluster_gcs_local_master_x_transfer_and_wait(struct BufferDesc *buf pg_attribute_unused(), - int32 holder_node pg_attribute_unused(), - bool clean_eligible pg_attribute_unused()) -{ - abort(); + const PcmAuthoritySnapshot *expected, + bool clean_eligible pg_attribute_unused(), + bool *out_retry_denied) +{ + *out_retry_denied = false; + fake_local_x_transfer_count++; + fake_local_x_transfer_expected = *expected; + fake_local_x_transfer_holder = expected->x_holder_node; + return true; } /* spec-2.35 D3 stub: HC110 master_holder lifecycle counter bump invoked @@ -1250,13 +1305,14 @@ UT_TEST(test_pcm_b_local_master_remote_x_holder_fail_closed) UT_TEST(test_pcm_d1_recovering_gate_fail_closed) { BufferDesc buf; + bool retry_denied = false; reset_fake_pcm_runtime(2); buf.tag = make_tag(77); fake_block_phase = GCS_BLOCK_RECOVERING; cluster_gcs_block_recovery_wait_ms = 0; /* immediate fail-closed, no sleep */ - UT_EXPECT_EREPORT(cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_S)); + UT_EXPECT_EREPORT(cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_S, &retry_denied)); fake_block_phase = GCS_BLOCK_NORMAL; cluster_gcs_block_recovery_wait_ms = 200; } @@ -1433,6 +1489,7 @@ UT_TEST(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades) BufferTag tag = make_tag(96); BufferDesc buf; bool save = cluster_gcs_block_local_cache; + bool retry_denied = false; reset_fake_pcm_runtime(4); fake_cssd_dead_node = -1; @@ -1447,7 +1504,8 @@ UT_TEST(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades) buf.tag = tag; buf.pcm_state = (uint8)PCM_STATE_N; cluster_node_id = 0; - UT_ASSERT(cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_X)); + UT_ASSERT(cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_X, &retry_denied)); + UT_ASSERT(!retry_denied); UT_ASSERT_EQ((int)cluster_pcm_lock_query(tag), (int)PCM_LOCK_MODE_X); UT_ASSERT(!cluster_pcm_master_other_live_holder_exists(tag, 0)); UT_ASSERT(cluster_pcm_master_requester_is_holder(tag, 0, PCM_TRANS_N_TO_X)); @@ -1455,6 +1513,246 @@ UT_TEST(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades) cluster_gcs_block_local_cache = save; } +/* + * P0-26 sibling race: the local-master buffer-aware X path observes shared S + * with no local S bit, then bootstraps a local S declaration before upgrade. + * A queue handoff may replace that S authority with remote X in between. The + * nested acquire must preserve the BufferDesc-aware exact transfer route, + * never escape through the tag-only legacy terminal. + */ +UT_TEST(test_pcm_acquire_buffer_s_bootstrap_revalidates_remote_x) +{ + BufferTag tag = make_tag(970); + BufferDesc buf; + PcmAuthoritySnapshot after; + bool retry_denied = false; + bool acquired = false; + bool escaped_error = false; + + reset_fake_pcm_runtime(4); + cluster_gcs_block_local_cache = true; + cluster_node_id = 1; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + + memset(&buf, 0, sizeof(buf)); + buf.tag = tag; + buf.pcm_state = (uint8)PCM_STATE_N; + cluster_node_id = 0; + fake_local_x_transfer_count = 0; + fake_local_x_transfer_holder = -1; + fake_acquire_entry_handoff_tag = tag; + fake_acquire_entry_handoff_source = 1; + fake_acquire_entry_handoff_target = 2; + fake_acquire_entry_handoff_release = PCM_TRANS_S_TO_N_RELEASE; + cluster_injection_armed_count = 1; + fake_acquire_entry_handoff_armed = true; + + if (sigsetjmp(ut_ereport_jump, 1) == 0) { + ut_ereport_jump_armed = true; + acquired = cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_X, &retry_denied); + ut_ereport_jump_armed = false; + } else { + ut_ereport_jump_armed = false; + escaped_error = true; + } + + UT_ASSERT(!escaped_error); + UT_ASSERT(acquired); + UT_ASSERT(!retry_denied); + UT_ASSERT_EQ(fake_local_x_transfer_count, 1); + UT_ASSERT_EQ(fake_local_x_transfer_holder, 2); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_X); + UT_ASSERT_EQ(after.x_holder_node, 2); + UT_ASSERT_EQ(memcmp(&fake_local_x_transfer_expected, &after, sizeof(after)), 0); +} + +/* P0-26 third entry: the ordinary buffer-aware S path also has an optimistic + * remote-X precheck. A local-X -> remote-X queue handoff between that check + * and the entry-lock acquire must retain the BufferDesc-aware read-image + * route; it must not fall through the tag-only legacy terminal. */ +UT_TEST(test_pcm_acquire_buffer_s_revalidates_remote_x_after_precheck) +{ + BufferTag tag = make_tag(971); + BufferDesc buf; + PcmAuthoritySnapshot after; + bool retry_denied = false; + bool acquired = true; + bool escaped_error = false; + + reset_fake_pcm_runtime(4); + cluster_gcs_block_local_cache = true; + cluster_node_id = 0; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_X); + + memset(&buf, 0, sizeof(buf)); + buf.tag = tag; + fake_acquire_entry_handoff_tag = tag; + fake_acquire_entry_handoff_source = 0; + fake_acquire_entry_handoff_target = 1; + cluster_injection_armed_count = 1; + fake_acquire_entry_handoff_armed = true; + + if (sigsetjmp(ut_ereport_jump, 1) == 0) { + ut_ereport_jump_armed = true; + acquired = cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_S, &retry_denied); + ut_ereport_jump_armed = false; + } else { + ut_ereport_jump_armed = false; + escaped_error = true; + } + + UT_ASSERT(!escaped_error); + UT_ASSERT(!acquired); /* one-shot READ_IMAGE is intentionally non-durable */ + UT_ASSERT(!retry_denied); + UT_ASSERT_EQ(fake_local_read_image_count, 1); + UT_ASSERT_EQ(fake_local_read_image_holder, 1); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_X); + UT_ASSERT_EQ(after.x_holder_node, 1); + UT_ASSERT_EQ(memcmp(&fake_local_read_image_expected, &after, sizeof(after)), 0); +} + +/* P0-26: the buffer-aware local-master X path used to inspect state/holder, + * then call the tag-only acquire without carrying an authoritative token. + * Commit a queue-style local-X -> remote-X handoff at the existing acquire + * entry injection point, exactly after the optimistic precheck and before the + * tag-only entry lock. The buffer-aware caller must redirect to the existing + * safe X-transfer path; the old code escapes through the legacy + * "cross-node block write transfer not supported" ERROR instead. + */ +UT_TEST(test_pcm_acquire_buffer_revalidates_remote_x_after_precheck) +{ + BufferTag tag = make_tag(97); + BufferDesc buf; + PcmAuthoritySnapshot after; + bool retry_denied = false; + bool acquired = false; + bool escaped_error = false; + + reset_fake_pcm_runtime(4); + cluster_gcs_block_local_cache = true; + cluster_node_id = 0; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_X); + + memset(&buf, 0, sizeof(buf)); + buf.tag = tag; + fake_local_x_transfer_count = 0; + fake_local_x_transfer_holder = -1; + fake_acquire_entry_handoff_tag = tag; + fake_acquire_entry_handoff_source = 0; + fake_acquire_entry_handoff_target = 1; + cluster_injection_armed_count = 1; + fake_acquire_entry_handoff_armed = true; + + if (sigsetjmp(ut_ereport_jump, 1) == 0) { + ut_ereport_jump_armed = true; + acquired = cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_X, &retry_denied); + ut_ereport_jump_armed = false; + } else { + ut_ereport_jump_armed = false; + escaped_error = true; + } + + UT_ASSERT(!escaped_error); + UT_ASSERT(acquired); + UT_ASSERT(!retry_denied); + UT_ASSERT_EQ(fake_local_x_transfer_count, 1); + UT_ASSERT_EQ(fake_local_x_transfer_holder, 1); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_X); + UT_ASSERT_EQ(after.x_holder_node, 1); + UT_ASSERT_EQ((long)after.s_holders_bitmap, 0L); + UT_ASSERT_EQ(memcmp(&fake_local_x_transfer_expected, &after, sizeof(after)), 0); +} + +UT_TEST(test_pcm_acquire_buffer_routes_unchanged_remote_x_with_exact_authority) +{ + BufferTag tag = make_tag(98); + BufferDesc buf; + PcmAuthoritySnapshot before; + PcmAuthoritySnapshot after; + bool retry_denied = false; + + reset_fake_pcm_runtime(4); + cluster_gcs_block_local_cache = true; + cluster_node_id = 0; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_X); + UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_X_TO_N_RELEASE, 0), 1); + UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_X, 1), 1); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); + + memset(&buf, 0, sizeof(buf)); + buf.tag = tag; + fake_local_x_transfer_count = 0; + fake_local_x_transfer_holder = -1; + + UT_ASSERT(cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_X, &retry_denied)); + UT_ASSERT(!retry_denied); + UT_ASSERT_EQ(fake_local_x_transfer_count, 1); + UT_ASSERT_EQ(fake_local_x_transfer_holder, 1); + UT_ASSERT_EQ(memcmp(&fake_local_x_transfer_expected, &before, sizeof(before)), 0); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ(memcmp(&after, &before, sizeof(before)), 0); +} + +UT_TEST(test_pcm_x_transfer_commit_is_exact_and_late_reply_safe) +{ + BufferTag tag = make_tag(99); + PcmAuthoritySnapshot expected; + PcmAuthoritySnapshot stale; + PcmAuthoritySnapshot after; + PcmAuthoritySnapshot committed; + + reset_fake_pcm_runtime(4); + cluster_node_id = 0; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_X); + UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_X_TO_N_RELEASE, 0), 1); + UT_ASSERT_EQ((int)cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_X, 1), 1); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &expected)); + UT_ASSERT(cluster_pcm_lock_authority_matches(tag, &expected)); + + stale = expected; + stale.transition_count++; + UT_ASSERT_EQ(cluster_pcm_lock_master_take_x_after_transfer(tag, &stale, (XLogRecPtr)0x1234, + InvalidScn, 1, 44, 900, 12), + PCM_X_TRANSFER_COMMIT_STALE); + stale = expected; + stale.master_holder.request_id++; + UT_ASSERT_EQ(cluster_pcm_lock_master_take_x_after_transfer(tag, &stale, (XLogRecPtr)0x1234, + InvalidScn, 1, 44, 900, 12), + PCM_X_TRANSFER_COMMIT_STALE); + stale = expected; + stale.master_holder.cluster_epoch++; + UT_ASSERT_EQ(cluster_pcm_lock_master_take_x_after_transfer(tag, &stale, (XLogRecPtr)0x1234, + InvalidScn, 1, 44, 900, 12), + PCM_X_TRANSFER_COMMIT_STALE); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ(memcmp(&after, &expected, sizeof(expected)), 0); + + UT_ASSERT_EQ(cluster_pcm_lock_master_take_x_after_transfer(tag, &expected, (XLogRecPtr)0x1234, + InvalidScn, 1, 44, 900, 12), + PCM_X_TRANSFER_COMMIT_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &committed)); + UT_ASSERT_EQ((int)committed.state, (int)PCM_STATE_X); + UT_ASSERT_EQ(committed.x_holder_node, 0); + UT_ASSERT_EQ((long)committed.s_holders_bitmap, 0L); + UT_ASSERT_EQ(committed.pending_x_requester_node, -1); + UT_ASSERT_EQ((long)committed.master_holder.node_id, 0L); + UT_ASSERT_EQ((long)committed.master_holder.procno, 44L); + UT_ASSERT_EQ((uint64)committed.master_holder.cluster_epoch, (uint64)12); + UT_ASSERT_EQ((uint64)committed.master_holder.request_id, (uint64)900); + UT_ASSERT_EQ((uint64)committed.transition_count, (uint64)expected.transition_count + 1); + + /* A duplicate/late reply carries the displaced remote-X token. */ + UT_ASSERT_EQ(cluster_pcm_lock_master_take_x_after_transfer(tag, &expected, (XLogRecPtr)0x1234, + InvalidScn, 1, 44, 900, 12), + PCM_X_TRANSFER_COMMIT_STALE); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ(memcmp(&after, &committed, sizeof(committed)), 0); +} + + UT_TEST(test_pcm_dead_node_cleanup_drops_holder_records) { BufferTag stag = make_tag(94); @@ -1537,6 +1835,33 @@ make_pcm_x_grd_handoff_token(BufferTag tag, const PcmAuthoritySnapshot *authorit return token; } +static char * +read_gcs_block_source(void) +{ + FILE *file; + char *source; + long length; + + file = fopen(GCS_BLOCK_SOURCE_PATH, "rb"); + UT_ASSERT_NOT_NULL(file); + if (file == NULL) + return NULL; + UT_ASSERT_EQ(fseek(file, 0, SEEK_END), 0); + length = ftell(file); + UT_ASSERT(length > 0); + UT_ASSERT_EQ(fseek(file, 0, SEEK_SET), 0); + source = malloc((size_t)length + 1); + UT_ASSERT_NOT_NULL(source); + if (source == NULL) { + fclose(file); + return NULL; + } + UT_ASSERT_EQ(fread(source, 1, (size_t)length, file), (size_t)length); + source[length] = '\0'; + fclose(file); + return source; +} + UT_TEST(test_pcm_authority_snapshot_is_one_entry_lock_view) { BufferTag tag = make_tag(97); @@ -1628,6 +1953,84 @@ UT_TEST(test_pcm_queue_pending_x_reservation_never_overwrites_another_node) PCM_PENDING_X_RESERVE_NO_CAPACITY); } +/* P0-25: pending-X is an admission barrier, not only an advisory wire gate. + * The final decision and the S-holder bitmap mutation share entry_lock, so an + * N->S request that raced past an earlier handler preflight cannot publish a + * new holder after the queue writer claimed the tag. An already-recorded S + * holder may re-enter without changing the authority bytes. */ +UT_TEST(test_pcm_pending_x_blocks_new_remote_s_holder_atomically) +{ + BufferTag tag = make_tag(110); + PcmAuthoritySnapshot before; + PcmAuthoritySnapshot after; + + reset_fake_pcm_runtime(4); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9010), PCM_PENDING_X_RESERVE_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); + UT_ASSERT_EQ((int)before.state, (int)PCM_STATE_N); + UT_ASSERT_EQ(cluster_pcm_lock_apply_gcs_transition_result(tag, PCM_TRANS_N_TO_S, 1), + PCM_GCS_TRANSITION_PENDING_X); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_N); + UT_ASSERT_EQ(after.s_holders_bitmap, (uint32)0); + UT_ASSERT_EQ(after.transition_count, before.transition_count); + + UT_ASSERT(cluster_pcm_lock_clear_queue_pending_x_exact(tag, 3, 9010)); + UT_ASSERT(cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_S, 1)); + UT_ASSERT_EQ(cluster_pcm_lock_query_s_holders_bitmap(tag), (uint32)(1u << 1)); + + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9011), PCM_PENDING_X_RESERVE_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); + UT_ASSERT(cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_S, 1)); + UT_ASSERT_EQ(cluster_pcm_lock_apply_gcs_transition_result(tag, PCM_TRANS_N_TO_S, 2), + PCM_GCS_TRANSITION_PENDING_X); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ(after.s_holders_bitmap, before.s_holders_bitmap); + UT_ASSERT_EQ(after.transition_count, before.transition_count); + + UT_ASSERT(cluster_pcm_lock_clear_queue_pending_x_exact(tag, 3, 9011)); + UT_ASSERT(cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_S, 2)); + UT_ASSERT_EQ(cluster_pcm_lock_query_s_holders_bitmap(tag), (uint32)((1u << 1) | (1u << 2))); +} + +/* P0-25 local-master mirror: an existing local S holder re-enters immediately, + * while a different local node waits until the queue cookie is cleared. The + * CV callback deterministically performs that clear in this single-threaded + * harness, proving both no pre-clear bitmap publication and post-clear liveness. */ +UT_TEST(test_pcm_pending_x_blocks_new_local_s_holder_until_clear) +{ + BufferTag tag = make_tag(111); + + reset_fake_pcm_runtime(4); + cluster_node_id = 0; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9012), PCM_PENDING_X_RESERVE_OK); + + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(fake_cv_sleep_count, 0); + UT_ASSERT_EQ(cluster_pcm_lock_query_s_holders_bitmap(tag), (uint32)(1u << 0)); + + fake_cv_wake_pending_x_clear.tag = tag; + fake_cv_wake_pending_x_clear.requester_node = 3; + fake_cv_wake_pending_x_clear.ticket_id = 9012; + fake_cv_wake_pending_x_clear.armed = true; + cluster_node_id = 1; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(fake_cv_sleep_count, 1); + UT_ASSERT_EQ((int)fake_cv_sleep_wait_event, (int)WAIT_EVENT_PCM_COMPATIBLE_STATE_WAIT); + UT_ASSERT_EQ(cluster_pcm_lock_query_pending_x_requester(tag), -1); + UT_ASSERT_EQ(cluster_pcm_lock_query_s_holders_bitmap(tag), (uint32)((1u << 0) | (1u << 1))); + + /* A reader woken by FINAL may observe the newly granted X and sleep again. + * The later holder X->S downgrade must wake it when S becomes compatible. */ + tag = make_tag(112); + reset_fake_pcm_runtime(4); + cluster_node_id = 0; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_X); + UT_ASSERT(cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_X_TO_S_DOWNGRADE, 0)); + UT_ASSERT_EQ(fake_cv_broadcast_count, 1); +} + UT_TEST(test_pcm_queue_handoff_x_exact_rejects_authority_drift) { BufferTag tag = make_tag(98); @@ -1642,12 +2045,13 @@ UT_TEST(test_pcm_queue_handoff_x_exact_rejects_authority_drift) UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); token = make_pcm_x_grd_handoff_token(tag, &before, 1, 3, 41, 9001); - cluster_node_id = 2; - cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + /* A holder release remains legal under pending-X and still invalidates the + * optimistic authority token; a new S-holder admission is now prohibited. */ + cluster_pcm_lock_release(tag); UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&token), PCM_X_GRD_HANDOFF_STALE); UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); - UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_S); - UT_ASSERT_EQ(after.s_holders_bitmap, (uint32)((1u << 1) | (1u << 2))); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_N); + UT_ASSERT_EQ(after.s_holders_bitmap, (uint32)0); } UT_TEST(test_pcm_queue_handoff_x_exact_rejects_residual_s_holder) @@ -1847,6 +2251,143 @@ UT_TEST(test_pcm_queue_handoff_x_exact_uses_scn_not_cross_stream_lsn) UT_ASSERT_EQ(after.pending_x_requester_node, 3); } +/* + * P0-20: reproduce the real ordering that failed t/400 at 14:23 without a + * scheduler race. A and B first hold S. A is the pre-acked transfer source; + * the production-equivalent effects of B's exact slotless INVALIDATE_ACK then + * remove B and advance the monotone watermark to W. A subsequently presents + * an older materialized image S. + * + * The final GRD handoff gate remains the non-negotiable last defence, but the + * protocol must classify this as a recoverable stale source before type 50 is + * accepted and PREPARE_GRANT is emitted. This test deliberately combines the + * real GRD authority/watermark operations with a bounded source contract over + * the production ACK and IMAGE_READY handlers. It does not call the static + * ACK handler directly; separate gate/provenance tests did not expose this + * inter-handler gap. + */ +UT_TEST(test_pcm_x_slotless_ack_floor_fences_stale_source_before_prepare) +{ + BufferTag tag = make_tag(111); + PcmAuthoritySnapshot authority; + PcmAuthoritySnapshot after; + PcmXGrdHandoffToken stale_handoff; + ClusterPcmWmProv provenance; + const SCN source_scn = (SCN)0x4000; + const SCN watermark_scn = (SCN)0x5000; + char *source; + const char *ack_handler; + const char *ack_end; + const char *ack_match; + const char *holder_remove; + const char *watermark_advance; + const char *bitmap_replace; + const char *drive; + const char *ready_handler; + const char *ready_end; + const char *floor_query; + const char *floor_verdict; + const char *image_ready; + const char *prepare; + bool source_floor_gate; + + reset_fake_pcm_runtime(4); + cluster_node_id = 0; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + cluster_node_id = 1; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9011), PCM_PENDING_X_RESERVE_OK); + + /* Source A (node 0) is pre-acked by the transfer driver. Apply the two + * production-equivalent state effects of non-source B's ACK (node 1), then + * use the bounded source contract below to prove their real handler order. */ + UT_ASSERT(cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_S_TO_N_INVALIDATE, 1)); + cluster_pcm_lock_pi_watermark_scn_advance(tag, watermark_scn, CLUSTER_PCM_WM_SRC_ACK_SLOTLESS, + 1, 9011, 17); + UT_ASSERT(cluster_pcm_lock_pi_watermark_prov_query(tag, &provenance)); + UT_ASSERT_EQ((int)provenance.source, (int)CLUSTER_PCM_WM_SRC_ACK_SLOTLESS); + UT_ASSERT_EQ(provenance.sender_node, 1); + UT_ASSERT_EQ(provenance.request_id, UINT64_C(9011)); + UT_ASSERT_EQ((uint64)provenance.new_scn, (uint64)watermark_scn); + UT_ASSERT_EQ(cluster_pcm_lock_query_s_holders_bitmap(tag), UINT32_C(1) << 0); + UT_ASSERT(cluster_pcm_lock_queue_pending_x_exact(tag, 3, 9011)); + UT_ASSERT_EQ((int)gcs_block_lost_write_verdict(watermark_scn, source_scn), + (int)GCS_LOST_WRITE_FAIL_STALE); + + /* The existing last line of defence must still refuse S < W and retain + * the pending-X authority for an exact retry/re-source path. */ + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &authority)); + stale_handoff = make_pcm_x_grd_handoff_token(tag, &authority, 0, 3, 52, 9011); + stale_handoff.page_scn = source_scn; + UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&stale_handoff), + PCM_X_GRD_HANDOFF_BAD_STATE); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_S); + UT_ASSERT_EQ(after.s_holders_bitmap, UINT32_C(1) << 0); + UT_ASSERT_EQ(after.pending_x_requester_node, 3); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_pi_watermark_scn_query(tag), (uint64)watermark_scn); + + /* Pin the production ordering: B's exact ACK must publish W before the + * bitmap can drive type 49. IMAGE_READY must then consume that current + * floor and classify SS downgrade) must NEVER + * become a legal finish shape. Writer conversions are ordered by the + * convert queue's FIFO/WFG; legalizing this shape at the finish would let + * that fallback bypass the arbitration entirely (the original S3 unordered + * multi-writer defect). The fallback must re-enter the queue instead; + * this classifier keeps refusing the bypass. + */ +UT_TEST(test_s_new_fresh_token_finish_shape_stays_invalid) +{ + ClusterPcmOwnSnapshot base; + ClusterPcmOwnSnapshot live; + + /* The exact loop9/loop10b production tuple stays refused. */ + memset(&base, 0, sizeof(base)); + base.generation = 10; + base.reservation_token = 5; + base.pcm_state = (uint8)PCM_STATE_S; + live = base; + live.reservation_token = 6; + live.flags = PCM_OWN_FLAG_GRANT_PENDING; + UT_ASSERT_EQ(cluster_pcm_x_grant_reservation_kind(&live, &base, 6), + CLUSTER_PCM_X_GRANT_RESERVATION_INVALID); + + /* A fresh-token X base is refused the same way: a live X cover never + * re-acquires through this path. */ + base.pcm_state = (uint8)PCM_STATE_X; + live = base; + live.reservation_token = 6; + live.flags = PCM_OWN_FLAG_GRANT_PENDING; + UT_ASSERT_EQ(cluster_pcm_x_grant_reservation_kind(&live, &base, 6), + CLUSTER_PCM_X_GRANT_RESERVATION_INVALID); +} + +UT_TEST(test_share_cover_reverify_accepts_stable_successor_grant) +{ + /* Once content authority is held, a stable current S/X successor is the + * exact node-level grant for a read. Generation drift alone must not open + * a fresh legacy reservation from S (the forbidden S_NEW shape). */ + UT_ASSERT(cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_S, UINT64_C(13), + UINT64_C(14), (uint8)PCM_STATE_S, 0)); + UT_ASSERT(cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_S, UINT64_C(13), + UINT64_C(14), (uint8)PCM_STATE_X, 0)); + + /* A writer keeps the stricter generation-exact gate and must re-enter the + * convert queue after any ownership round. A non-covering or live + * lifecycle remains closed for both modes. */ + UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_X, UINT64_C(13), + UINT64_C(14), (uint8)PCM_STATE_X, 0)); + UT_ASSERT(cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_X, UINT64_C(14), + UINT64_C(14), (uint8)PCM_STATE_X, 0)); + UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_X, UINT64_C(14), + UINT64_C(14), (uint8)PCM_STATE_S, 0)); + UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_S, UINT64_C(14), + UINT64_C(14), (uint8)PCM_STATE_S, + PCM_OWN_FLAG_GRANT_PENDING)); + UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_S, UINT64_C(14), + UINT64_C(14), (uint8)PCM_STATE_S, + PCM_OWN_FLAG_REVOKING)); + UT_ASSERT(!cluster_pcm_x_cached_cover_reverify_accepts((uint8)PCM_LOCK_MODE_N, UINT64_C(14), + UINT64_C(14), (uint8)PCM_STATE_X, 0)); +} + UT_TEST(test_revoke_commit_is_exact_and_classifies_live_races) { uint64 committed = UINT64_MAX; @@ -740,7 +868,7 @@ UT_TEST(test_lockbuffer_content_error_uses_post_master_rollback_contract) assert_ordered_in_function(source, "\ncluster_pcm_own_rollback_grant_after_error_and_rethrow(", "\nstatic ", rethrow_contract, lengthof(rethrow_contract)); assert_ordered_in_function( - source, "\nLockBuffer(Buffer buffer, int mode)", + source, "\nLockBufferInternal(Buffer buffer, int mode", "\n/*\n * Acquire the content_lock for the buffer, but only if we don't have to wait.", content_error_contract, lengthof(content_error_contract)); free(source); @@ -775,11 +903,13 @@ UT_TEST(test_bufmgr_generation_bump_failure_is_classified_under_header_lock) UT_TEST(test_lockbuffer_reservation_failures_use_busy_corrupt_classifier) { static const char *const initial_reservation_contract[] - = { "cluster_pcm_own_begin_grant_reservation", "pcm_pending_result != CLUSTER_PCM_OWN_OK", - "cluster_pcm_own_report_bump_failure", "pcm_pending_set = true" }; + = { "cluster_bufmgr_pcm_begin_grant_reservation_wait", + "pcm_pending_result != CLUSTER_PCM_OWN_OK", "cluster_pcm_own_report_bump_failure", + "pcm_pending_set = true" }; static const char *const revalidate_reservation_contract[] - = { "cluster_pcm_own_begin_grant_reservation", "pcm_pending_result != CLUSTER_PCM_OWN_OK", - "cluster_pcm_own_report_bump_failure", "pcm_pending_set = true" }; + = { "cluster_bufmgr_pcm_begin_grant_reservation_wait", + "pcm_pending_result != CLUSTER_PCM_OWN_OK", "cluster_pcm_own_report_bump_failure", + "pcm_pending_set = true" }; char *source = read_bufmgr_source(); /* A live queue reservation is BUSY, not damaged metadata. Both legacy @@ -795,6 +925,30 @@ UT_TEST(test_lockbuffer_reservation_failures_use_busy_corrupt_classifier) free(source); } +UT_TEST(test_pending_x_denied_retry_leaves_master_invalidate_gap) +{ + static const char *const retry_contract[] + = { "cluster_pcm_own_abort_grant_reservation", + "cluster_bufmgr_pcm_pending_x_retry_delay_ms(wait_index)", "WaitLatch", + "cluster_bufmgr_pcm_begin_grant_reservation_wait" }; + static const char *const delay_contract[] + = { "cluster_gcs_block_starvation_backoff_ms", "cluster_lmon_main_loop_interval", + "retry_delay_ms <<= shift", "retry_delay_ms * 2" }; + char *source = read_bufmgr_source(); + + /* A queue INVALIDATE that met mirror-N + GRANT_PENDING returns BUSY and + * the PCM-X master schedules its retry no sooner than the LMON interval. + * A DENIED_PENDING_X reader must therefore leave the exact reservation + * absent for strictly longer than that master retry delay. Otherwise the + * two exponential schedules phase-lock forever: each reader request + * republishes GRANT_PENDING just as every INVALIDATE retry arrives. */ + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_retry_denied_rearm(", "\nstatic ", + retry_contract, lengthof(retry_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_pending_x_retry_delay_ms(", + "\nstatic ", delay_contract, lengthof(delay_contract)); + free(source); +} + UT_TEST(test_bufmgr_finish_rejects_invalid_state_and_initializes_acquire_result) { static const char *const finish_gate[] @@ -834,6 +988,119 @@ UT_TEST(test_bufmgr_finish_and_abort_gate_on_exact_base_state) free(source); } +UT_TEST(test_retained_release_retag_respects_pin_contract) +{ + BufferDesc buf; + uint32 buf_state; + uint8 type_before; + uint32 state_before; + + /* Unpinned: the released retained image is dropped -- !BM_VALID plus a + * BUF_TYPE_CURRENT retag makes the next ordinary read reload the current + * page bytes through the buffer-IO protocol. */ + memset(&buf, 0, sizeof(buf)); + buf.buffer_type = (uint8)BUF_TYPE_PI; + buf_state = BM_VALID | BM_TAG_VALID; + UT_ASSERT(cluster_pcm_x_retained_release_retag(&buf.buffer_type, &buf_state)); + UT_ASSERT_EQ(buf_state & BM_VALID, 0); + UT_ASSERT_EQ(buf.buffer_type, (uint8)BUF_TYPE_CURRENT); + + /* Pinned: a pre-existing PG pin freezes the page image -- the bytes may + * neither vanish (!BM_VALID under a pin breaks the pin contract and + * re-arms the legacy begin-over-invalid-base S_NEW mint) nor be reloaded + * in place (an image swap under the pin holder). The release must keep + * the established PI+BM_VALID never-write/never-serve N mirror byte-exact + * (the passive-pin invalidate release shape): the next S acquire installs + * over it via an exact GRANT_PENDING and republishes CURRENT, an X + * convert rides the convert queue, and eviction retags after the last + * pin drains. */ + memset(&buf, 0, sizeof(buf)); + buf.buffer_type = (uint8)BUF_TYPE_PI; + buf_state = (BM_VALID | BM_TAG_VALID) + BUF_REFCOUNT_ONE * 2; + type_before = buf.buffer_type; + state_before = buf_state; + UT_ASSERT(!cluster_pcm_x_retained_release_retag(&buf.buffer_type, &buf_state)); + UT_ASSERT_EQ(buf.buffer_type, type_before); + UT_ASSERT_EQ(buf_state, state_before); + + /* One pin behaves like many. */ + buf.buffer_type = (uint8)BUF_TYPE_PI; + buf_state = (BM_VALID | BM_TAG_VALID) + BUF_REFCOUNT_ONE; + UT_ASSERT(!cluster_pcm_x_retained_release_retag(&buf.buffer_type, &buf_state)); + UT_ASSERT(buf_state & BM_VALID); + UT_ASSERT_EQ(buf.buffer_type, (uint8)BUF_TYPE_PI); + + /* The kept mirror is republished CURRENT only by a byte-currency proof + * inside the exact open legacy grant lifecycle (a shipped-image install, + * a storage refresh, or an SCN PASS proof): pcm N + live GRANT_PENDING + + * nonzero token + BM_VALID + PI. Anything else must stay frozen so the + * finish valid-image gate keeps refusing an unproven stale cover. */ + UT_ASSERT(cluster_pcm_x_grant_pending_republish_shape( + (uint8)PCM_STATE_N, PCM_OWN_FLAG_GRANT_PENDING, 7, true, (uint8)BUF_TYPE_PI)); + UT_ASSERT(!cluster_pcm_x_grant_pending_republish_shape( + (uint8)PCM_STATE_S, PCM_OWN_FLAG_GRANT_PENDING, 7, true, (uint8)BUF_TYPE_PI)); + UT_ASSERT(!cluster_pcm_x_grant_pending_republish_shape((uint8)PCM_STATE_N, 0, 7, true, + (uint8)BUF_TYPE_PI)); + UT_ASSERT(!cluster_pcm_x_grant_pending_republish_shape( + (uint8)PCM_STATE_N, PCM_OWN_FLAG_REVOKING, 7, true, (uint8)BUF_TYPE_PI)); + UT_ASSERT(!cluster_pcm_x_grant_pending_republish_shape( + (uint8)PCM_STATE_N, PCM_OWN_FLAG_GRANT_PENDING, 0, true, (uint8)BUF_TYPE_PI)); + UT_ASSERT(!cluster_pcm_x_grant_pending_republish_shape( + (uint8)PCM_STATE_N, PCM_OWN_FLAG_GRANT_PENDING, 7, false, (uint8)BUF_TYPE_PI)); + UT_ASSERT(!cluster_pcm_x_grant_pending_republish_shape( + (uint8)PCM_STATE_N, PCM_OWN_FLAG_GRANT_PENDING, 7, true, (uint8)BUF_TYPE_CURRENT)); +} + +UT_TEST(test_retained_release_and_finish_never_cover_invalid_bytes) +{ + static const char *const release_retag_contract[] + = { "cluster_pcm_own_revoke_retain_release_exact", "cluster_pcm_x_retained_release_retag", + "UnlockBufHdr" }; + static const char *const finish_valid_gate[] + = { "cluster_pcm_x_grant_reservation_kind", "BUF_TYPE_PI", + "(buf_state & (BM_VALID | BM_IO_IN_PROGRESS)) == 0", "CLUSTER_PCM_OWN_CORRUPT", + "cluster_pcm_own_grant_commit_exact" }; + char *source = read_bufmgr_source(); + + /* The retained release must route its descriptor retag through the shared + * pin-aware decision helper under the same header-lock hold that released + * the exact write-fence token. And a grant finish must never commit a + * durable S/X mirror over a page image that is not current (!BM_VALID or + * a PI mirror): that silent cover of stale bytes is how the pinned + * descriptor was previously stamped S over an invalid base, re-arming the + * refused legacy S_NEW convert (deterministic client ERROR) -- and, on + * the PI arm, a Rule 8.A stale read. The one legal !BM_VALID commit + * shape is the direct-init window (EXTEND/READ_MISS), whose gate still + * owns BM_IO_IN_PROGRESS between StartBufferIO and + * TerminateBufferIO(BM_VALID). Reloading the bytes in place under a + * foreign pin is equally forbidden. */ + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_release_retained_image(", + "\ncluster_bufmgr_pcm_own_self_handoff_probe(", + release_retag_contract, lengthof(release_retag_contract)); + assert_ordered_in_function(source, "\ncluster_pcm_own_finish_grant_reservation(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_finish_x_commit(", + finish_valid_gate, lengthof(finish_valid_gate)); + UT_ASSERT_NULL(strstr(source, "cluster_bufmgr_pcm_reload_invalid_pinned")); + free(source); +} + +UT_TEST(test_legacy_byte_proof_republishes_kept_pi_mirror) +{ + static const char *const republish_contract[] + = { "cluster_pcm_x_grant_pending_republish_shape", "BUF_TYPE_CURRENT", "UnlockBufHdr" }; + char *bufmgr_source = read_bufmgr_source(); + + /* A kept-pinned PI mirror regains CURRENT only where its bytes were just + * proven current inside the still-open legacy grant lifecycle (the + * gcs_block install / storage-fallback call sites are pinned in + * test_cluster_gcs_block). The helper flips only the exact republish + * shape under header authority. */ + assert_ordered_in_function( + bufmgr_source, "\ncluster_bufmgr_pcm_own_republish_grant_pending_image(", + "\nstatic ClusterPcmOwnResult", republish_contract, lengthof(republish_contract)); + free(bufmgr_source); +} + UT_TEST(test_d5a_release_error_keeps_descriptor_out_of_freelist) { static const char *const fail_closed_contract[] @@ -903,9 +1170,13 @@ UT_TEST(test_queue_contract_exposes_opaque_retained_revoke_api) typedef ClusterPcmOwnResult (*PrepareNSourceFn)(BufferDesc *, const ClusterPcmOwnSnapshot *, ClusterPcmOwnSnapshot *, char *, XLogRecPtr *, uint64 *); + typedef ClusterPcmOwnResult (*PrepareSSourceFn)( + BufferDesc *, const ClusterPcmOwnSnapshot *, SCN, ClusterPcmOwnSnapshot *, char *, + XLogRecPtr *, uint64 *, ClusterPcmOwnSourcePrepareRefusal *); typedef ClusterPcmOwnResult (*AbortRevokeFn)(BufferDesc *, const ClusterPcmOwnSnapshot *); typedef ClusterPcmOwnResult (*FinishRetainFn)(BufferDesc *, const ClusterPcmOwnSnapshot *, - XLogRecPtr, ClusterPcmOwnSnapshot *); + XLogRecPtr, ClusterPcmOwnSnapshot *, + ClusterPcmOwnFinishRefusal *); typedef ClusterPcmOwnResult (*ReleaseRetainedFn)(const BufferTag *, uint64); typedef bool (*ContentWriteFn)(BufferDesc *); @@ -917,6 +1188,8 @@ UT_TEST(test_queue_contract_exposes_opaque_retained_revoke_api) BeginRevokeFn)); UT_ASSERT(__builtin_types_compatible_p( __typeof__(&cluster_bufmgr_pcm_own_prepare_n_source_image), PrepareNSourceFn)); + UT_ASSERT(__builtin_types_compatible_p( + __typeof__(&cluster_bufmgr_pcm_own_prepare_s_source_image), PrepareSSourceFn)); UT_ASSERT(__builtin_types_compatible_p(__typeof__(&cluster_bufmgr_pcm_own_abort_n_revoke), AbortRevokeFn)); UT_ASSERT(__builtin_types_compatible_p(__typeof__(&cluster_bufmgr_pcm_own_abort_s_revoke), @@ -941,6 +1214,7 @@ UT_TEST(test_queue_n_source_refresh_is_exact_and_publishes_only_complete_image) "BM_IO_ERROR", "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED", "PinBuffer_Locked", + "LWLockConditionalAcquire(content_lock, LW_SHARED)", "FlushBuffer", "UnpinBuffer", "BM_IO_IN_PROGRESS", @@ -948,7 +1222,7 @@ UT_TEST(test_queue_n_source_refresh_is_exact_and_publishes_only_complete_image) "PCM_OWN_FLAG_REVOKING", "smgrread", "PageIsVerifiedExtended", - "LWLockAcquire(content_lock, LW_EXCLUSIVE)", + "LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)", "cluster_pcm_own_snapshot_matches_locked", "PCM_OWN_FLAG_REVOKING", "BM_VALID", @@ -975,6 +1249,38 @@ UT_TEST(test_queue_n_source_refresh_is_exact_and_publishes_only_complete_image) free(source); } +UT_TEST(test_queue_s_source_dirty_flush_makes_progress_and_reports_exact_refusal) +{ + static const char *const prepare_contract[] + = { "ReservePrivateRefCountEntry", + "ResourceOwnerEnlargeBuffers(CurrentResourceOwner)", + "BM_IO_ERROR", + "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED", + "PinBuffer_Locked", + "LWLockConditionalAcquire(content_lock, LW_SHARED)", + "CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_CONTENT_LOCK", + "FlushBuffer", + "CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_DIRTY_FLUSHED", + "UnpinBuffer", + "return CLUSTER_PCM_OWN_BUSY", + "BM_IO_IN_PROGRESS", + "CLUSTER_PCM_OWN_SOURCE_PREPARE_REFUSAL_IO_IN_PROGRESS", + "cluster_bufmgr_pcm_own_begin_s_revoke", + "smgrread", + "LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)" }; + char *source = read_bufmgr_source(); + + /* A clean checkpoint can be dirtied again by a SELECT's commit hint before + * the S->X self handoff. That is progress work, not a permanent refusal: + * pin and flush it before opening REVOKING, then retry. Keep I/O and both + * content-lock refusals separately diagnosed so a native stall cannot be + * collapsed back into an opaque materialize-begin BUSY. */ + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_prepare_s_source_image(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_abort_s_revoke(", + prepare_contract, lengthof(prepare_contract)); + free(source); +} + UT_TEST(test_revoke_finish_mode_rejects_pinned_vm_fsm_and_retains_main) { BufferTag tag; @@ -997,6 +1303,104 @@ UT_TEST(test_revoke_finish_mode_rejects_pinned_vm_fsm_and_retains_main) UT_ASSERT_EQ(cluster_pcm_x_revoke_finish_mode(NULL, 0), CLUSTER_PCM_X_REVOKE_FINISH_INVALID); } +UT_TEST(test_pcm_tracking_excludes_only_fsm_for_user_and_shared_catalog_relations) +{ + BufferTag tag; + + memset(&tag, 0, sizeof(tag)); + tag.relNumber = FirstNormalObjectId; + tag.forkNum = MAIN_FORKNUM; + UT_ASSERT(cluster_pcm_x_buffer_tag_tracked(&tag, false)); + tag.forkNum = INIT_FORKNUM; + UT_ASSERT(cluster_pcm_x_buffer_tag_tracked(&tag, false)); + tag.forkNum = VISIBILITYMAP_FORKNUM; + UT_ASSERT(cluster_pcm_x_buffer_tag_tracked(&tag, false)); + tag.forkNum = FSM_FORKNUM; + UT_ASSERT(!cluster_pcm_x_buffer_tag_tracked(&tag, false)); + + /* shared_catalog widens the relation-number domain, but must not put its + * advisory FSM fork back into the PCM/PCM-X authority domain. */ + tag.relNumber = FirstNormalObjectId - 1; + tag.forkNum = MAIN_FORKNUM; + UT_ASSERT(!cluster_pcm_x_buffer_tag_tracked(&tag, false)); + UT_ASSERT(cluster_pcm_x_buffer_tag_tracked(&tag, true)); + tag.forkNum = INIT_FORKNUM; + UT_ASSERT(cluster_pcm_x_buffer_tag_tracked(&tag, true)); + tag.forkNum = VISIBILITYMAP_FORKNUM; + UT_ASSERT(cluster_pcm_x_buffer_tag_tracked(&tag, true)); + tag.forkNum = FSM_FORKNUM; + UT_ASSERT(!cluster_pcm_x_buffer_tag_tracked(&tag, true)); + UT_ASSERT(!cluster_pcm_x_buffer_tag_tracked(NULL, true)); +} + +UT_TEST(test_pcm_tracking_uses_one_tag_gate_for_acquire_direct_init_and_eviction) +{ + char *source = read_bufmgr_source(); + const char *should_track; + const char *should_track_end; + const char *invalidate_commit; + const char *invalidate_tail; + const char *victim; + const char *victim_end; + const char *direct_init; + const char *direct_init_end; + const char *direct_gate; + const char *arm; + const char *consume; + const char *content_lock; + + /* A saved tag must make exactly the same fork decision as a live + * BufferDesc. A relnumber-only eviction gate would leak an FSM release + * back into a domain from which acquire was excluded. */ + UT_ASSERT_NULL(strstr(source, "cluster_bufmgr_reln_pcm_tracked")); + should_track = strstr(source, "\ncluster_bufmgr_should_pcm_track("); + should_track_end = should_track != NULL ? strstr(should_track, "\n}") : NULL; + invalidate_commit = strstr(source, "\nInvalidateBufferCommitLocked("); + invalidate_tail = strstr(source, "\nInvalidateBufferCommitTailLocked("); + victim = strstr(source, "\nInvalidateVictimBuffer("); + victim_end = strstr(source, "\nstatic Buffer\nGetVictimBuffer("); + UT_ASSERT_NOT_NULL(should_track); + UT_ASSERT_NOT_NULL(should_track_end); + UT_ASSERT_NOT_NULL(invalidate_commit); + UT_ASSERT_NOT_NULL(invalidate_tail); + UT_ASSERT_NOT_NULL(victim); + UT_ASSERT_NOT_NULL(victim_end); + if (should_track != NULL && should_track_end != NULL) + assert_source_range_contains(should_track, should_track_end, + "cluster_pcm_x_buffer_tag_tracked"); + if (invalidate_commit != NULL && invalidate_tail != NULL) + assert_source_range_contains(invalidate_commit, invalidate_tail, + "cluster_pcm_x_buffer_tag_tracked"); + if (victim != NULL && victim_end != NULL) + assert_source_range_contains(victim, victim_end, "cluster_pcm_x_buffer_tag_tracked"); + + /* The dedicated VM/FSM initialization wrapper retains provenance. Its + * common tag gate decides whether PCM proof is armed: FSM falls through to + * the local content lock, while VM still arms and consumes the exact proof. */ + direct_init = strstr(source, "\nLockBufferForAuxiliaryPageInit("); + direct_init_end = strstr(source, "\nvoid\nLockBufferForVisibilityMapPageInit("); + UT_ASSERT_NOT_NULL(direct_init); + UT_ASSERT_NOT_NULL(direct_init_end); + direct_gate + = direct_init != NULL ? strstr(direct_init, "cluster_bufmgr_should_pcm_track(buf)") : NULL; + arm = direct_init != NULL ? strstr(direct_init, "cluster_bufmgr_pcm_arm_direct_init") : NULL; + consume + = direct_init != NULL ? strstr(direct_init, "cluster_bufmgr_pcm_gate_direct_init") : NULL; + content_lock = direct_init != NULL + ? strstr(direct_init, + "LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE)") + : NULL; + UT_ASSERT_NOT_NULL(direct_gate); + UT_ASSERT_NOT_NULL(arm); + UT_ASSERT_NOT_NULL(consume); + UT_ASSERT_NOT_NULL(content_lock); + if (direct_gate != NULL && arm != NULL && consume != NULL && content_lock != NULL) + UT_ASSERT(direct_gate < arm && arm < consume && consume < content_lock + && content_lock < direct_init_end); + + free(source); +} + UT_TEST(test_queue_revoke_retains_main_but_drops_unpinned_vm_fsm) { static const char *const begin_contract[] = { "expected_s->pcm_state != (uint8) PCM_STATE_S", @@ -1015,24 +1419,46 @@ UT_TEST(test_queue_revoke_retains_main_but_drops_unpinned_vm_fsm) "cluster_pcm_own_reservation_abort_exact", "PCM_OWN_FLAG_REVOKING", "UnlockBufHdr" }; - static const char *const finish_contract[] = { "LWLockAcquire(content_lock, LW_EXCLUSIVE)", - "LockBufHdr", - "cluster_pcm_own_gen_get", - "cluster_bufmgr_pcm_current_image_locked", - "BM_IO_IN_PROGRESS", - "PageGetLSN", - "cluster_pcm_own_revoke_retain_commit_exact", - "buf->pcm_state = (uint8) PCM_STATE_N", - "buf->buffer_type = (uint8) BUF_TYPE_PI", - "BM_DIRTY | BM_JUST_DIRTIED", - "BM_CHECKPOINT_NEEDED | BM_IO_ERROR", - "cluster_pcm_own_snapshot_locked" }; + static const char *const finish_contract[] + = { "BufTableHashCode", + "LWLockAcquire(partition_lock, LW_SHARED)", + "BufTableLookup", + "LockBufHdr", + "cluster_pcm_own_snapshot_matches_locked", + "cluster_bufmgr_pcm_current_image_locked", + "BM_IO_IN_PROGRESS", + "cluster_bufmgr_pin_for_gcs_locked", + "LWLockRelease(partition_lock)", + "LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)", + "PG_TRY();", + "cluster_pcm_own_snapshot_matches_locked", + "PageGetLSN", + "FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL)", + "LockBufHdr", + "cluster_pcm_own_snapshot_matches_locked", + "cluster_bufmgr_pcm_current_image_locked", + "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED", + "result = CLUSTER_PCM_OWN_BUSY", + "PageGetLSN", + "cluster_pcm_own_revoke_retain_commit_exact", + "buf->pcm_state = (uint8)PCM_STATE_N", + "buf->buffer_type = (uint8)BUF_TYPE_PI", + "BM_DIRTY | BM_JUST_DIRTIED", + "BM_CHECKPOINT_NEEDED | BM_IO_ERROR", + "cluster_pcm_own_snapshot_locked", + "LWLockRelease(content_lock)", + "cluster_bufmgr_unpin_for_gcs" }; static const char *const drop_contract[] = { "BufMappingPartitionLock", "LWLockAcquire(partition_lock, LW_EXCLUSIVE)", "BufTableLookup", "LockBufHdr", "BUF_STATE_GET_REFCOUNT", + "cluster_pcm_own_flags_get", + "BM_IO_IN_PROGRESS", "CLUSTER_PCM_X_REVOKE_FINISH_BUSY", + "CLUSTER_PCM_OWN_FINISH_REFUSAL_VM_FSM_PINNED", + "CLUSTER_PCM_OWN_FINISH_REFUSAL_IO_IN_PROGRESS", + "CLUSTER_PCM_OWN_FINISH_REFUSAL_LIVE_FLAGS", "PageGetLSN", "cluster_pcm_own_revoke_commit_exact", "buf->pcm_state = (uint8)PCM_STATE_N", @@ -1098,11 +1524,16 @@ UT_TEST(test_queue_revoke_retains_main_but_drops_unpinned_vm_fsm) const char *drop = strstr(finish, "InvalidateBuffer"); const char *legacy_pi = strstr(finish, "cluster_bufmgr_convert_to_pi_locked"); const char *mapping = strstr(finish, "partition_lock"); + const char *conditional + = strstr(finish, "LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)"); UT_ASSERT(refcount == NULL || refcount >= finish_end); UT_ASSERT(drop == NULL || drop >= finish_end); UT_ASSERT(legacy_pi == NULL || legacy_pi >= finish_end); - UT_ASSERT(mapping == NULL || mapping >= finish_end); + UT_ASSERT_NOT_NULL(mapping); + UT_ASSERT_NOT_NULL(conditional); + if (mapping != NULL && conditional != NULL) + UT_ASSERT(mapping < conditional); } free(source); } @@ -1124,7 +1555,7 @@ UT_TEST(test_retained_image_release_and_writeback_gates_are_exact) "LockBufHdr", "PCM_OWN_FLAG_REVOKING", "LWLockRelease(partition_lock)", - "LWLockAcquire(content_lock, LW_EXCLUSIVE)", + "LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)", "LockBufHdr", "BufferTagsEqual", "BM_VALID", @@ -1161,7 +1592,7 @@ UT_TEST(test_retained_image_release_and_writeback_gates_are_exact) flush = strstr(source, "\nFlushBuffer("); dirty = strstr(source, "\nMarkBufferDirty(Buffer buffer)"); hint = strstr(source, "\nMarkBufferDirtyHint(Buffer buffer, bool buffer_std)"); - lockbuffer = strstr(source, "\nLockBuffer(Buffer buffer, int mode)"); + lockbuffer = strstr(source, "\nLockBufferInternal(Buffer buffer, int mode"); conditional = strstr(source, "\nConditionalLockBuffer(Buffer buffer)"); resident_stamp = strstr(source, "\ncluster_bufmgr_lock_resident_for_stamp("); storage_refresh = strstr(source, "\ncluster_bufmgr_refresh_block_from_storage_for_gcs("); @@ -1202,19 +1633,33 @@ UT_TEST(test_retained_image_release_and_writeback_gates_are_exact) if (dirty != NULL) UT_ASSERT(strstr(dirty, "cluster_bufmgr_pcm_x_retained_image_locked") < strstr(dirty, "buf_state |= BM_DIRTY")); - if (hint != NULL) - UT_ASSERT(strstr(hint, "cluster_bufmgr_pcm_x_retained_image_locked") - < strstr(hint, "BM_DIRTY | BM_JUST_DIRTIED")); + if (hint != NULL) { + const char *tracked = strstr(hint, "cluster_bufmgr_should_pcm_track(bufHdr)"); + const char *current = strstr(hint, "cluster_bufmgr_pcm_current_image_locked"); + const char *refuse = current != NULL ? strstr(current, "return;") : NULL; + const char *dirty_flags = strstr(hint, "BM_DIRTY | BM_JUST_DIRTIED"); + + /* Hint dirt is optional. A kept pinned N/PI mirror may have its + * in-memory hint byte touched, but without live S/X current authority it + * must not regain writeback eligibility and block a later storage refresh. */ + UT_ASSERT_NOT_NULL(tracked); + UT_ASSERT_NOT_NULL(current); + UT_ASSERT_NOT_NULL(refuse); + UT_ASSERT_NOT_NULL(dirty_flags); + if (tracked != NULL && current != NULL && refuse != NULL && dirty_flags != NULL) + UT_ASSERT(tracked < current && current < refuse && refuse < dirty_flags); + } if (lockbuffer != NULL) { - const char *reserve = strstr(lockbuffer, "cluster_pcm_own_begin_grant_reservation("); + const char *reserve + = strstr(lockbuffer, "cluster_bufmgr_pcm_begin_grant_reservation_wait("); const char *content = strstr(lockbuffer, "LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED)"); - const char *w1_revoke - = strstr(lockbuffer, "PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING"); + const char *w1_reverify + = strstr(lockbuffer, "cluster_pcm_x_cached_cover_reverify_accepts("); UT_ASSERT_NOT_NULL(reserve); UT_ASSERT_NOT_NULL(content); - UT_ASSERT_NOT_NULL(w1_revoke); + UT_ASSERT_NOT_NULL(w1_reverify); if (reserve != NULL && content != NULL) UT_ASSERT(reserve < content); } @@ -1257,12 +1702,9 @@ UT_TEST(test_retained_image_release_and_writeback_gates_are_exact) UT_TEST(test_retained_drain_retags_invalid_only_after_exact_token_release) { - static const char *const drain_contract[] = { "cluster_pcm_own_revoke_retain_release_exact", - "result == CLUSTER_PCM_OWN_OK", - "buf_state &= ~BM_VALID", - "buf->buffer_type", - "BUF_TYPE_CURRENT", - "UnlockBufHdr" }; + static const char *const drain_contract[] + = { "cluster_pcm_own_revoke_retain_release_exact", "result == CLUSTER_PCM_OWN_OK", + "cluster_pcm_x_retained_release_retag", "&buf->buffer_type", "UnlockBufHdr" }; char *source = read_bufmgr_source(); const char *release; const char *release_end; @@ -1315,10 +1757,10 @@ UT_TEST(test_lockbuffer_pcm_x_holder_ledger_brackets_both_content_acquires) "LWLockRelease(BufferDescriptorGetContentLock(buf))", "cluster_bufmgr_pcm_x_holder_unregister" }; static const char *const acquire_contract[] - = { "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf)", + = { "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf,", "LWLockAcquire(BufferDescriptorGetContentLock(buf)", "LWLockRelease(BufferDescriptorGetContentLock(buf))", - "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf)", + "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf,", "LWLockAcquire(BufferDescriptorGetContentLock(buf)", "cluster_bufmgr_pcm_x_holder_activate(pcm_x_holder)" }; char *source = read_bufmgr_source(); @@ -1327,11 +1769,11 @@ UT_TEST(test_lockbuffer_pcm_x_holder_ledger_brackets_both_content_acquires) * published across the W1 release/reacquire fallback. Normal release * publishes RELEASING before unlocking and unlinks only afterwards. */ assert_ordered_in_function( - source, "\nLockBuffer(Buffer buffer, int mode)", + source, "\nLockBufferInternal(Buffer buffer, int mode", "\n/*\n * Acquire the content_lock for the buffer, but only if we don't have to wait.", unlock_contract, lengthof(unlock_contract)); assert_ordered_in_function( - source, "\nLockBuffer(Buffer buffer, int mode)", + source, "\nLockBufferInternal(Buffer buffer, int mode", "\n/*\n * Acquire the content_lock for the buffer, but only if we don't have to wait.", acquire_contract, lengthof(acquire_contract)); free(source); @@ -1458,6 +1900,10 @@ UT_TEST(test_bufmgr_pcm_x_holder_gate_retry_is_bounded_outside_content_lock) CLUSTER_PCM_X_HOLDER_RETRY_WAIT); UT_ASSERT_EQ(cluster_pcm_x_holder_register_retry_action(PCM_X_QUEUE_NOT_READY, false), CLUSTER_PCM_X_HOLDER_RETRY_FAIL); + UT_ASSERT_EQ(cluster_pcm_x_holder_register_retry_action(PCM_X_QUEUE_BUSY, true), + CLUSTER_PCM_X_HOLDER_RETRY_WAIT); + UT_ASSERT_EQ(cluster_pcm_x_holder_register_retry_action(PCM_X_QUEUE_BUSY, false), + CLUSTER_PCM_X_HOLDER_RETRY_FAIL); UT_ASSERT_EQ(cluster_pcm_x_holder_register_retry_action(PCM_X_QUEUE_NO_CAPACITY, true), CLUSTER_PCM_X_HOLDER_RETRY_FAIL); for (i = 0; i < CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS; i++) { @@ -1633,12 +2079,14 @@ UT_TEST(test_queue_passive_pinned_s_release_serializes_bytes_and_ownership) "cluster_pcm_x_revoke_finish_mode(tag, shared_refcount)", "cluster_bufmgr_pin_for_gcs_locked", "LWLockRelease(partition_lock)", - "LWLockAcquire(content_lock, LW_EXCLUSIVE)", + "LWLockConditionalAcquire(content_lock, LW_EXCLUSIVE)", "cluster_pcm_own_snapshot_matches_locked", "PageGetLSN", - "XLogFlush", + "FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL)", "LockBufHdr", "cluster_pcm_own_snapshot_matches_locked", + "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED", + "result = CLUSTER_PCM_OWN_BUSY", "cluster_pcm_own_bump_locked", "buf->pcm_state = (uint8) PCM_STATE_N", "buf->buffer_type = (uint8) BUF_TYPE_PI", @@ -1683,24 +2131,29 @@ UT_TEST(test_queue_passive_n_mirror_is_never_gcs_ship_authority) { static const char *const probe_contract[] = { "LockBufHdr", "cluster_bufmgr_pcm_current_image_locked", "UnlockBufHdr" }; - static const char *const copy_contract[] = { "LockBufHdr", - "cluster_bufmgr_pcm_current_image_locked", - "cluster_bufmgr_pin_for_gcs_locked", - "LWLockAcquire(content_lock, LW_SHARED)", - "cluster_bufmgr_pcm_current_image_locked", - "memcpy(dst, page, BLCKSZ)" }; - static const char *const live_sge_contract[] = { "LockBufHdr", - "cluster_bufmgr_pcm_current_image_locked", - "cluster_bufmgr_pin_for_gcs_locked", - "LWLockAcquire(content_lock, LW_SHARED)", - "cluster_bufmgr_pcm_current_image_locked", - "*out_page_addr = page" }; - static const char *const smart_contract[] = { "LockBufHdr", - "cluster_bufmgr_pcm_current_image_locked", - "cluster_bufmgr_pin_for_gcs_locked", - "LWLockAcquire(content_lock, LW_SHARED)", - "cluster_bufmgr_pcm_current_image_locked", - "memcpy(dst, page, BLCKSZ)" }; + static const char *const copy_contract[] + = { "LockBufHdr", + "cluster_bufmgr_pcm_current_image_locked", + "cluster_bufmgr_pin_for_gcs_locked", + "LWLockConditionalAcquire(content_lock, LW_SHARED)", + "cluster_bufmgr_pcm_current_image_locked", + "FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL)", + "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED", + "memcpy(dst, page, BLCKSZ)" }; + static const char *const live_sge_contract[] + = { "LockBufHdr", + "cluster_bufmgr_pcm_current_image_locked", + "cluster_bufmgr_pin_for_gcs_locked", + "LWLockConditionalAcquire(content_lock, LW_SHARED)", + "cluster_bufmgr_pcm_current_image_locked", + "*out_page_addr = page" }; + static const char *const smart_contract[] + = { "LockBufHdr", + "cluster_bufmgr_pcm_current_image_locked", + "cluster_bufmgr_pin_for_gcs_locked", + "LWLockConditionalAcquire(content_lock, LW_SHARED)", + "cluster_bufmgr_pcm_current_image_locked", + "memcpy(dst, page, BLCKSZ)" }; char *source = read_bufmgr_source(); assert_ordered_in_function(source, "\ncluster_bufmgr_probe_block_for_gcs(", @@ -1718,6 +2171,38 @@ UT_TEST(test_queue_passive_n_mirror_is_never_gcs_ship_authority) free(source); } +UT_TEST(test_gcs_ship_copy_reports_exact_nonblocking_refusal_stage) +{ + static const char *const refusal_contract[] + = { "CLUSTER_BUFMGR_GCS_COPY_REFUSAL_NOT_RESIDENT", + "CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CURRENT_INVALID", + "CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_FIRST", + "CLUSTER_BUFMGR_GCS_COPY_REFUSAL_CONTENT_LOCK_SECOND", + "CLUSTER_BUFMGR_GCS_COPY_REFUSAL_HC89_LSN_DRIFT" }; + char *source = read_bufmgr_source(); + const char *copy + = source != NULL ? strstr(source, "\ncluster_bufmgr_copy_block_for_gcs(") : NULL; + const char *copy_end + = copy != NULL ? strstr(copy, "\n/*\n * Borrow a live shared_buffers page") : NULL; + int i; + + /* P0-21 observation contract: every nonblocking false return that can + * become holder-side DENIED_MASTER_NOT_HOLDER identifies the precise + * residency/current-image/content-lock/HC89 refusal stage. */ + UT_ASSERT_NOT_NULL(copy); + UT_ASSERT_NOT_NULL(copy_end); + if (copy != NULL && copy_end != NULL) { + for (i = 0; i < lengthof(refusal_contract); i++) { + const char *hit = strstr(copy, refusal_contract[i]); + + UT_ASSERT_NOT_NULL(hit); + if (hit != NULL) + UT_ASSERT(hit < copy_end); + } + } + free(source); +} + UT_TEST(test_queue_installed_image_publication_is_exact_and_content_locked) { typedef ClusterPcmOwnResult (*PublishImageFn)(BufferDesc *, const ClusterPcmOwnSnapshot *, @@ -1752,12 +2237,12 @@ UT_TEST(test_queue_self_source_handoff_is_single_lifecycle_and_readonly_drain) "cluster_pcm_own_revoke_to_grant_handoff_exact", "UnlockBufHdr" }; static const char *const drain_proof_contract[] = { "BufMappingPartitionLock", + "BufTableLookup", + "CLUSTER_PCM_OWN_OK", "LockBufHdr", - "source_generation + 1", - "flags != 0", - "PCM_STATE_X", - "BUF_TYPE_XCUR", - "UnlockBufHdr" }; + "cluster_pcm_own_classify_live_flags", + "UnlockBufHdr", + "CLUSTER_PCM_OWN_CORRUPT" }; char *source = read_bufmgr_source(); const char *handoff; const char *handoff_end; @@ -1781,7 +2266,7 @@ UT_TEST(test_queue_self_source_handoff_is_single_lifecycle_and_readonly_drain) forbidden = strstr(handoff, "cluster_pcm_own_reservation_abort_exact("); UT_ASSERT(forbidden == NULL || forbidden >= handoff_end); } - assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_self_handoff_x_exact(", + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_self_handoff_probe(", "\n/* ==========", drain_proof_contract, lengthof(drain_proof_contract)); free(source); @@ -1822,6 +2307,19 @@ UT_TEST(test_queue_writer_grant_snapshot_is_claim_and_generation_exact) live.pcm_state = (uint8)PCM_STATE_S; UT_ASSERT(!cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); live = granted; + /* A' rebase: a published effective grant base supersedes the enqueue-time + * identity base in the exact granted-generation binding. */ + claim.grant_base_own_generation = 14; + UT_ASSERT(!cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); + granted.generation = 15; + live = granted; + UT_ASSERT(cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); + claim.grant_base_own_generation = UINT64_MAX; + UT_ASSERT(!cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); + claim.grant_base_own_generation = 0; + granted.generation = 11; + live = granted; + UT_ASSERT(cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); claim.active_slot.slot_generation++; UT_ASSERT(!cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); @@ -1859,8 +2357,8 @@ UT_TEST(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_aut "cluster_pcm_lock_unlock_content_buffer(buf, old_mode)" }; static const char *const acquire_contract[] = { "pcm_covered = cluster_pcm_x_cached_cover_bypasses_queue(", - "pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode)", - "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf)", + "pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode,", + "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf,", "LWLockAcquire(BufferDescriptorGetContentLock(buf)", "cluster_bufmgr_pcm_x_writer_activate(pcm_x_writer)" }; static const char *const cleanup_contract[] @@ -1874,7 +2372,7 @@ UT_TEST(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_aut "if (!writer_retry && !holder_retry)", "pg_usleep(1000L)" }; static const char *const snapshot_failure_contract[] = { "release_result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim)", - "cluster_bufmgr_pcm_x_writer_clear(entry)", + "entry->claim_cleanup_complete = true", "entry->phase = PCM_X_WRITER_LEDGER_DEFERRED", "cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_CORRUPT, buf" }; static const char *const writer_holder_publish_contract[] @@ -1892,7 +2390,8 @@ UT_TEST(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_aut "cluster_gcs_pcm_x_writer_claim_release_and_wake_exact(&entry->claim)" }; static const char *const deferred_cleanup_contract[] = { "result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim)", - "cluster_bufmgr_pcm_x_writer_clear(entry)" }; + "entry->claim_cleanup_complete = true", + "cluster_bufmgr_pcm_x_writer_finish_claim_cleanup(entry" }; static const char *const abort_cleanup_contract[] = { "result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim)", "entry->phase = PCM_X_WRITER_LEDGER_DEFERRED" }; @@ -1964,11 +2463,11 @@ UT_TEST(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_aut writer_holder_runtime_contract, lengthof(writer_holder_runtime_contract)); assert_ordered_in_function( - source, "\nLockBuffer(Buffer buffer, int mode)", + source, "\nLockBufferInternal(Buffer buffer, int mode", "\n/*\n * Acquire the content_lock for the buffer, but only if we don't have to wait.", acquire_contract, lengthof(acquire_contract)); assert_ordered_in_function( - source, "\nLockBuffer(Buffer buffer, int mode)", + source, "\nLockBufferInternal(Buffer buffer, int mode", "\n/*\n * Acquire the content_lock for the buffer, but only if we don't have to wait.", unlock_contract, lengthof(unlock_contract)); assert_ordered_in_function(source, "\nUnlockBuffers(void)", @@ -1994,16 +2493,262 @@ UT_TEST(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_aut free(source); } +/* review P1-4: deterministic execution of the preflight transient-BUSY arm + * over the REAL ownership object. A live GRANT_PENDING (then REVOKING) + * lifecycle preflights BUSY and the requester site classifies WAIT -- never + * fail-closed; once the lifecycle clears, the SAME identity's re-snapshot + * preflights OK and the real grant reservation begins on the idle tuple. + * Formation/session drift observed across the wait fails the exact runtime + * proof, so the driver exits before touching the reservation. */ +UT_TEST(test_preflight_busy_waits_then_clean_resnapshot_begins_reservation) +{ + ClusterPcmOwnSnapshot live; + PcmXWaitIdentity identity; + PcmXRuntimeSnapshot current; + PcmXRuntimeSnapshot start; + uint64 blocker_token = 0; + uint64 token = 0; + + reset_fixture(); + memset(&identity, 0, sizeof(identity)); + identity.tag.relNumber = 20001; + identity.tag.blockNum = 3; + identity.base_own_generation = 0; + memset(&live, 0, sizeof(live)); + live.tag = identity.tag; + live.pcm_state = (uint8)PCM_STATE_N; + + /* A concurrent grant lifecycle holds GRANT_PENDING: BUSY -> WAIT. */ + UT_ASSERT_EQ( + cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_GRANT_PENDING, &blocker_token), + CLUSTER_PCM_OWN_OK); + live.generation = pg_atomic_read_u64(&ClusterPcmOwnArray[0].generation); + live.reservation_token = pg_atomic_read_u64(&ClusterPcmOwnArray[0].reservation_token); + live.flags = pg_atomic_read_u32(&ClusterPcmOwnArray[0].flags); + UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, PCM_X_QUEUE_BUSY), + GCS_BLOCK_PCM_X_RETRY_WAIT); + UT_ASSERT_EQ( + cluster_pcm_own_reservation_abort_exact(0, 0, blocker_token, PCM_OWN_FLAG_GRANT_PENDING), + CLUSTER_PCM_OWN_OK); + + /* A live revoke lifecycle classifies exactly the same way. */ + UT_ASSERT_EQ( + cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_REVOKING, &blocker_token), + CLUSTER_PCM_OWN_OK); + live.reservation_token = pg_atomic_read_u64(&ClusterPcmOwnArray[0].reservation_token); + live.flags = pg_atomic_read_u32(&ClusterPcmOwnArray[0].flags); + UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_RESERVATION_PREFLIGHT, PCM_X_QUEUE_BUSY), + GCS_BLOCK_PCM_X_RETRY_WAIT); + UT_ASSERT_EQ( + cluster_pcm_own_reservation_abort_exact(0, 0, blocker_token, PCM_OWN_FLAG_REVOKING), + CLUSTER_PCM_OWN_OK); + + /* The wait ends: the clean-N re-snapshot (idle nonzero token is legal) + * preflights OK and the REAL reservation begins on the same tuple. */ + live.reservation_token = pg_atomic_read_u64(&ClusterPcmOwnArray[0].reservation_token); + live.flags = pg_atomic_read_u32(&ClusterPcmOwnArray[0].flags); + UT_ASSERT_EQ(live.flags, (uint32)0); + UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_GRANT_PENDING, &token), + CLUSTER_PCM_OWN_OK); + UT_ASSERT(token > blocker_token); + assert_entry(0, token, PCM_OWN_FLAG_GRANT_PENDING); + + /* Formation/session drift across the wait refuses the re-entry. */ + memset(&start, 0, sizeof(start)); + start.state = PCM_X_RUNTIME_ACTIVE; + start.gate_generation = 3; + start.master_session_incarnation = 9; + current = start; + UT_ASSERT(cluster_gcs_pcm_x_requester_runtime_exact(&start, ¤t)); + current.gate_generation = 4; + UT_ASSERT(!cluster_gcs_pcm_x_requester_runtime_exact(&start, ¤t)); + current = start; + current.master_session_incarnation = 10; + UT_ASSERT(!cluster_gcs_pcm_x_requester_runtime_exact(&start, ¤t)); + current = start; + current.state = PCM_X_RUNTIME_RECOVERY_BLOCKED; + UT_ASSERT(!cluster_gcs_pcm_x_requester_runtime_exact(&start, ¤t)); +} + +UT_TEST(test_own_lifecycle_counters_land_on_exact_begin_and_x_commit) +{ + static const char *const exact_begin_contract[] + = { "cluster_pcm_own_reservation_begin_exact", "PCM_OWN_FLAG_GRANT_PENDING", "UnlockBufHdr", + "result == CLUSTER_PCM_OWN_OK", "cluster_pcm_x_stats_note_own_begin();" }; + static const char *const handoff_begin_contract[] + = { "flags == PCM_OWN_FLAG_GRANT_PENDING", + "result = CLUSTER_PCM_OWN_OK", + "cluster_pcm_own_revoke_to_grant_handoff_exact", + "handoff_transitioned = result == CLUSTER_PCM_OWN_OK", + "UnlockBufHdr", + "if (handoff_transitioned)", + "cluster_pcm_x_stats_note_own_begin();" }; + static const char *const direct_init_begin_contract[] + = { "cluster_pcm_own_reservation_begin_exact", "PCM_OWN_FLAG_GRANT_PENDING", "UnlockBufHdr", + "pending_result == CLUSTER_PCM_OWN_OK", "cluster_pcm_x_stats_note_own_begin();" }; + static const char *const x_commit_contract[] + = { "cluster_pcm_own_grant_commit_exact", "UnlockBufHdr", + "result == CLUSTER_PCM_OWN_OK && new_pcm_state == (uint8)PCM_STATE_X", + "cluster_pcm_x_stats_note_own_commit();" }; + char *source = read_bufmgr_source(); + + /* t/400 own-lifecycle counters: every counted begin sits after the header + * spinlock proved an exact GRANT_PENDING linearization (a fresh token, a + * REVOKING->GRANT_PENDING handoff, or a consumed direct-init proof), and + * a replayed/refused begin never counts. The idempotent duplicate + * PREPARE that merely observes GRANT_PENDING must stay uncounted, so the + * handoff increments only behind its transition evidence. */ + assert_ordered_in_function(source, "\ncluster_pcm_own_begin_grant_reservation(", + "\ncluster_bufmgr_pcm_own_begin_x_reservation(", + exact_begin_contract, lengthof(exact_begin_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_begin_x_reservation(", + "\ncluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation(", + exact_begin_contract, lengthof(exact_begin_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation(", + "\ncluster_bufmgr_pcm_own_handoff_s_revoke_to_x_reservation(", + handoff_begin_contract, lengthof(handoff_begin_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_gate_direct_init(", + "#define BUF_DROP_FULL_SCAN_THRESHOLD", direct_init_begin_contract, + lengthof(direct_init_begin_contract)); + + /* The X ownership commit counts exactly once, at the single exact-commit + * funnel shared by the queue finish, the LockBuffer finalize, and the + * direct-init grant; an S-grant finish and every refusal stay silent. */ + assert_ordered_in_function(source, "\ncluster_pcm_own_finish_grant_reservation(", + "\ncluster_bufmgr_pcm_own_finish_x_commit(", x_commit_contract, + lengthof(x_commit_contract)); + + /* Pin the production wiring against silent duplication or removal: four + * begin sites, one commit funnel, no counter call under LockBufHdr. */ + UT_ASSERT_EQ(count_occurrences(source, "cluster_pcm_x_stats_note_own_begin();"), 4); + UT_ASSERT_EQ(count_occurrences(source, "cluster_pcm_x_stats_note_own_commit();"), 1); + free(source); +} + +UT_TEST(test_pcm_x_retain_flush_error_injection_is_exact_and_pre_write) +{ + static const char *const finish_flush_contract[] + = { "cluster_injection_is_armed(\"cluster-pcm-x-retain-flush-error\")", + "buf_state |= BM_DIRTY | BM_JUST_DIRTIED", + "needs_flush =", + "cluster_pcm_x_finish_retain_flush_active = true", + "FlushBuffer(buf, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL)", + "cluster_pcm_x_finish_retain_flush_active = false", + "cluster PCM-X retained-image finish FlushBuffer succeeded" }; + static const char *const flush_error_contract[] + = { "if (!StartBufferIO(buf, false))", + "cluster_pcm_x_finish_retain_flush_io_active = true", + "cluster_pcm_x_finish_retain_flush_active", + "cluster_pcm_own_flags_get(buf->buf_id) == PCM_OWN_FLAG_REVOKING", + "CLUSTER_INJECTION_POINT(\"cluster-pcm-x-retain-flush-error\")", + "cluster_injection_should_skip(\"cluster-pcm-x-retain-flush-error\")", + "errmsg(\"injected PCM-X retained-image FlushBuffer failure\")", + "smgrwrite(", + "TerminateBufferIO(buf, true, 0)", + "cluster_pcm_x_finish_retain_flush_io_active = false" }; + static const char *const catch_contract[] + = { "PG_CATCH();", + "cluster_pcm_x_finish_retain_flush_active = false", + "if (cluster_pcm_x_finish_retain_flush_error_context_pushed)", + "error_context_stack = cluster_pcm_x_finish_retain_flush_error_context_previous", + "if (content_locked && LWLockHeldByMe(content_lock))", + "HOLD_INTERRUPTS();", + "LWLockRelease(content_lock)", + "if (cluster_pcm_x_finish_retain_flush_io_active)", + "cluster_pcm_x_finish_retain_flush_io_active = false", + "AbortBufferIO(BufferDescriptorGetBuffer(buf))", + "cluster_bufmgr_unpin_for_gcs(buf)", + "PG_RE_THROW();" }; + char *source = read_bufmgr_source(); + const char *finish; + const char *catch; + const char *rethrow; + const char *resume; + + /* The point is armed only at the finish-revoke-retain seam. Test arming + * makes an otherwise already-flushed copy dirty without changing bytes, + * so the caller-pin/content-EXCLUSIVE FlushBuffer leg is deterministic. + * The generic flush path dispatches the point only while that exact call + * is active and the ownership token is still REVOKING. ERROR clears the + * process interrupt holdoff count before longjmp, so the cleanup must + * restore FlushBuffer's stack-local error callback, release content + * authority with a replacement hold, abort the exact ResourceOwner-tracked + * BufferIO while its raw pin is still live, and only then unpin. Otherwise + * an aux worker that absorbs the ERROR reaches commit-style resource-owner + * cleanup and PANICs with "lost track of buffer IO". LWLockRelease consumes + * the replacement hold itself; a second RESUME would underflow in cassert + * builds and an unconditional HOLD would leak on the no-lock path. */ + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_finish_revoke_retain(", + "\ncluster_bufmgr_pcm_own_release_retained_image(", + finish_flush_contract, lengthof(finish_flush_contract)); + assert_ordered_in_function(source, "\nFlushBuffer(", "\n/*\n * RelationGetNumberOfBlocksInFork", + flush_error_contract, lengthof(flush_error_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_finish_revoke_retain(", + "\ncluster_bufmgr_pcm_own_release_retained_image(", catch_contract, + lengthof(catch_contract)); + finish = strstr(source, "\ncluster_bufmgr_pcm_own_finish_revoke_retain("); + UT_ASSERT_NOT_NULL(finish); + catch = strstr(finish, "PG_CATCH();"); + UT_ASSERT_NOT_NULL(catch); + rethrow = strstr(catch, "PG_RE_THROW();"); + UT_ASSERT_NOT_NULL(rethrow); + resume = strstr(catch, "RESUME_INTERRUPTS();"); + UT_ASSERT(resume == NULL || resume > rethrow); + free(source); +} + +UT_TEST(test_writer_activation_diagnostic_covers_commit_clear_and_unguarded_n_boundaries) +{ + static const char *const commit_contract[] + = { "cluster_pcm_own_writer_grant_commit_exact(", "buf->pcm_state = new_pcm_state", + "cluster_pcm_own_snapshot_locked(buf, &activation_diag)", + "UnlockBufHdr(buf, buf_state)", + "cluster_pcm_own_activation_diag_emit(\"writer-commit\"" }; + static const char *const clear_contract[] + = { "cluster_pcm_own_snapshot_locked(buf, &live)", + "cluster_pcm_own_writer_activation_clear_exact(", "UnlockBufHdr(buf, buf_state)", + "cluster_pcm_own_activation_diag_emit(\"writer-activation-clear\"" }; + char *source = read_bufmgr_source(); + + UT_ASSERT_EQ(sizeof(ClusterPcmOwnSnapshot), 56); + UT_ASSERT_NOT_NULL(strstr(source, "out->writer_activation_token")); + assert_ordered_in_function(source, "\ncluster_pcm_own_finish_grant_reservation(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_finish_x_commit(", + commit_contract, lengthof(commit_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_writer_activation_clear(", + "\nstatic bool\ncluster_bufmgr_pcm_x_writer_finish_claim_cleanup(", + clear_contract, lengthof(clear_contract)); + UT_ASSERT_NOT_NULL(strstr(source, "\"invalidate-stage-n-pi\"")); + UT_ASSERT_NOT_NULL(strstr(source, "\"invalidate-stage-n-drop\"")); + UT_ASSERT_NOT_NULL(strstr(source, "\"drop-no-wire-stage-n-pi\"")); + UT_ASSERT_NOT_NULL(strstr(source, "\"drop-no-wire-stage-n-drop\"")); + UT_ASSERT_NOT_NULL(strstr(source, "\"discard-pi-stage-n\"")); + free(source); +} + int main(void) { - UT_PLAN(45); + UT_PLAN(60); UT_RUN(test_shmem_initializes_complete_entry); + UT_RUN(test_writer_activation_fence_blocks_revoke_until_exact_clear); UT_RUN(test_begin_abort_is_exact_and_monotonic); UT_RUN(test_invalid_live_flag_shapes_are_corrupt_not_busy); UT_RUN(test_grant_commit_is_exact_and_bumps_once); UT_RUN(test_s_revoke_handoff_reuses_exact_token_and_bumps_once); UT_RUN(test_revoke_handoff_kinds_cover_n_s_x_with_one_lifecycle); + UT_RUN(test_s_new_fresh_token_finish_shape_stays_invalid); + UT_RUN(test_share_cover_reverify_accepts_stable_successor_grant); + UT_RUN(test_retained_release_retag_respects_pin_contract); + UT_RUN(test_retained_release_and_finish_never_cover_invalid_bytes); + UT_RUN(test_legacy_byte_proof_republishes_kept_pi_mirror); UT_RUN(test_revoke_commit_is_exact_and_classifies_live_races); UT_RUN(test_revoke_retain_commit_keeps_exact_token_until_release); UT_RUN(test_revoke_commit_exhaustion_is_side_effect_free); @@ -2017,6 +2762,7 @@ main(void) UT_RUN(test_lockbuffer_content_error_uses_post_master_rollback_contract); UT_RUN(test_bufmgr_generation_bump_failure_is_classified_under_header_lock); UT_RUN(test_lockbuffer_reservation_failures_use_busy_corrupt_classifier); + UT_RUN(test_pending_x_denied_retry_leaves_master_invalidate_gap); UT_RUN(test_bufmgr_finish_rejects_invalid_state_and_initializes_acquire_result); UT_RUN(test_bufmgr_finish_and_abort_gate_on_exact_base_state); UT_RUN(test_d5a_release_error_keeps_descriptor_out_of_freelist); @@ -2024,7 +2770,10 @@ main(void) UT_RUN(test_queue_contract_exposes_prepare_only_begin_api); UT_RUN(test_queue_contract_exposes_opaque_retained_revoke_api); UT_RUN(test_queue_n_source_refresh_is_exact_and_publishes_only_complete_image); + UT_RUN(test_queue_s_source_dirty_flush_makes_progress_and_reports_exact_refusal); UT_RUN(test_revoke_finish_mode_rejects_pinned_vm_fsm_and_retains_main); + UT_RUN(test_pcm_tracking_excludes_only_fsm_for_user_and_shared_catalog_relations); + UT_RUN(test_pcm_tracking_uses_one_tag_gate_for_acquire_direct_init_and_eviction); UT_RUN(test_queue_revoke_retains_main_but_drops_unpinned_vm_fsm); UT_RUN(test_retained_image_release_and_writeback_gates_are_exact); UT_RUN(test_retained_drain_retags_invalid_only_after_exact_token_release); @@ -2041,8 +2790,13 @@ main(void) UT_RUN(test_queue_installed_image_publication_is_exact_and_content_locked); UT_RUN(test_queue_self_source_handoff_is_single_lifecycle_and_readonly_drain); UT_RUN(test_queue_passive_n_mirror_is_never_gcs_ship_authority); + UT_RUN(test_gcs_ship_copy_reports_exact_nonblocking_refusal_stage); UT_RUN(test_queue_writer_grant_snapshot_is_claim_and_generation_exact); UT_RUN(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_authority); + UT_RUN(test_preflight_busy_waits_then_clean_resnapshot_begins_reservation); + UT_RUN(test_own_lifecycle_counters_land_on_exact_begin_and_x_commit); + UT_RUN(test_pcm_x_retain_flush_error_injection_is_exact_and_pre_write); + UT_RUN(test_writer_activation_diagnostic_covers_commit_clear_and_unguarded_n_boundaries); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index 0b2e15d2dd..805b44f99c 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -25,6 +25,7 @@ #include #include "cluster/cluster_ic_envelope.h" +#include "cluster/cluster_pcm_x_bufmgr.h" #include "cluster/cluster_pcm_x_convert.h" #include "cluster/cluster_shmem.h" #include "storage/buf_internals.h" @@ -111,6 +112,11 @@ static bool grant_interlock_armed; static LWLock *grant_interlock_lock; static PcmXMasterTicketSlot *grant_interlock_ticket; static uint64 grant_interlock_generation; +static bool drive_terminal_interlock_armed; +static LWLock *drive_terminal_interlock_lock; +static PcmXMasterTagSlot *drive_terminal_interlock_tag; +static PcmXMasterTicketSlot *drive_terminal_interlock_ticket; +static bool drive_terminal_interlock_reclaim; static LWLock *local_generation_interlock_lock; static PcmXSlotHeader *local_generation_interlock_slot; static LWLock *holder_abort_interlock_lock; @@ -150,6 +156,7 @@ static int lock_acquire_during_iteration_count; static void maybe_publish_staged_prehandle_insert_exists(void); static void maybe_inject_local_rekey_insert_failure(void); static uint64 test_slot_generation(PcmXSlotHeader *slot); +static void test_set_slot_generation(PcmXSlotHeader *slot, uint64 generation); static uint32 test_slot_state(PcmXSlotHeader *slot); static void test_set_slot_state(PcmXSlotHeader *slot, uint32 state); static uint32 test_slot_flags(PcmXSlotHeader *slot); @@ -250,6 +257,10 @@ cluster_shmem_register_region(const ClusterShmemRegion *region) registered_region_count++; } +/* Referenced by the fail-closed logging guard in cluster_pcm_x_convert.c; + * unit tests never run inside a critical section. */ +volatile uint32 CritSectionCount = 0; + bool errstart(int elevel, const char *domain pg_attribute_unused()) { @@ -379,6 +390,48 @@ LWLockAcquire(LWLock *lock, LWLockMode mode) UT_ASSERT_NOT_NULL(grant_interlock_ticket); grant_interlock_ticket->ref.grant_generation = grant_interlock_generation; } + if (lock->tranche == LWTRANCHE_CLUSTER_PCM_X_MASTER && drive_terminal_interlock_armed + && lock == drive_terminal_interlock_lock && mode == LW_EXCLUSIVE) { + uint32 node_bit; + + drive_terminal_interlock_armed = false; + UT_ASSERT_NOT_NULL(drive_terminal_interlock_tag); + UT_ASSERT_NOT_NULL(drive_terminal_interlock_ticket); + node_bit = UINT32_C(1) << drive_terminal_interlock_ticket->ref.identity.node_id; + /* Model an exact FINAL_ACK/FINAL_CONFIRM actor that wins after the + * retry pump's allocator lookup but before its master-domain lock. + * Terminal state is published last, as in final_confirm_exact(). */ + drive_terminal_interlock_tag->head_index = PCM_X_INVALID_SLOT_INDEX; + drive_terminal_interlock_tag->tail_index = PCM_X_INVALID_SLOT_INDEX; + drive_terminal_interlock_tag->active_index = PCM_X_INVALID_SLOT_INDEX; + drive_terminal_interlock_tag->active_slot_generation = 0; + drive_terminal_interlock_tag->queue_state_sequence++; + (void)pg_atomic_fetch_and_u32(&drive_terminal_interlock_tag->queued_node_bitmap, + ~node_bit); + drive_terminal_interlock_ticket->next_index = PCM_X_INVALID_SLOT_INDEX; + drive_terminal_interlock_ticket->prev_index = PCM_X_INVALID_SLOT_INDEX; + drive_terminal_interlock_ticket->reliable.retry_deadline_ms = 0; + drive_terminal_interlock_ticket->reliable.expected_responder_session = 0; + drive_terminal_interlock_ticket->reliable.retry_count = 0; + drive_terminal_interlock_ticket->reliable.last_responder_node + = (uint32)drive_terminal_interlock_ticket->ref.identity.node_id; + drive_terminal_interlock_ticket->reliable.expected_responder_node = 0; + drive_terminal_interlock_ticket->reliable.pending_opcode = 0; + drive_terminal_interlock_ticket->reliable.last_response_opcode + = PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM; + drive_terminal_interlock_ticket->reliable.phase = 0; + drive_terminal_interlock_ticket->reliable.flags = 0; + drive_terminal_interlock_ticket->reliable.reserved = 0; + drive_terminal_interlock_ticket->reliable.response_tombstone_mask + |= PCM_X_RESPONSE_TOMBSTONE_COMPLETE; + test_set_slot_state(&drive_terminal_interlock_ticket->slot, PCM_XT_COMPLETE); + if (drive_terminal_interlock_reclaim) { + test_set_slot_generation( + &drive_terminal_interlock_ticket->slot, + test_slot_generation(&drive_terminal_interlock_ticket->slot) + 1); + test_set_slot_state(&drive_terminal_interlock_ticket->slot, PCM_XT_FREE); + } + } if (lock->tranche == LWTRANCHE_CLUSTER_PCM_X_LOCAL && lock == local_generation_interlock_lock && mode == LW_SHARED && local_generation_interlock_slot != NULL) { @@ -575,6 +628,11 @@ reset_fake_shmem(void) grant_interlock_lock = NULL; grant_interlock_ticket = NULL; grant_interlock_generation = 0; + drive_terminal_interlock_armed = false; + drive_terminal_interlock_lock = NULL; + drive_terminal_interlock_tag = NULL; + drive_terminal_interlock_ticket = NULL; + drive_terminal_interlock_reclaim = false; local_generation_interlock_lock = NULL; local_generation_interlock_slot = NULL; holder_abort_interlock_lock = NULL; @@ -1524,6 +1582,25 @@ init_active_pcm_x(uint64 master_session_incarnation) UT_ASSERT(cluster_pcm_x_runtime_activate(master_session_incarnation)); } +/* + * A' rebase: the formation-wide V2 coverage flag is written before activation + * and immutable for the runtime's lifetime; an ACTIVE snapshot exposes it and + * an activation without the flag (a formation with any old member) leaves + * rebase publication disabled. + */ +UT_TEST(test_runtime_rebase_wire_active_is_activation_bound) +{ + init_active_pcm_x(UINT64_C(77)); + UT_ASSERT(!cluster_pcm_x_runtime_snapshot().rebase_wire_active); + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + cluster_pcm_x_runtime_set_rebase_wire_active(true); + UT_ASSERT(cluster_pcm_x_runtime_activate(UINT64_C(77))); + UT_ASSERT(cluster_pcm_x_runtime_snapshot().rebase_wire_active); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + static bool ticket_refs_equal(const PcmXTicketRef *left, const PcmXTicketRef *right) { @@ -2035,6 +2112,13 @@ test_slot_flags(PcmXSlotHeader *slot) return pg_atomic_read_u32(&slot->state_flags) >> PCM_X_SLOT_FLAGS_SHIFT; } +/* Direct poke for protocol-impossible flag shapes (CORRUPT-arm probes). */ +static void +test_clear_slot_flags(PcmXSlotHeader *slot, uint32 flags_to_clear) +{ + pg_atomic_fetch_and_u32(&slot->state_flags, ~(flags_to_clear << PCM_X_SLOT_FLAGS_SHIFT)); +} + static PcmXSlotHeader * reserve_slot(PcmXAllocatorKind kind, PcmXSlotRef *ref) { @@ -2091,8 +2175,11 @@ UT_TEST(test_wire_abi_sizes_are_exact) UT_ASSERT_EQ(sizeof(PcmXAdmitAckPayload), 112); UT_ASSERT_EQ(sizeof(PcmXPhasePayload), 96); UT_ASSERT_EQ(sizeof(PcmXRevokePayload), 96); + UT_ASSERT_EQ(sizeof(PcmXRevokePayloadV2), 104); UT_ASSERT_EQ(sizeof(PcmXGrantPayload), 128); - UT_ASSERT_EQ(sizeof(PcmXInstallReadyPayload), 104); + UT_ASSERT_EQ(sizeof(PcmXInstallReadyPayload), 112); + UT_ASSERT_EQ(offsetof(PcmXInstallReadyPayload, rebased_own_generation), + PCM_X_INSTALL_READY_V1_LEN); UT_ASSERT_EQ(sizeof(PcmXFinalAckPayload), 104); UT_ASSERT_EQ(sizeof(PcmXBlockerSetHeaderPayload), 104); UT_ASSERT_EQ(sizeof(PcmXBlockerChunkPayload), 128); @@ -2119,6 +2206,7 @@ UT_TEST(test_wire_abi_offsets_are_exact) UT_ASSERT_EQ(offsetof(PcmXEnqueuePayload, prehandle), 64); UT_ASSERT_EQ(offsetof(PcmXAdmitAckPayload, prehandle), 88); UT_ASSERT_EQ(offsetof(PcmXAdmitAckPayload, result), 104); + UT_ASSERT_EQ(offsetof(PcmXRevokePayloadV2, required_page_scn), 96); UT_ASSERT_EQ(offsetof(PcmXGrantPayload, image), 88); UT_ASSERT_EQ(offsetof(PcmXBlockerSetHeaderPayload, set_generation), 88); UT_ASSERT_EQ(offsetof(PcmXBlockerChunkPayload, blocker), 80); @@ -2144,13 +2232,13 @@ UT_TEST(test_wire_abi_offsets_are_exact) UT_TEST(test_runtime_layout_abi_and_offsets_are_exact) { - UT_ASSERT_EQ(PCM_X_SHMEM_LAYOUT_VERSION, 12); + UT_ASSERT_EQ(PCM_X_SHMEM_LAYOUT_VERSION, 14); UT_ASSERT_EQ(PCM_X_LOCK_PARTITIONS, NUM_BUFFER_PARTITIONS); UT_ASSERT_EQ(PCM_X_LWLOCK_COUNT, 257); UT_ASSERT_EQ(sizeof(PcmXShmemLayout), 440); UT_ASSERT_EQ(sizeof(PcmXAllocatorState), 32); - UT_ASSERT_EQ(sizeof(PcmXStats), 176); - UT_ASSERT_EQ(sizeof(PcmXStatsSnapshot), 224); + UT_ASSERT_EQ(sizeof(PcmXStats), 184); + UT_ASSERT_EQ(sizeof(PcmXStatsSnapshot), 232); UT_ASSERT_EQ(sizeof(PcmXSlotHeader), 24); UT_ASSERT_EQ(offsetof(PcmXSlotHeader, next_free), 0); UT_ASSERT_EQ(offsetof(PcmXSlotHeader, generation_change_seq), 8); @@ -2158,17 +2246,20 @@ UT_TEST(test_runtime_layout_abi_and_offsets_are_exact) UT_ASSERT_EQ(offsetof(PcmXSlotHeader, slot_generation_hi), 16); UT_ASSERT_EQ(offsetof(PcmXSlotHeader, state_flags), 20); UT_ASSERT_EQ(sizeof(PcmXReliableLegState), 56); - UT_ASSERT_EQ(sizeof(PcmXLocalProgress), 240); + UT_ASSERT_EQ(sizeof(PcmXLocalProgress), 248); UT_ASSERT_EQ(offsetof(PcmXLocalProgress, master_session_incarnation), 224); UT_ASSERT_EQ(offsetof(PcmXLocalProgress, master_node), 232); - UT_ASSERT_EQ(sizeof(PcmXLocalHolderProgress), 160); - UT_ASSERT_EQ(offsetof(PcmXLocalHolderProgress, master_session_incarnation), 144); - UT_ASSERT_EQ(offsetof(PcmXLocalHolderProgress, master_node), 152); + UT_ASSERT_EQ(sizeof(PcmXLocalHolderProgress), 168); + UT_ASSERT_EQ(offsetof(PcmXLocalHolderProgress, required_page_scn), 128); + UT_ASSERT_EQ(offsetof(PcmXLocalHolderProgress, master_session_incarnation), 152); + UT_ASSERT_EQ(offsetof(PcmXLocalHolderProgress, master_node), 160); UT_ASSERT_EQ(sizeof(PcmXLocalBlockerSnapshot), 112); UT_ASSERT_EQ(sizeof(PcmXMasterTagSlot), 120); - UT_ASSERT_EQ(sizeof(PcmXMasterTicketSlot), 384); + UT_ASSERT_EQ(sizeof(PcmXMasterTicketSlot), 392); + UT_ASSERT_EQ(offsetof(PcmXMasterTicketSlot, grant_base_own_generation), 384); UT_ASSERT_EQ(sizeof(PcmXBlockerSlot), 128); - UT_ASSERT_EQ(sizeof(PcmXLocalTagSlot), 752); + UT_ASSERT_EQ(sizeof(PcmXLocalTagSlot), 768); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, grant_base_own_generation), 760); UT_ASSERT_EQ(sizeof(PcmXLocalMembershipSlot), 168); UT_ASSERT_EQ(sizeof(PcmXPeerFrontier), 48); UT_ASSERT_EQ(sizeof(PcmXPeerBinding), 16); @@ -2196,10 +2287,11 @@ UT_TEST(test_runtime_layout_abi_and_offsets_are_exact) UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, committed_own_generation), 408); UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_ref), 416); UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_image), 504); - UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_reliable), 544); - UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_terminal_drain_generation), 600); - UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, blocker_snapshot_ref), 608); - UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, blocker_snapshot_reliable), 696); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_required_page_scn), 544); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_reliable), 552); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_terminal_drain_generation), 608); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, blocker_snapshot_ref), 616); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, blocker_snapshot_reliable), 704); UT_ASSERT_EQ(offsetof(PcmXLocalMembershipSlot, identity), 24); UT_ASSERT_EQ(offsetof(PcmXLocalMembershipSlot, tag_slot_index), 128); UT_ASSERT_EQ(offsetof(PcmXLocalMembershipSlot, admitted_round), 160); @@ -2210,8 +2302,8 @@ UT_TEST(test_runtime_layout_abi_and_offsets_are_exact) UT_ASSERT_EQ(offsetof(PcmXShmemHeader, local_locks), 17280); UT_ASSERT_EQ(offsetof(PcmXShmemHeader, peer_frontiers), 33664); UT_ASSERT_EQ(offsetof(PcmXShmemHeader, stats), 35200); - UT_ASSERT_EQ(offsetof(PcmXShmemHeader, outbound_targets), 35376); - UT_ASSERT_EQ(sizeof(PcmXShmemHeader), 36400); + UT_ASSERT_EQ(offsetof(PcmXShmemHeader, outbound_targets), 35384); + UT_ASSERT_EQ(sizeof(PcmXShmemHeader), 36520); } UT_TEST(test_lwlock_held_limit_is_shared_200) @@ -2259,19 +2351,19 @@ UT_TEST(test_exactly_five_pools_and_bounded_directories) UT_ASSERT_EQ(layout.local_holder.directory_capacity, layout.local_holder.capacity * 2); } -UT_TEST(test_layout_v12_records_transfer_and_terminal_frontiers) +UT_TEST(test_layout_v13_records_transfer_and_terminal_frontiers) { PcmXShmemLayout layout; cluster_pcm_x_layout_compute(122, 25, 16384, 1024, &layout); - UT_ASSERT_EQ(layout.version, 12); + UT_ASSERT_EQ(layout.version, 14); UT_ASSERT_EQ(layout.lock_partition_count, PCM_X_LOCK_PARTITIONS); UT_ASSERT_EQ(layout.lwlock_count, PCM_X_LWLOCK_COUNT); UT_ASSERT_EQ(sizeof(PcmXPeerFrontier), 48); UT_ASSERT_EQ(sizeof(PcmXOutboundTargetFrontier), 32); UT_ASSERT_EQ(offsetof(PcmXShmemHeader, peer_frontiers), 33664); UT_ASSERT_EQ(offsetof(PcmXShmemHeader, stats), 35200); - UT_ASSERT_EQ(offsetof(PcmXShmemHeader, outbound_targets), 35376); + UT_ASSERT_EQ(offsetof(PcmXShmemHeader, outbound_targets), 35384); } UT_TEST(test_offsets_are_aligned_ordered_and_bounded) @@ -3803,6 +3895,7 @@ UT_TEST(test_local_queue_invalidate_authority_is_exact_and_read_only) PcmXLocalHolderSnapshot holder_snapshot; PcmXLocalBlockerSnapshot blocker_snapshot; PcmXTicketRef probe_ref; + ClusterLmdVertex blocker; uint64 master_session = UINT64_C(8302); init_active_pcm_x(UINT64_C(7216)); @@ -3825,6 +3918,13 @@ UT_TEST(test_local_queue_invalidate_authority_is_exact_and_read_only) &holder_snapshot, &holder, 1, NULL, 0, &blocker_snapshot), PCM_X_QUEUE_OK); + /* An exact INVALIDATE is itself master transfer-phase evidence. An + * already-staged empty blocker set has no graph edge to omit, so passive + * pinned S release must not wait for its reordered type-48 ACK. */ + UT_ASSERT_EQ(cluster_pcm_x_local_queue_invalidate_authorize_exact( + &probe_ref.identity.tag, probe_ref.identity.cluster_epoch, + probe_ref.identity.request_id, probe_ref.identity.node_id, 1, master_session), + PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, 1, master_session), PCM_X_QUEUE_OK); @@ -3847,6 +3947,17 @@ UT_TEST(test_local_queue_invalidate_authority_is_exact_and_read_only) probe_ref.identity.request_id, probe_ref.identity.node_id, 1, master_session + 1), PCM_X_QUEUE_STALE); + /* An in-flight nonempty set still needs its exact ACK/graph proof. */ + blocker = make_blocker(holder_key.identity.node_id, holder_key.identity.procno, + holder_key.identity.request_id); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, &holder, 1, + &blocker, 1, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_queue_invalidate_authorize_exact( + &probe_ref.identity.tag, probe_ref.identity.cluster_epoch, + probe_ref.identity.request_id, probe_ref.identity.node_id, 1, master_session), + PCM_X_QUEUE_NOT_READY); UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); } @@ -6247,6 +6358,11 @@ UT_TEST(test_master_transfer_wire_49_56_is_generation_exact) UT_ASSERT_EQ(header->next_image_id, 2); UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(&transfer, 2, source_session, &revoke), PCM_X_QUEUE_DUPLICATE); + /* Each exact re-arm is one retry-pump pass; leg_retry must count it. */ + UT_ASSERT_EQ(ticket->reliable.retry_count, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(&transfer, 2, source_session, &revoke), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(ticket->reliable.retry_count, 2); memset(&image_ready, 0, sizeof(image_ready)); image_ready.ref = transfer; @@ -6271,7 +6387,7 @@ UT_TEST(test_master_transfer_wire_49_56_is_generation_exact) install_ready.result = PCM_X_QUEUE_OK; install_ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; UT_ASSERT_EQ( - cluster_pcm_x_master_install_ready_exact(&install_ready, 0, requester_session, &commit), + cluster_pcm_x_master_install_ready_exact(&install_ready, 0, 0, requester_session, &commit), PCM_X_QUEUE_OK); UT_ASSERT_EQ(commit.phase, PGRAC_IC_MSG_PCM_X_COMMIT_X); @@ -6323,62 +6439,46 @@ UT_TEST(test_master_transfer_wire_49_56_is_generation_exact) PCM_X_QUEUE_DUPLICATE); } -UT_TEST(test_master_grant_generation_exhaustion_never_wraps) +/* Admit with an explicit nonzero enqueue-time base_own_generation, so the + * strictly-newer rebase arms have a real lower bound to violate. */ +static void +admit_active_probe_with_base(BlockNumber block, int32 node_id, uint32 procno, uint64 request_id, + uint64 sender_session, uint64 prehandle_sequence, + uint64 base_own_generation, PcmXMasterAdmission *admission_out) { - PcmXShmemHeader *header; - PcmXMasterAdmission admission; - PcmXMasterBlockerSnapshot snapshot; - PcmXBlockerSetHeaderPayload blocker_commit; - PcmXMasterTicketSlot *ticket; - PcmXTicketRef transfer; - const uint64 source_session = UINT64_C(8270); - const uint64 graph_generation = UINT64_C(9270); + PcmXWaitIdentity identity = make_wait_identity(block, node_id, procno, request_id); + PcmXEnqueuePayload request; + PcmXTicketRef active; - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - admit_active_probe(801, 0, 21, UINT64_C(8270), UINT64_C(7270), 1, &admission); - arm_blocker_probe(&admission.ref, 2, source_session); - blocker_commit = make_blocker_header(&admission.ref, 1, NULL, 0); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 2, source_session), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 2, source_session, - NULL, 0, &snapshot), - PCM_X_QUEUE_OK); + identity.base_own_generation = base_own_generation; + request = make_enqueue(identity, sender_session, prehandle_sequence); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, admission_out), PCM_X_QUEUE_OK); + UT_ASSERT_NOT_NULL(admission_out); UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, graph_generation), + cluster_pcm_x_master_admit_confirm_exact(&admission_out->ref, UINT64_C(9000) + request_id), PCM_X_QUEUE_OK); UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 2, source_session), + cluster_pcm_x_master_promote_head_exact(&identity.tag, identity.cluster_epoch, &active), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - ticket->reliable.state_sequence = UINT64_MAX; - UT_ASSERT_EQ( - cluster_pcm_x_master_begin_transfer_exact(&admission.ref, graph_generation, &transfer), - PCM_X_QUEUE_COUNTER_EXHAUSTED); - UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_ACTIVE_PROBE); - UT_ASSERT_EQ(ticket->ref.grant_generation, 0); - UT_ASSERT_EQ(ticket->reliable.state_sequence, UINT64_MAX); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT(ticket_refs_equal(&active, &admission_out->ref)); } -UT_TEST(test_master_image_id_allocator_encodes_node31_and_never_wraps) +/* Drive an admitted ACTIVE_PROBE ticket through blocker commit, transfer, + * revoke and IMAGE_READY, leaving its reliable leg armed at PREPARE_GRANT. */ +static void +drive_master_ticket_to_prepared(PcmXMasterAdmission *admission, uint64 source_session, + uint64 graph_generation, PcmXTicketRef *transfer_out, + uint64 *image_id_out) { - PcmXShmemHeader *header; - PcmXMasterAdmission admission; PcmXMasterBlockerSnapshot snapshot; PcmXBlockerSetHeaderPayload blocker_commit; PcmXRevokePayload revoke; - PcmXTicketRef transfer; - uint64 expected_image_id; - const uint64 source_session = UINT64_C(8271); - const uint64 graph_generation = UINT64_C(9271); + PcmXGrantPayload image_ready; + PcmXGrantPayload prepare; - init_active_pcm_x(UINT64_C(77)); - cluster_node_id = 31; - header = ClusterPcmXConvertShmem; - admit_active_probe(802, 0, 22, UINT64_C(8271), UINT64_C(7271), 1, &admission); - arm_blocker_probe(&admission.ref, 2, source_session); - blocker_commit = make_blocker_header(&admission.ref, 1, NULL, 0); + arm_blocker_probe(&admission->ref, 2, source_session); + blocker_commit = make_blocker_header(&admission->ref, 1, NULL, 0); UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 2, source_session), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 2, source_session, @@ -6388,3253 +6488,5453 @@ UT_TEST(test_master_image_id_allocator_encodes_node31_and_never_wraps) cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, graph_generation), PCM_X_QUEUE_OK); UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 2, source_session), + cluster_pcm_x_master_blocker_probe_complete_exact(&admission->ref, 1, 2, source_session), PCM_X_QUEUE_OK); UT_ASSERT_EQ( - cluster_pcm_x_master_begin_transfer_exact(&admission.ref, graph_generation, &transfer), + cluster_pcm_x_master_begin_transfer_exact(&admission->ref, graph_generation, transfer_out), PCM_X_QUEUE_OK); - header->next_image_id = PCM_X_IMAGE_ID_SEQ_MASK; - UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(&transfer, 2, source_session, &revoke), + UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(transfer_out, 2, source_session, &revoke), PCM_X_QUEUE_OK); - UT_ASSERT(cluster_pcm_x_image_id_encode(31, PCM_X_IMAGE_ID_SEQ_MASK, &expected_image_id)); - UT_ASSERT_EQ(revoke.image_id, expected_image_id); - UT_ASSERT_EQ(header->next_image_id, PCM_X_IMAGE_ID_SEQ_MASK + 1); + memset(&image_ready, 0, sizeof(image_ready)); + image_ready.ref = *transfer_out; + image_ready.image.image_id = revoke.image_id; + image_ready.image.source_own_generation = UINT64_C(44); + image_ready.image.page_scn = UINT64_C(55); + image_ready.image.page_lsn = UINT64_C(66); + image_ready.image.source_node = 2; + image_ready.image.page_checksum = UINT32_C(77); + UT_ASSERT_EQ(cluster_pcm_x_master_image_ready_exact(&image_ready, 2, source_session, &prepare), + PCM_X_QUEUE_OK); + *image_id_out = revoke.image_id; +} - /* The first raw value beyond the 56-bit domain is a durable exhausted - * marker. It is neither masked nor wrapped into another wire identity. */ - init_active_pcm_x(UINT64_C(78)); - cluster_node_id = 31; +typedef struct CommitRetryStageCapture { + int calls; + bool accept; + uint8 msg_type; + int32 dest_node_id; + Size payload_len; + PcmXPhasePayload payload; +} CommitRetryStageCapture; + +static bool +capture_commit_retry_stage(uint8 msg_type, int32 dest_node_id, const void *payload, + Size payload_len, void *callback_arg) +{ + CommitRetryStageCapture *capture = (CommitRetryStageCapture *)callback_arg; + + capture->calls++; + capture->msg_type = msg_type; + capture->dest_node_id = dest_node_id; + capture->payload_len = payload_len; + memset(&capture->payload, 0, sizeof(capture->payload)); + if (payload != NULL && payload_len == sizeof(capture->payload)) + memcpy(&capture->payload, payload, sizeof(capture->payload)); + return capture->accept; +} + +/* + * P0-20/type49: an exact queue INVALIDATE BUSY is a zero-mutation negative + * proof. It must pause this ticket's retry pump without crediting the busy + * holder, changing authority, or minting a new transfer identity. + */ +UT_TEST(test_master_invalidate_busy_backs_off_exact_revoke_ticket) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerSnapshot blocker_snapshot; + PcmXBlockerSetHeaderPayload blocker_commit; + PcmXMasterDriveSnapshot armed; + PcmXMasterDriveSnapshot waiting; + PcmXMasterDriveSnapshot backed_off; + PcmXMasterDriveSnapshot replay; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot after; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef transfer; + PcmXRevokePayload revoke; + const uint64 requester_session = UINT64_C(7484); + const uint64 source_session = UINT64_C(8484); + const uint64 graph_generation = UINT64_C(9484); + const uint32 source_bit = UINT32_C(1) << 2; + const uint32 busy_bit = UINT32_C(1) << 3; + + init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - admit_active_probe(803, 0, 23, UINT64_C(8272), UINT64_C(7272), 1, &admission); + admit_active_probe(814, 0, 24, UINT64_C(8484), requester_session, 1, &admission); arm_blocker_probe(&admission.ref, 2, source_session); blocker_commit = make_blocker_header(&admission.ref, 1, NULL, 0); UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 2, source_session), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 2, source_session, - NULL, 0, &snapshot), + NULL, 0, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_graph_commit_exact(&blocker_snapshot, NULL, 0, + graph_generation), PCM_X_QUEUE_OK); - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, graph_generation), - PCM_X_QUEUE_OK); UT_ASSERT_EQ( cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 2, source_session), PCM_X_QUEUE_OK); UT_ASSERT_EQ( cluster_pcm_x_master_begin_transfer_exact(&admission.ref, graph_generation, &transfer), PCM_X_QUEUE_OK); - header->next_image_id = PCM_X_IMAGE_ID_SEQ_MASK + 1; UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(&transfer, 2, source_session, &revoke), - PCM_X_QUEUE_COUNTER_EXHAUSTED); - UT_ASSERT_EQ(revoke.image_id, 0); - UT_ASSERT_EQ(header->next_image_id, PCM_X_IMAGE_ID_SEQ_MASK + 1); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact(&transfer.identity.tag, + transfer.identity.cluster_epoch, &armed), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_bitmap_replace_exact(&armed, source_bit | busy_bit, + source_bit, &waiting), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + before = *ticket; + + UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact(&waiting, 3, UINT64_C(1000), + UINT64_C(100), &backed_off), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(backed_off.retry_deadline_ms, UINT64_C(1100)); + UT_ASSERT_EQ(backed_off.pending_s_holders_bitmap, source_bit | busy_bit); + UT_ASSERT_EQ(backed_off.acked_s_holders_bitmap, source_bit); + UT_ASSERT_EQ(backed_off.retry_count, waiting.retry_count); + after = *ticket; + after.reliable.retry_deadline_ms = before.reliable.retry_deadline_ms; + UT_ASSERT(memcmp(&after, &before, sizeof(after)) == 0); + + /* The pre-BUSY snapshot is consumed exactly. Replayed BUSY during the + * same live window is an idempotent scheduling no-op. */ + memset(&replay, 0xA5, sizeof(replay)); + UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact(&waiting, 3, UINT64_C(1001), + UINT64_C(100), &replay), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(&replay, &(PcmXMasterDriveSnapshot){ 0 }, sizeof(replay)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact(&backed_off, 3, UINT64_C(1099), + UINT64_C(100), &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(replay.retry_deadline_ms, UINT64_C(1100)); + UT_ASSERT_EQ(ticket->reliable.retry_deadline_ms, UINT64_C(1100)); + + /* The source already supplied the retained image and is ACKed; a BUSY + * attributed to it cannot pause invalidation of another holder. */ + UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact(&backed_off, 2, UINT64_C(1100), + UINT64_C(100), &replay), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(ticket->reliable.retry_deadline_ms, UINT64_C(1100)); + + /* Expiry permits one new bounded window, still on the same identity. */ + UT_ASSERT_EQ(cluster_pcm_x_master_invalidate_busy_backoff_exact(&backed_off, 3, UINT64_C(1100), + UINT64_C(100), &replay), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(replay.retry_deadline_ms, UINT64_C(1200)); + UT_ASSERT(ticket_refs_equal(&replay.ref, &waiting.ref)); + UT_ASSERT(memcmp(&replay.image, &waiting.image, sizeof(replay.image)) == 0); } -UT_TEST(test_master_blocker_wire_stage_preserves_old_set_until_commit) +/* + * A requester can consume COMMIT_X and finish the exact FINAL protocol after + * the periodic pump's allocator lookup but before its master-domain lock. + * That monotone terminal successor is a stale retry, never queue corruption. + */ +UT_TEST(test_master_commit_retry_loses_race_to_terminal_progress_without_fail_closed) { PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXMasterBlockerEntry old_entry; - PcmXMasterBlockerSnapshot empty_snapshot; - PcmXMasterBlockerSnapshot old_snapshot; - PcmXBlockerSetHeaderPayload empty; + PcmXMasterDriveSnapshot armed; + PcmXMasterDriveSnapshot retried; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; PcmXTicketRef transfer; - ClusterLmdVertex old_blocker = make_blocker(1, 5, UINT64_C(8761)); - const uint64 source_session = UINT64_C(82601); + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload first_commit; + PcmXPhasePayload retry_commit; + uint64 image_id; + uint32 partition; + int mode; + const uint64 requester_session = UINT64_C(7383); + const uint64 source_session = UINT64_C(8383); + + for (mode = 0; mode < 2; mode++) { + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe_with_base(816 + mode, 0, 22 + mode, UINT64_C(8383), requester_session, 1, + UINT64_C(5), &admission); + drive_master_ticket_to_prepared(&admission, source_session, UINT64_C(9383), &transfer, + &image_id); + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + + memset(&install_ready, 0, sizeof(install_ready)); + install_ready.ref = transfer; + install_ready.image_id = image_id; + install_ready.result = PCM_X_QUEUE_OK; + install_ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, 0, 0, + requester_session, &first_commit), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact( + &transfer.identity.tag, transfer.identity.cluster_epoch, &armed), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(armed.pending_opcode, PGRAC_IC_MSG_PCM_X_COMMIT_X); + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&transfer.identity.tag)); + drive_terminal_interlock_lock = &header->master_locks[partition].lock; + drive_terminal_interlock_tag = tag_slot; + drive_terminal_interlock_ticket = ticket; + drive_terminal_interlock_reclaim = mode != 0; + drive_terminal_interlock_armed = true; + memset(&retry_commit, 0xA5, sizeof(retry_commit)); + memset(&retried, 0xA5, sizeof(retried)); + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_exact(&armed, UINT64_C(1000), UINT64_C(1100), + &retry_commit, &retried), + mode == 0 ? PCM_X_QUEUE_NOT_READY : PCM_X_QUEUE_STALE); + UT_ASSERT(!drive_terminal_interlock_armed); + UT_ASSERT(memcmp(&retry_commit, &(PcmXPhasePayload){ 0 }, sizeof(retry_commit)) == 0); + UT_ASSERT(memcmp(&retried, &(PcmXMasterDriveSnapshot){ 0 }, sizeof(retried)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + } +} + +/* + * P0-14: the master has crossed the handoff boundary and durably armed type + * 53, but the first COMMIT_X frame is dropped. The real bounded master-work + * scan must still select the ACTIVE_TRANSFER ticket; an exact deadline retry + * then reconstructs the same logical leg without minting a ticket or moving + * authority. FINAL_ACK remains the only exact successor that consumes it. + */ +UT_TEST(test_master_periodic_drive_replays_post_handoff_commit_x_exactly) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterDriveSnapshot armed; + PcmXMasterDriveSnapshot retried; + PcmXMasterDriveSnapshot retried_again; + PcmXMasterDriveSnapshot stale; + PcmXMasterTicketSlot before_blocked; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef transfer; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload first_commit; + PcmXPhasePayload retry_commit; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + PcmXFinalAckPayload final_ack; + PcmXMasterFinalAckToken final_ack_token; + PcmXQueueResult result; + CommitRetryStageCapture stage; + BufferTag drive_tag; + Size cursor; + uint64 drive_epoch; + uint64 image_id; + const uint64 requester_session = UINT64_C(7384); + const uint64 source_session = UINT64_C(8384); init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - admit_active_probe(792, 0, 12, UINT64_C(8261), UINT64_C(7261), 1, &admission); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact( - &admission.ref, 1, &old_blocker, 1, blocker_set_crc32c(&old_blocker, 1)), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &old_entry, 1, &old_snapshot), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_graph_commit_exact(&old_snapshot, &old_entry, 1, - UINT64_C(9161)), + admit_active_probe_with_base(813, 0, 23, UINT64_C(8384), requester_session, 1, UINT64_C(5), + &admission); + drive_master_ticket_to_prepared(&admission, source_session, UINT64_C(9384), &transfer, + &image_id); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + + memset(&install_ready, 0, sizeof(install_ready)); + install_ready.ref = transfer; + install_ready.image_id = image_id; + install_ready.result = PCM_X_QUEUE_OK; + install_ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, 0, 0, requester_session, + &first_commit), PCM_X_QUEUE_OK); - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &old_entry, 1, &old_snapshot), - PCM_X_QUEUE_OK); - empty = make_blocker_header(&admission.ref, 2, NULL, 0); - arm_blocker_probe(&admission.ref, 3, source_session); + UT_ASSERT_EQ(first_commit.phase, PGRAC_IC_MSG_PCM_X_COMMIT_X); + UT_ASSERT(ticket_refs_equal(&first_commit.ref, &transfer)); + UT_ASSERT_EQ(ticket->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_COMMIT_X); + UT_ASSERT_EQ(ticket->reliable.retry_count, 0); + UT_ASSERT_EQ(ticket->reliable.retry_deadline_ms, 0); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&empty, 3, source_session), + /* Drop first_commit: no requester delivery and therefore no FINAL_ACK. */ + cursor = admission.tag_slot.slot_index; + UT_ASSERT_EQ(cluster_pcm_x_master_drive_work_next(&cursor, 1, &drive_tag, &drive_epoch), PCM_X_QUEUE_OK); - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&old_snapshot, &old_entry, 1), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&empty, 3, source_session, NULL, 0, - &empty_snapshot), + UT_ASSERT(BufferTagsEqual(&drive_tag, &transfer.identity.tag)); + UT_ASSERT_EQ(drive_epoch, transfer.identity.cluster_epoch); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact(&drive_tag, drive_epoch, &armed), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 0); - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&old_snapshot, &old_entry, 1), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&empty_snapshot, NULL, 0, UINT64_C(9261)), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 2, 3, source_session), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ( - cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9261), &transfer), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&empty, 3, source_session), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&empty, 3, source_session, NULL, 0, - &empty_snapshot), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&empty, 3, source_session + 1, - NULL, 0, &empty_snapshot), + UT_ASSERT_EQ(armed.ticket_state, PCM_XT_ACTIVE_TRANSFER); + UT_ASSERT_EQ(armed.pending_opcode, PGRAC_IC_MSG_PCM_X_COMMIT_X); + UT_ASSERT_EQ(armed.expected_responder_session, requester_session); + + memset(&stage, 0, sizeof(stage)); + stage.accept = true; + result = cluster_pcm_x_master_commit_retry_drive(&armed, UINT64_C(1000), UINT64_C(100), + capture_commit_retry_stage, &stage, &retried); + UT_ASSERT_EQ(result, PCM_X_QUEUE_OK); + if (result != PCM_X_QUEUE_OK) + return; + UT_ASSERT_EQ(stage.calls, 1); + UT_ASSERT_EQ(stage.msg_type, PGRAC_IC_MSG_PCM_X_COMMIT_X); + UT_ASSERT_EQ(stage.dest_node_id, transfer.identity.node_id); + UT_ASSERT_EQ(stage.payload_len, sizeof(PcmXPhasePayload)); + UT_ASSERT_EQ(stage.payload.phase, PGRAC_IC_MSG_PCM_X_COMMIT_X); + UT_ASSERT(ticket_refs_equal(&stage.payload.ref, &first_commit.ref)); + UT_ASSERT_EQ(retried.state_sequence, armed.state_sequence); + UT_ASSERT_EQ(retried.retry_count, 1); + UT_ASSERT_EQ(retried.retry_deadline_ms, UINT64_C(1100)); + UT_ASSERT_EQ(ticket->reliable.retry_count, 1); + + /* A periodic pass before the deadline is a byte-stable no-op. */ + memset(&retried_again, 0xA5, sizeof(retried_again)); + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_drive(&retried, UINT64_C(1099), UINT64_C(100), + capture_commit_retry_stage, &stage, + &retried_again), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(&retried_again, &(PcmXMasterDriveSnapshot){ 0 }, sizeof(retried_again)) == 0); + UT_ASSERT_EQ(stage.calls, 1); + UT_ASSERT_EQ(ticket->reliable.retry_count, 1); + UT_ASSERT_EQ(ticket->reliable.retry_deadline_ms, UINT64_C(1100)); + + /* The pre-retry token and a forged responder session cannot mutate it. */ + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_exact(&armed, UINT64_C(1100), UINT64_C(1200), + &retry_commit, &retried_again), + PCM_X_QUEUE_STALE); + stale = retried; + stale.expected_responder_session++; + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_exact(&stale, UINT64_C(1100), UINT64_C(1200), + &retry_commit, &retried_again), PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&transfer), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&empty, 3, source_session), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&empty, 3, source_session, NULL, 0, - &empty_snapshot), - PCM_X_QUEUE_DUPLICATE); -} -UT_TEST(test_master_blocker_wire_stage_rejects_reorder_conflict_and_wrong_source) -{ - PcmXMasterAdmission admission; - PcmXMasterBlockerEntry entries[2]; - PcmXMasterBlockerSnapshot snapshot; - PcmXBlockerSetHeaderPayload begin; - PcmXBlockerSetHeaderPayload conflict_begin; - PcmXBlockerChunkPayload edge[2]; - PcmXBlockerChunkPayload conflict_edge; - ClusterLmdVertex blockers[2]; - ClusterLmdVertex noncanonical; - const uint64 source_session = UINT64_C(82701); - - init_active_pcm_x(UINT64_C(77)); - admit_active_probe(793, 0, 13, UINT64_C(8271), UINT64_C(7271), 1, &admission); - blockers[0] = make_blocker(2, 6, UINT64_C(8771)); - blockers[1] = make_blocker(2, 7, UINT64_C(8772)); - begin = make_blocker_header(&admission.ref, 1, blockers, lengthof(blockers)); - edge[0] = make_blocker_chunk(&admission.ref, 1, 0, blockers[0]); - edge[1] = make_blocker_chunk(&admission.ref, 1, 1, blockers[1]); - arm_blocker_probe(&admission.ref, 2, source_session); - - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[0], 3, source_session), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[1], 2, source_session), + /* A full outbound ring still advances the deadline and cannot busy-spin. */ + stage.accept = false; + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_drive(&retried, UINT64_C(1100), UINT64_C(100), + capture_commit_retry_stage, &stage, + &retried_again), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(stage.calls, 2); + UT_ASSERT(ticket_refs_equal(&stage.payload.ref, &transfer)); + UT_ASSERT_EQ(retried_again.state_sequence, armed.state_sequence); + UT_ASSERT_EQ(retried_again.retry_count, 2); + UT_ASSERT_EQ(retried_again.retry_deadline_ms, UINT64_C(1200)); + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_drive(&retried_again, UINT64_C(1100), + UINT64_C(100), capture_commit_retry_stage, + &stage, &stale), PCM_X_QUEUE_NOT_READY); - conflict_begin = begin; - conflict_begin.set_crc32c++; - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&conflict_begin, 2, source_session), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[0], 2, source_session), + UT_ASSERT_EQ(stage.calls, 2); + + /* Deadline expiry stages the same logical leg and advances scheduling. */ + stage.accept = true; + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_drive(&retried_again, UINT64_C(1200), + UINT64_C(100), capture_commit_retry_stage, + &stage, &stale), PCM_X_QUEUE_OK); - conflict_begin = begin; - conflict_begin.nblockers = UINT32_MAX; - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&conflict_begin, 2, source_session, - NULL, 0, &snapshot), - PCM_X_QUEUE_INVALID); - conflict_edge = edge[0]; - conflict_edge.blocker.wait_seq++; - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&conflict_edge, 2, source_session), + UT_ASSERT_EQ(stage.calls, 3); + UT_ASSERT(ticket_refs_equal(&stage.payload.ref, &transfer)); + UT_ASSERT_EQ(stale.state_sequence, armed.state_sequence); + UT_ASSERT_EQ(stale.retry_count, 3); + UT_ASSERT_EQ(stale.retry_deadline_ms, UINT64_C(1300)); + + /* Old-session ACK is refused; exact FINAL_ACK advances and replays idempotently. */ + memset(&final_ack, 0, sizeof(final_ack)); + final_ack.ref = transfer; + final_ack.image_id = image_id; + final_ack.committed_own_generation = UINT64_C(6); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session + 1, + &final_ack_token), PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&begin, 2, source_session, entries, - lengthof(entries), &snapshot), - PCM_X_QUEUE_NOT_READY); - noncanonical = make_blocker(1, 8, UINT64_C(8773)); - conflict_edge = make_blocker_chunk(&admission.ref, 1, 1, noncanonical); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&conflict_edge, 2, source_session), - PCM_X_QUEUE_INVALID); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[1], 2, source_session), + UT_ASSERT_EQ(ticket->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_COMMIT_X); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session, + &final_ack_token), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&begin, 3, source_session, entries, - lengthof(entries), &snapshot), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&begin, 2, source_session, entries, - lengthof(entries), &snapshot), + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_finalize_exact(&final_ack_token, &final_commit), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[0], 2, source_session), + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session, + &final_ack_token), PCM_X_QUEUE_DUPLICATE); - conflict_edge = edge[0]; - conflict_edge.blocker.wait_seq++; - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&conflict_edge, 2, source_session), - PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_finalize_exact(&final_ack_token, &final_commit), + PCM_X_QUEUE_DUPLICATE); + + memset(&final_confirm, 0, sizeof(final_confirm)); + final_confirm.ref = transfer; + final_confirm.phase = PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM; + UT_ASSERT_EQ(cluster_pcm_x_master_final_confirm_exact(&final_confirm, 0, requester_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_COMPLETE); + UT_ASSERT_EQ(ticket->reliable.pending_opcode, 0); + UT_ASSERT_EQ(ticket->reliable.retry_count, 0); + UT_ASSERT_EQ(ticket->reliable.retry_deadline_ms, 0); + + /* A non-ACTIVE runtime never resurrects or replays the consumed leg. */ + before_blocked = *ticket; + UT_ASSERT( + cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, PCM_X_RUNTIME_RECOVERY_BLOCKED)); + UT_ASSERT_EQ(cluster_pcm_x_master_commit_retry_drive(&stale, UINT64_C(1300), UINT64_C(100), + capture_commit_retry_stage, &stage, + &armed), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(stage.calls, 3); + UT_ASSERT(memcmp(ticket, &before_blocked, sizeof(*ticket)) == 0); } -UT_TEST(test_master_blocker_wire_stage_fences_transfer_and_cancel_until_commit) +/* + * A' rebase (t/400 item 2): an interleaved revoke on the requester node + * consumes the enqueue-time base_own_generation while the request is queued. + * INSTALL_READY may publish a strictly-newer effective grant base exactly + * once; the ticket persists it and every later exact "+1" check runs against + * the effective base while the wire ref identity stays immutable. + */ +UT_TEST(test_master_install_ready_rebase_grant_base_is_one_shot) { PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXMasterBlockerSnapshot empty_snapshot; - PcmXBlockerSetHeaderPayload begin; + PcmXMasterTicketSlot *ticket; PcmXTicketRef transfer; - ClusterLmdVertex blocker = make_blocker(2, 9, UINT64_C(8781)); - const uint64 source_session = UINT64_C(82801); + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXMasterFinalAckToken final_ack_token; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + uint64 image_id; + const uint64 requester_session = UINT64_C(7281); + const uint64 source_session = UINT64_C(8281); init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - admit_active_probe(794, 0, 14, UINT64_C(8281), UINT64_C(7281), 1, &admission); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, NULL, 0, - blocker_set_crc32c(NULL, 0)), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, NULL, 0, &empty_snapshot), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&empty_snapshot, NULL, 0, UINT64_C(9281)), - PCM_X_QUEUE_OK); - begin = make_blocker_header(&admission.ref, 2, &blocker, 1); - arm_blocker_probe(&admission.ref, 2, source_session); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + admit_active_probe_with_base(810, 0, 20, UINT64_C(8281), requester_session, 1, UINT64_C(5), + &admission); + drive_master_ticket_to_prepared(&admission, source_session, UINT64_C(9281), &transfer, + &image_id); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(ticket->grant_base_own_generation, 0); - UT_ASSERT_EQ( - cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9281), &transfer), - PCM_X_QUEUE_BAD_STATE); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + memset(&install_ready, 0, sizeof(install_ready)); + install_ready.ref = transfer; + install_ready.image_id = image_id; + install_ready.result = PCM_X_QUEUE_OK; + install_ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; + + /* Shape violation: the exhausted sentinel can never be a grant base. */ + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, UINT64_MAX, 0, + requester_session, &commit), + PCM_X_QUEUE_INVALID); + /* Not strictly newer than the enqueue-time base: lower, then equal. */ + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, UINT64_C(4), 0, + requester_session, &commit), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, UINT64_C(5), 0, + requester_session, &commit), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(ticket->grant_base_own_generation, 0); + UT_ASSERT_EQ(ticket->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT); + + /* One-shot publish of a >1 drift (5 -> 8), then a same-value replay. */ + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, UINT64_C(8), 0, + requester_session, &commit), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(commit.phase, PGRAC_IC_MSG_PCM_X_COMMIT_X); + UT_ASSERT_EQ(ticket->grant_base_own_generation, UINT64_C(8)); + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, UINT64_C(8), 0, + requester_session, &commit), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(ticket->grant_base_own_generation, UINT64_C(8)); - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - admit_active_probe(795, 0, 15, UINT64_C(8282), UINT64_C(7282), 1, &admission); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, NULL, 0, - blocker_set_crc32c(NULL, 0)), + /* The effective grant base now drives the exact final-ack "+1" math: the + * enqueue-time math (5+1) is refused, the rebased math (8+1) commits. */ + memset(&final_ack, 0, sizeof(final_ack)); + final_ack.ref = transfer; + final_ack.image_id = image_id; + final_ack.committed_own_generation = UINT64_C(6); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session, + &final_ack_token), + PCM_X_QUEUE_STALE); + final_ack.committed_own_generation = UINT64_C(9); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session, + &final_ack_token), PCM_X_QUEUE_OK); - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, NULL, 0, &empty_snapshot), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&empty_snapshot, NULL, 0, UINT64_C(9282)), - PCM_X_QUEUE_OK); - begin = make_blocker_header(&admission.ref, 2, &blocker, 1); - arm_blocker_probe(&admission.ref, 2, source_session); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_finalize_exact(&final_ack_token, &final_commit), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), - PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(final_commit.phase, PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK); + memset(&final_confirm, 0, sizeof(final_confirm)); + final_confirm.ref = transfer; + final_confirm.phase = PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM; + UT_ASSERT_EQ(cluster_pcm_x_master_final_confirm_exact(&final_confirm, 0, requester_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_COMPLETE); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); } -UT_TEST(test_master_blocker_graph_commit_revalidates_exact_snapshot_and_generation) +/* A second, different rebase value is evidence divergence, never contention. */ +UT_TEST(test_master_install_ready_rebase_conflict_fails_closed) { PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXMasterBlockerEntry entry; - PcmXMasterBlockerEntry stale_entry; - PcmXMasterBlockerSnapshot snapshot; - PcmXMasterBlockerSnapshot stale_snapshot; - PcmXMasterTicketSlot before; PcmXMasterTicketSlot *ticket; - ClusterLmdVertex blocker = make_blocker(1, 1, UINT64_C(8801)); + PcmXTicketRef transfer; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + uint64 image_id; + const uint64 requester_session = UINT64_C(7282); + const uint64 source_session = UINT64_C(8282); init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - admit_active_probe(792, 0, 8, UINT64_C(8203), UINT64_C(7203), 1, &admission); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, &blocker, 1, - UINT32_C(0x701)), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &entry, 1, &snapshot), - PCM_X_QUEUE_OK); + admit_active_probe_with_base(811, 0, 21, UINT64_C(8282), requester_session, 1, UINT64_C(5), + &admission); + drive_master_ticket_to_prepared(&admission, source_session, UINT64_C(9282), &transfer, + &image_id); ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - stale_snapshot = snapshot; - stale_snapshot.set_generation++; - before = *ticket; - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&stale_snapshot, &entry, 1, UINT64_C(9101)), - PCM_X_QUEUE_STALE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + memset(&install_ready, 0, sizeof(install_ready)); + install_ready.ref = transfer; + install_ready.image_id = image_id; + install_ready.result = PCM_X_QUEUE_OK; + install_ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, UINT64_C(8), 0, + requester_session, &commit), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_install_ready_exact(&install_ready, UINT64_C(9), 0, + requester_session, &commit), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(ticket->grant_base_own_generation, UINT64_C(8)); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} - stale_entry = entry; - stale_entry.slot_ref.slot_generation++; - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &stale_entry, 1, UINT64_C(9101)), - PCM_X_QUEUE_STALE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, 0), - PCM_X_QUEUE_INVALID); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); +/* Without a rebase the ticket keeps the enqueue-time math bit for bit, and + * the locked exact check still refuses a wrong committed generation. */ +UT_TEST(test_master_install_ready_no_drift_keeps_identity_base_math) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef transfer; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXMasterFinalAckToken final_ack_token; + PcmXPhasePayload final_commit; + uint64 image_id; + const uint64 requester_session = UINT64_C(7283); + const uint64 source_session = UINT64_C(8283); + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe_with_base(812, 0, 22, UINT64_C(8283), requester_session, 1, UINT64_C(5), + &admission); + drive_master_ticket_to_prepared(&admission, source_session, UINT64_C(9283), &transfer, + &image_id); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + + memset(&install_ready, 0, sizeof(install_ready)); + install_ready.ref = transfer; + install_ready.image_id = image_id; + install_ready.result = PCM_X_QUEUE_OK; + install_ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9101)), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(ticket->graph_generation, UINT64_C(9101)); - before = *ticket; - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9101)), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9100)), - PCM_X_QUEUE_STALE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9102)), + cluster_pcm_x_master_install_ready_exact(&install_ready, 0, 0, requester_session, &commit), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(ticket->graph_generation, UINT64_C(9102)); - UT_ASSERT_EQ(max_held_lwlock_count, 1); + UT_ASSERT_EQ(ticket->grant_base_own_generation, 0); + + memset(&final_ack, 0, sizeof(final_ack)); + final_ack.ref = transfer; + final_ack.image_id = image_id; + /* Monotonic but not exactly base+1: passes ingress sanity, refused by the + * locked exact check. */ + final_ack.committed_own_generation = UINT64_C(7); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session, + &final_ack_token), + PCM_X_QUEUE_STALE); + final_ack.committed_own_generation = UINT64_C(6); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session, + &final_ack_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_finalize_exact(&final_ack_token, &final_commit), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); } -UT_TEST(test_master_transfer_requires_committed_exact_empty_blocker_set) +UT_TEST(test_master_grant_generation_exhaustion_never_wraps) { PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXMasterBlockerEntry entry; PcmXMasterBlockerSnapshot snapshot; - PcmXMasterTicketSlot before; + PcmXBlockerSetHeaderPayload blocker_commit; PcmXMasterTicketSlot *ticket; PcmXTicketRef transfer; - ClusterLmdVertex blocker = make_blocker(1, 2, UINT64_C(8802)); + const uint64 source_session = UINT64_C(8270); + const uint64 graph_generation = UINT64_C(9270); init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - admit_active_probe(793, 0, 9, UINT64_C(8204), UINT64_C(7204), 1, &admission); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, &blocker, 1, - UINT32_C(0x702)), + admit_active_probe(801, 0, 21, UINT64_C(8270), UINT64_C(7270), 1, &admission); + arm_blocker_probe(&admission.ref, 2, source_session); + blocker_commit = make_blocker_header(&admission.ref, 1, NULL, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 2, source_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &entry, 1, &snapshot), + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 2, source_session, + NULL, 0, &snapshot), PCM_X_QUEUE_OK); UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9201)), - PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); - before = *ticket; - UT_ASSERT_EQ( - cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9201), &transfer), - PCM_X_QUEUE_BAD_STATE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); - - UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 2, NULL, 0, UINT32_C(0x703)), + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, graph_generation), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, NULL, 0, &snapshot), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(ticket->graph_generation, 0); - before = *ticket; - UT_ASSERT_EQ( - cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9201), &transfer), - PCM_X_QUEUE_STALE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, UINT64_C(9202)), + cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 2, source_session), PCM_X_QUEUE_OK); - /* The direct replace fixture bypasses the 45-48 wire ACK. */ - ticket->reliable.last_response_opcode = PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT; - before = *ticket; - UT_ASSERT_EQ( - cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9201), &transfer), - PCM_X_QUEUE_STALE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + ticket->reliable.state_sequence = UINT64_MAX; UT_ASSERT_EQ( - cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9202), &transfer), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(transfer.grant_generation, ticket->reliable.state_sequence); + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, graph_generation, &transfer), + PCM_X_QUEUE_COUNTER_EXHAUSTED); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_ACTIVE_PROBE); + UT_ASSERT_EQ(ticket->ref.grant_generation, 0); + UT_ASSERT_EQ(ticket->reliable.state_sequence, UINT64_MAX); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); } -UT_TEST(test_master_active_probe_cancel_requires_committed_exact_empty_blocker_set) +UT_TEST(test_master_image_id_allocator_encodes_node31_and_never_wraps) { PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXMasterBlockerEntry entry; PcmXMasterBlockerSnapshot snapshot; - PcmXMasterBlockerSnapshot stale_snapshot; - PcmXMasterTagSlot tag_before; - PcmXMasterTagSlot *tag_slot; - PcmXMasterTicketSlot ticket_before; - PcmXMasterTicketSlot *ticket; - ClusterLmdVertex blocker = make_blocker(1, 3, UINT64_C(8803)); + PcmXBlockerSetHeaderPayload blocker_commit; + PcmXRevokePayload revoke; + PcmXTicketRef transfer; + uint64 expected_image_id; + const uint64 source_session = UINT64_C(8271); + const uint64 graph_generation = UINT64_C(9271); init_active_pcm_x(UINT64_C(77)); + cluster_node_id = 31; header = ClusterPcmXConvertShmem; - admit_active_probe(794, 0, 10, UINT64_C(8205), UINT64_C(7205), 1, &admission); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, &blocker, 1, - UINT32_C(0x704)), + admit_active_probe(802, 0, 22, UINT64_C(8271), UINT64_C(7271), 1, &admission); + arm_blocker_probe(&admission.ref, 2, source_session); + blocker_commit = make_blocker_header(&admission.ref, 1, NULL, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 2, source_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &entry, 1, &snapshot), + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 2, source_session, + NULL, 0, &snapshot), PCM_X_QUEUE_OK); UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9401)), + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, graph_generation), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; - ticket_before = *ticket; - tag_before = *tag_slot; - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); - - stale_snapshot = snapshot; UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 2, NULL, 0, UINT32_C(0x705)), + cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 2, source_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&stale_snapshot, &entry, 1, UINT64_C(9402)), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, NULL, 0, &snapshot), + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, graph_generation, &transfer), + PCM_X_QUEUE_OK); + header->next_image_id = PCM_X_IMAGE_ID_SEQ_MASK; + UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(&transfer, 2, source_session, &revoke), + PCM_X_QUEUE_OK); + UT_ASSERT(cluster_pcm_x_image_id_encode(31, PCM_X_IMAGE_ID_SEQ_MASK, &expected_image_id)); + UT_ASSERT_EQ(revoke.image_id, expected_image_id); + UT_ASSERT_EQ(header->next_image_id, PCM_X_IMAGE_ID_SEQ_MASK + 1); + + /* The first raw value beyond the 56-bit domain is a durable exhausted + * marker. It is neither masked nor wrapped into another wire identity. */ + init_active_pcm_x(UINT64_C(78)); + cluster_node_id = 31; + header = ClusterPcmXConvertShmem; + admit_active_probe(803, 0, 23, UINT64_C(8272), UINT64_C(7272), 1, &admission); + arm_blocker_probe(&admission.ref, 2, source_session); + blocker_commit = make_blocker_header(&admission.ref, 1, NULL, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 2, source_session, + NULL, 0, &snapshot), PCM_X_QUEUE_OK); UT_ASSERT_EQ( - cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, UINT64_C(9402)), + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, graph_generation), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 0); - UT_ASSERT_EQ(tag_slot->active_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, graph_generation, &transfer), + PCM_X_QUEUE_OK); + header->next_image_id = PCM_X_IMAGE_ID_SEQ_MASK + 1; + UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(&transfer, 2, source_session, &revoke), + PCM_X_QUEUE_COUNTER_EXHAUSTED); + UT_ASSERT_EQ(revoke.image_id, 0); + UT_ASSERT_EQ(header->next_image_id, PCM_X_IMAGE_ID_SEQ_MASK + 1); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); } -UT_TEST(test_master_locator_defers_mutable_grant_exactness_to_domain_lock) +UT_TEST(test_master_blocker_wire_stage_preserves_old_set_until_commit) { PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXMasterTicketSlot *ticket; - PcmXEnqueuePayload request; - PcmXTicketRef active; + PcmXMasterBlockerEntry old_entry; + PcmXMasterBlockerSnapshot empty_snapshot; + PcmXMasterBlockerSnapshot old_snapshot; + PcmXBlockerSetHeaderPayload empty; PcmXTicketRef transfer; - PcmXQueueResult result; - uint64 grant_generation; - uint32 partition; + ClusterLmdVertex old_blocker = make_blocker(1, 5, UINT64_C(8761)); + const uint64 source_session = UINT64_C(82601); init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - request = make_enqueue(make_wait_identity(743, 0, 9, UINT64_C(52001)), UINT64_C(5301), 1); - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(401)), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, - request.identity.cluster_epoch, &active), + admit_active_probe(792, 0, 12, UINT64_C(8261), UINT64_C(7261), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact( + &admission.ref, 1, &old_blocker, 1, blocker_set_crc32c(&old_blocker, 1)), PCM_X_QUEUE_OK); - commit_empty_blocker_graph(&active, UINT64_C(401)); - UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(401), &transfer), + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &old_entry, 1, &old_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_graph_commit_exact(&old_snapshot, &old_entry, 1, + UINT64_C(9161)), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - grant_generation = transfer.grant_generation; + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &old_entry, 1, &old_snapshot), + PCM_X_QUEUE_OK); + empty = make_blocker_header(&admission.ref, 2, NULL, 0); + arm_blocker_probe(&admission.ref, 3, source_session); - /* Model a torn allocator-lock observation while the domain writer finishes - * publishing grant_generation. Locator lookup must use immutable fields; - * the full ref is accepted only after the master-domain interlock restores - * the stable value. */ - ticket->ref.grant_generation = UINT64CONST(0x3333333322222222); - partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&transfer.identity.tag)); - grant_interlock_lock = &header->master_locks[partition].lock; - grant_interlock_ticket = ticket; - grant_interlock_generation = grant_generation; - grant_interlock_armed = true; - result = cluster_pcm_x_master_complete_exact(&transfer); - UT_ASSERT_EQ(result, PCM_X_QUEUE_OK); - UT_ASSERT(!grant_interlock_armed); - grant_interlock_armed = false; - grant_interlock_lock = NULL; - grant_interlock_ticket = NULL; + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&empty, 3, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&old_snapshot, &old_entry, 1), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&empty, 3, source_session, NULL, 0, + &empty_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&old_snapshot, &old_entry, 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&empty_snapshot, NULL, 0, UINT64_C(9261)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 2, 3, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9261), &transfer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&empty, 3, source_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&empty, 3, source_session, NULL, 0, + &empty_snapshot), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&empty, 3, source_session + 1, + NULL, 0, &empty_snapshot), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&transfer), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&empty, 3, source_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&empty, 3, source_session, NULL, 0, + &empty_snapshot), + PCM_X_QUEUE_DUPLICATE); } -UT_TEST(test_admission_replay_and_confirm_stay_frozen_after_transfer) +UT_TEST(test_master_blocker_wire_stage_rejects_reorder_conflict_and_wrong_source) { - PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXMasterAdmission replay; - PcmXEnqueuePayload request; - PcmXMasterTicketSlot *ticket; - PcmXTicketRef active; - PcmXTicketRef transfer; + PcmXMasterBlockerEntry entries[2]; + PcmXMasterBlockerSnapshot snapshot; + PcmXBlockerSetHeaderPayload begin; + PcmXBlockerSetHeaderPayload conflict_begin; + PcmXBlockerChunkPayload edge[2]; + PcmXBlockerChunkPayload conflict_edge; + ClusterLmdVertex blockers[2]; + ClusterLmdVertex noncanonical; + const uint64 source_session = UINT64_C(82701); init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - request = make_enqueue(make_wait_identity(713, 0, 22, UINT64_C(16001)), UINT64_C(1701), - UINT64_C(1)); - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(501)), + admit_active_probe(793, 0, 13, UINT64_C(8271), UINT64_C(7271), 1, &admission); + blockers[0] = make_blocker(2, 6, UINT64_C(8771)); + blockers[1] = make_blocker(2, 7, UINT64_C(8772)); + begin = make_blocker_header(&admission.ref, 1, blockers, lengthof(blockers)); + edge[0] = make_blocker_chunk(&admission.ref, 1, 0, blockers[0]); + edge[1] = make_blocker_chunk(&admission.ref, 1, 1, blockers[1]); + arm_blocker_probe(&admission.ref, 2, source_session); + + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, - request.identity.cluster_epoch, &active), + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[0], 3, source_session), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[1], 2, source_session), + PCM_X_QUEUE_NOT_READY); + conflict_begin = begin; + conflict_begin.set_crc32c++; + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&conflict_begin, 2, source_session), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[0], 2, source_session), PCM_X_QUEUE_OK); - commit_empty_blocker_graph(&active, UINT64_C(501)); - UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(501), &transfer), + conflict_begin = begin; + conflict_begin.nblockers = UINT32_MAX; + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&conflict_begin, 2, source_session, + NULL, 0, &snapshot), + PCM_X_QUEUE_INVALID); + conflict_edge = edge[0]; + conflict_edge.blocker.wait_seq++; + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&conflict_edge, 2, source_session), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&begin, 2, source_session, entries, + lengthof(entries), &snapshot), + PCM_X_QUEUE_NOT_READY); + noncanonical = make_blocker(1, 8, UINT64_C(8773)); + conflict_edge = make_blocker_chunk(&admission.ref, 1, 1, noncanonical); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&conflict_edge, 2, source_session), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[1], 2, source_session), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - UT_ASSERT_EQ(ticket->ref.grant_generation, transfer.grant_generation); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &replay), PCM_X_QUEUE_DUPLICATE); - UT_ASSERT_EQ(replay.ref.grant_generation, 0); - UT_ASSERT_EQ(replay.ref.handle.ticket_id, admission.ref.handle.ticket_id); - UT_ASSERT_EQ(replay.ref.handle.queue_generation, admission.ref.handle.queue_generation); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(501)), + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&begin, 3, source_session, entries, + lengthof(entries), &snapshot), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&begin, 2, source_session, entries, + lengthof(entries), &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[0], 2, source_session), PCM_X_QUEUE_DUPLICATE); - UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_ACTIVE_TRANSFER); - UT_ASSERT_EQ(ticket->ref.grant_generation, transfer.grant_generation); - UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&transfer), PCM_X_QUEUE_OK); - admission.ref = transfer; - drain_retire_and_detach_master(&admission); - assert_master_queue_baseline(header); + conflict_edge = edge[0]; + conflict_edge.blocker.wait_seq++; + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&conflict_edge, 2, source_session), + PCM_X_QUEUE_STALE); } -UT_TEST(test_master_application_replays_survive_terminal_successors) +UT_TEST(test_master_blocker_wire_stage_fences_transfer_and_cancel_until_commit) { - static const uint32 confirm_states[] - = { PCM_XT_QUEUED, PCM_XT_ACTIVE_PROBE, PCM_XT_ACTIVE_TRANSFER, - PCM_XT_COMPLETE, PCM_XT_CANCELLED, PCM_XT_RETIRE_CREDIT }; - static const uint32 complete_states[] = { PCM_XT_COMPLETE, PCM_XT_RETIRE_CREDIT }; - static const uint32 cancel_states[] = { PCM_XT_CANCELLED, PCM_XT_RETIRE_CREDIT }; + PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXEnqueuePayload request; - Size i; - - for (i = 0; i < lengthof(confirm_states); i++) { - PcmXMasterTagSlot tag_before; - PcmXMasterTagSlot *tag_slot; - PcmXMasterTicketSlot ticket_before; - PcmXMasterTicketSlot *ticket; - - init_active_pcm_x(UINT64_C(77)); - request = make_enqueue(make_wait_identity(719, 0, 26, UINT64_C(24001)), UINT64_C(2501), - UINT64_C(1)); - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; - tag_slot = &master_tag_slots(ClusterPcmXConvertShmem)[admission.tag_slot.slot_index]; - ticket->graph_generation = UINT64_C(701); - if (confirm_states[i] == PCM_XT_ACTIVE_TRANSFER || confirm_states[i] == PCM_XT_COMPLETE) - ticket->ref.grant_generation = UINT64_C(2701); - ticket->reliable.response_tombstone_mask = PCM_X_RESPONSE_TOMBSTONE_ADMIT_CONFIRM; - test_set_slot_state(&ticket->slot, confirm_states[i]); - ticket_before = *ticket; - tag_before = *tag_slot; - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(701)), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - UT_ASSERT(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(702)) - != PCM_X_QUEUE_OK); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - } - - for (i = 0; i < lengthof(complete_states); i++) { - PcmXMasterTagSlot tag_before; - PcmXMasterTagSlot *tag_slot; - PcmXMasterTicketSlot ticket_before; - PcmXMasterTicketSlot *ticket; - PcmXTicketRef completion_ref; - PcmXTicketRef stale_ref; - - init_active_pcm_x(UINT64_C(77)); - request = make_enqueue(make_wait_identity(720, 0, 27, UINT64_C(24002)), UINT64_C(2502), - UINT64_C(1)); - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; - tag_slot = &master_tag_slots(ClusterPcmXConvertShmem)[admission.tag_slot.slot_index]; - ticket->graph_generation = UINT64_C(703); - ticket->ref.grant_generation = UINT64_C(2702); - ticket->reliable.response_tombstone_mask = PCM_X_RESPONSE_TOMBSTONE_COMPLETE; - completion_ref = ticket->ref; - test_set_slot_state(&ticket->slot, complete_states[i]); - ticket_before = *ticket; - tag_before = *tag_slot; - UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&completion_ref), PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - stale_ref = completion_ref; - stale_ref.grant_generation++; - UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&stale_ref), PCM_X_QUEUE_STALE); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - } - - for (i = 0; i < lengthof(cancel_states); i++) { - PcmXMasterTagSlot tag_before; - PcmXMasterTagSlot *tag_slot; - PcmXMasterTicketSlot ticket_before; - PcmXMasterTicketSlot *ticket; - PcmXTicketRef stale_ref; - - init_active_pcm_x(UINT64_C(77)); - request = make_enqueue(make_wait_identity(721, 0, 28, UINT64_C(24003)), UINT64_C(2503), - UINT64_C(1)); - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; - tag_slot = &master_tag_slots(ClusterPcmXConvertShmem)[admission.tag_slot.slot_index]; - ticket->graph_generation = UINT64_C(704); - ticket->reliable.response_tombstone_mask = PCM_X_RESPONSE_TOMBSTONE_CANCEL; - test_set_slot_state(&ticket->slot, cancel_states[i]); - ticket_before = *ticket; - tag_before = *tag_slot; - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - stale_ref = admission.ref; - stale_ref.handle.queue_generation++; - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&stale_ref), PCM_X_QUEUE_STALE); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - } + PcmXMasterBlockerSnapshot empty_snapshot; + PcmXBlockerSetHeaderPayload begin; + PcmXTicketRef transfer; + ClusterLmdVertex blocker = make_blocker(2, 9, UINT64_C(8781)); + const uint64 source_session = UINT64_C(82801); - /* RETIRE_CREDIT alone is not outcome authority. */ init_active_pcm_x(UINT64_C(77)); - request - = make_enqueue(make_wait_identity(723, 0, 4, UINT64_C(24004)), UINT64_C(2504), UINT64_C(1)); - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - { - PcmXMasterTagSlot tag_before; - PcmXMasterTagSlot *tag_slot; - PcmXMasterTicketSlot ticket_before; - PcmXMasterTicketSlot *ticket; - PcmXTicketRef cross_complete; + header = ClusterPcmXConvertShmem; + admit_active_probe(794, 0, 14, UINT64_C(8281), UINT64_C(7281), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, NULL, 0, + blocker_set_crc32c(NULL, 0)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, NULL, 0, &empty_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&empty_snapshot, NULL, 0, UINT64_C(9281)), + PCM_X_QUEUE_OK); + begin = make_blocker_header(&admission.ref, 2, &blocker, 1); + arm_blocker_probe(&admission.ref, 2, source_session); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); - ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; - tag_slot = &master_tag_slots(ClusterPcmXConvertShmem)[admission.tag_slot.slot_index]; - ticket->ref.grant_generation = UINT64_C(2703); - ticket->reliable.response_tombstone_mask = PCM_X_RESPONSE_TOMBSTONE_CANCEL; - test_set_slot_state(&ticket->slot, PCM_XT_RETIRE_CREDIT); - cross_complete = ticket->ref; - ticket_before = *ticket; - tag_before = *tag_slot; - UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&cross_complete), PCM_X_QUEUE_BAD_STATE); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); - } + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9281), &transfer), + PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + PCM_X_QUEUE_DUPLICATE); init_active_pcm_x(UINT64_C(77)); - request - = make_enqueue(make_wait_identity(724, 0, 5, UINT64_C(24005)), UINT64_C(2505), UINT64_C(1)); - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - { - PcmXMasterTagSlot tag_before; - PcmXMasterTagSlot *tag_slot; - PcmXMasterTicketSlot ticket_before; - PcmXMasterTicketSlot *ticket; - - ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; - tag_slot = &master_tag_slots(ClusterPcmXConvertShmem)[admission.tag_slot.slot_index]; - ticket->reliable.response_tombstone_mask = PCM_X_RESPONSE_TOMBSTONE_COMPLETE; - test_set_slot_state(&ticket->slot, PCM_XT_RETIRE_CREDIT); - ticket_before = *ticket; - tag_before = *tag_slot; - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); - } + header = ClusterPcmXConvertShmem; + admit_active_probe(795, 0, 15, UINT64_C(8282), UINT64_C(7282), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, NULL, 0, + blocker_set_crc32c(NULL, 0)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, NULL, 0, &empty_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&empty_snapshot, NULL, 0, UINT64_C(9282)), + PCM_X_QUEUE_OK); + begin = make_blocker_header(&admission.ref, 2, &blocker, 1); + arm_blocker_probe(&admission.ref, 2, source_session); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + PCM_X_QUEUE_DUPLICATE); } -UT_TEST(test_delayed_pretransfer_cancel_fences_active_transfer) +UT_TEST(test_master_blocker_graph_commit_revalidates_exact_snapshot_and_generation) { PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXEnqueuePayload request; + PcmXMasterBlockerEntry entry; + PcmXMasterBlockerEntry stale_entry; + PcmXMasterBlockerSnapshot snapshot; + PcmXMasterBlockerSnapshot stale_snapshot; + PcmXMasterTicketSlot before; PcmXMasterTicketSlot *ticket; - PcmXRuntimeSnapshot snapshot; - PcmXTicketRef active; - PcmXTicketRef transfer; + ClusterLmdVertex blocker = make_blocker(1, 1, UINT64_C(8801)); init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - request = make_enqueue(make_wait_identity(714, 0, 23, UINT64_C(16002)), UINT64_C(1702), - UINT64_C(1)); - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(502)), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, - request.identity.cluster_epoch, &active), + admit_active_probe(792, 0, 8, UINT64_C(8203), UINT64_C(7203), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, &blocker, 1, + UINT32_C(0x701)), PCM_X_QUEUE_OK); - commit_empty_blocker_graph(&active, UINT64_C(502)); - UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(502), &transfer), + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &entry, 1, &snapshot), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RECOVERY_BLOCKED); - UT_ASSERT_EQ(ticket->ref.grant_generation, transfer.grant_generation); - snapshot = cluster_pcm_x_runtime_snapshot(); - UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + stale_snapshot = snapshot; + stale_snapshot.set_generation++; + before = *ticket; + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&stale_snapshot, &entry, 1, UINT64_C(9101)), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + + stale_entry = entry; + stale_entry.slot_ref.slot_generation++; + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &stale_entry, 1, UINT64_C(9101)), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, 0), + PCM_X_QUEUE_INVALID); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9101)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ticket->graph_generation, UINT64_C(9101)); + before = *ticket; + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9101)), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9100)), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9102)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ticket->graph_generation, UINT64_C(9102)); + UT_ASSERT_EQ(max_held_lwlock_count, 1); } -UT_TEST(test_recovery_blocked_runtime_cannot_promote_or_begin_transfer) +UT_TEST(test_master_transfer_requires_committed_exact_empty_blocker_set) { PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXMasterAdmission queued_admission; - PcmXEnqueuePayload request; - PcmXEnqueuePayload queued_request; + PcmXMasterBlockerEntry entry; + PcmXMasterBlockerSnapshot snapshot; + PcmXMasterTicketSlot before; PcmXMasterTicketSlot *ticket; - PcmXMasterTagSlot *tag_slot; - PcmXMasterTicketSlot ticket_before; - PcmXMasterTagSlot tag_before; - PcmXTicketRef active; PcmXTicketRef transfer; - PcmXSlotRef found; - uint64 queue_state; + ClusterLmdVertex blocker = make_blocker(1, 2, UINT64_C(8802)); init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - request = make_enqueue(make_wait_identity(716, 0, 25, UINT64_C(21001)), UINT64_C(2201), - UINT64_C(1)); - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(601)), + admit_active_probe(793, 0, 9, UINT64_C(8204), UINT64_C(7204), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, &blocker, 1, + UINT32_C(0x702)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &entry, 1, &snapshot), PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9201)), + PCM_X_QUEUE_OK); ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; - queue_state = tag_slot->queue_state_sequence; - ticket_before = *ticket; - tag_before = *tag_slot; - UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish( - request.identity.node_id, request.identity.cluster_epoch, - request.prehandle.sender_session_incarnation + 1), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, - request.identity.cluster_epoch, &active), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_QUEUED); - UT_ASSERT_EQ(tag_slot->active_index, PCM_X_INVALID_SLOT_INDEX); - UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_NOT_READY); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); + before = *ticket; + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9201), &transfer), + PCM_X_QUEUE_BAD_STATE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 2, NULL, 0, UINT32_C(0x703)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, NULL, 0, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ticket->graph_generation, 0); + before = *ticket; + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9201), &transfer), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, UINT64_C(9202)), + PCM_X_QUEUE_OK); + /* The direct replace fixture bypasses the 45-48 wire ACK. */ + ticket->reliable.last_response_opcode = PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT; + before = *ticket; + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9201), &transfer), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9202), &transfer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(transfer.grant_generation, ticket->reliable.state_sequence); +} + +UT_TEST(test_master_active_probe_cancel_requires_committed_exact_empty_blocker_set) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerEntry entry; + PcmXMasterBlockerSnapshot snapshot; + PcmXMasterBlockerSnapshot stale_snapshot; + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + ClusterLmdVertex blocker = make_blocker(1, 3, UINT64_C(8803)); init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - request = make_enqueue(make_wait_identity(717, 0, 25, UINT64_C(21002)), UINT64_C(2202), - UINT64_C(1)); - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(602)), + admit_active_probe(794, 0, 10, UINT64_C(8205), UINT64_C(7205), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, &blocker, 1, + UINT32_C(0x704)), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, - request.identity.cluster_epoch, &active), + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &entry, 1, &snapshot), PCM_X_QUEUE_OK); - commit_empty_blocker_graph(&active, UINT64_C(602)); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9401)), + PCM_X_QUEUE_OK); ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; - queue_state = tag_slot->queue_state_sequence; ticket_before = *ticket; tag_before = *tag_slot; - UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish( - request.identity.node_id, request.identity.cluster_epoch, - request.prehandle.sender_session_incarnation + 1), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(602), &transfer), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_ACTIVE_PROBE); - UT_ASSERT_EQ(ticket->ref.grant_generation, 0); - UT_ASSERT_EQ(tag_slot->active_index, admission.ticket_slot.slot_index); - UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - request = make_enqueue(make_wait_identity(718, 0, 25, UINT64_C(21003)), UINT64_C(2203), - UINT64_C(1)); - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); - arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, - admission.ref.identity.node_id); - arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, - admission.ref.identity.node_id); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; - ticket_before = *ticket; - tag_before = *tag_slot; - UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish( - request.identity.node_id, request.identity.cluster_epoch, - request.prehandle.sender_session_incarnation + 1), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_NOT_READY); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); + stale_snapshot = snapshot; UT_ASSERT_EQ( - cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_HANDLE, &admission.ref.handle, &found), - PCM_X_DIRECTORY_OK); + cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 2, NULL, 0, UINT32_C(0x705)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&stale_snapshot, &entry, 1, UINT64_C(9402)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, NULL, 0, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, UINT64_C(9402)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + UT_ASSERT_EQ(tag_slot->active_index, PCM_X_INVALID_SLOT_INDEX); +} + +UT_TEST(test_master_locator_defers_mutable_grant_exactness_to_domain_lock) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXTicketRef active; + PcmXTicketRef transfer; + PcmXQueueResult result; + uint64 grant_generation; + uint32 partition; - /* A blocked runtime has no FINAL_ACK proof and must retain transfer evidence. */ init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - request - = make_enqueue(make_wait_identity(733, 0, 2, UINT64_C(40001)), UINT64_C(4101), UINT64_C(1)); - queued_request - = make_enqueue(make_wait_identity(733, 1, 3, UINT64_C(40002)), UINT64_C(4102), UINT64_C(1)); + request = make_enqueue(make_wait_identity(743, 0, 9, UINT64_C(52001)), UINT64_C(5301), 1); bind_enqueue_peer(&request); - bind_enqueue_peer(&queued_request); UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&queued_request, &queued_admission), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(801)), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&queued_admission.ref, UINT64_C(802)), + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(401)), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, request.identity.cluster_epoch, &active), PCM_X_QUEUE_OK); - commit_empty_blocker_graph(&active, UINT64_C(801)); - UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(801), &transfer), + commit_empty_blocker_graph(&active, UINT64_C(401)); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(401), &transfer), PCM_X_QUEUE_OK); ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; - UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish( - request.identity.node_id, request.identity.cluster_epoch, - request.prehandle.sender_session_incarnation + 1), - PCM_X_QUEUE_STALE); - ticket_before = *ticket; - tag_before = *tag_slot; - UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&transfer), PCM_X_QUEUE_NOT_READY); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - UT_ASSERT_EQ( - test_slot_state(&master_ticket_slots(header)[admission.ticket_slot.slot_index].slot), - PCM_XT_ACTIVE_TRANSFER); - UT_ASSERT_EQ( - test_slot_state(&master_ticket_slots(header)[queued_admission.ticket_slot.slot_index].slot), - PCM_XT_QUEUED); - UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, - request.identity.cluster_epoch, &active), - PCM_X_QUEUE_NOT_READY); + grant_generation = transfer.grant_generation; + + /* Model a torn allocator-lock observation while the domain writer finishes + * publishing grant_generation. Locator lookup must use immutable fields; + * the full ref is accepted only after the master-domain interlock restores + * the stable value. */ + ticket->ref.grant_generation = UINT64CONST(0x3333333322222222); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&transfer.identity.tag)); + grant_interlock_lock = &header->master_locks[partition].lock; + grant_interlock_ticket = ticket; + grant_interlock_generation = grant_generation; + grant_interlock_armed = true; + result = cluster_pcm_x_master_complete_exact(&transfer); + UT_ASSERT_EQ(result, PCM_X_QUEUE_OK); + UT_ASSERT(!grant_interlock_armed); + grant_interlock_armed = false; + grant_interlock_lock = NULL; + grant_interlock_ticket = NULL; } -UT_TEST(test_recovery_blocked_runtime_refuses_ack_and_local_mutators) +UT_TEST(test_admission_replay_and_confirm_stay_frozen_after_transfer) { PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXMasterTicketSlot ticket_before; - PcmXMasterTicketSlot *ticket; + PcmXMasterAdmission replay; PcmXEnqueuePayload request; - PcmXRetirePayload retire_ack; - PcmXTerminalLegToken retire_token; - PcmXLocalCutoff cutoff; - PcmXLocalHandle leader; - PcmXLocalHandle output; - PcmXLocalMembershipSlot member_before; - PcmXLocalMembershipSlot *member; - PcmXLocalTagSlot tag_before; - PcmXLocalTagSlot *tag_slot; - PcmXWaitIdentity identity; - - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - request - = make_enqueue(make_wait_identity(734, 0, 2, UINT64_C(43001)), UINT64_C(4401), UINT64_C(1)); - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - ticket_before = *ticket; - UT_ASSERT_EQ( - cluster_pcm_x_peer_bind_ack_publish(0, request.identity.cluster_epoch, - request.prehandle.sender_session_incarnation + 1), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(901)), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - ticket_before = *ticket; - UT_ASSERT_EQ( - cluster_pcm_x_peer_bind_ack_publish(0, request.identity.cluster_epoch, - request.prehandle.sender_session_incarnation + 1), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact( - &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, admission.ref.identity.node_id, - request.prehandle.sender_session_incarnation), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + PcmXMasterTicketSlot *ticket; + PcmXTicketRef active; + PcmXTicketRef transfer; init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(713, 0, 22, UINT64_C(16001)), UINT64_C(1701), + UINT64_C(1)); bind_enqueue_peer(&request); UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); - arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, - admission.ref.identity.node_id); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, admission.ref.identity.node_id, - request.prehandle.sender_session_incarnation, &retire_token), + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(501)), PCM_X_QUEUE_OK); - retire_ack = make_retire_ack(&admission.ref, admission.ref.identity.node_id); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - ticket_before = *ticket; - UT_ASSERT_EQ( - cluster_pcm_x_peer_bind_ack_publish(0, request.identity.cluster_epoch, - request.prehandle.sender_session_incarnation + 1), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ( - cluster_pcm_x_master_retire_ack_exact(&retire_ack, admission.ref.identity.node_id, - request.prehandle.sender_session_incarnation), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - identity = make_wait_identity(735, 0, 2, UINT64_C(43002)); - bind_local_master(1, identity.cluster_epoch, UINT64_C(4501)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(4501), &leader), + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), PCM_X_QUEUE_OK); - tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; - member = &membership_slots(header)[leader.membership_slot.slot_index]; - tag_before = *tag_slot; - member_before = *member; - UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(1, identity.cluster_epoch, UINT64_C(4502)), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, &output), PCM_X_QUEUE_NOT_READY); - UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_NOT_READY); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - UT_ASSERT(memcmp(member, &member_before, sizeof(*member)) == 0); - - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - bind_local_master(1, identity.cluster_epoch, UINT64_C(4501)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(4501), &leader), + commit_empty_blocker_graph(&active, UINT64_C(501)); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(501), &transfer), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); - tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; - member = &membership_slots(header)[leader.membership_slot.slot_index]; - tag_before = *tag_slot; - member_before = *member; - UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(1, identity.cluster_epoch, UINT64_C(4502)), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&identity.tag, identity.cluster_epoch, - &cutoff, &output), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - UT_ASSERT(memcmp(member, &member_before, sizeof(*member)) == 0); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(ticket->ref.grant_generation, transfer.grant_generation); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &replay), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(replay.ref.grant_generation, 0); + UT_ASSERT_EQ(replay.ref.handle.ticket_id, admission.ref.handle.ticket_id); + UT_ASSERT_EQ(replay.ref.handle.queue_generation, admission.ref.handle.queue_generation); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(501)), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_ACTIVE_TRANSFER); + UT_ASSERT_EQ(ticket->ref.grant_generation, transfer.grant_generation); + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&transfer), PCM_X_QUEUE_OK); + admission.ref = transfer; + drain_retire_and_detach_master(&admission); + assert_master_queue_baseline(header); } -UT_TEST(test_master_cancel_is_exact_and_unlinks_middle_without_fifo_damage) +UT_TEST(test_master_application_replays_survive_terminal_successors) { - PcmXShmemHeader *header; - PcmXMasterAdmission admission[3]; - PcmXMasterTagSlot *tag_slot; - PcmXMasterTicketSlot *tickets; - PcmXTerminalLegToken drain_token; - PcmXTerminalLegToken retire_token; - PcmXRetirePayload retire_ack0; - PcmXRetirePayload retire_ack1; - PcmXTicketRef stale; - PcmXSlotRef found; - uint64 queue_state_before; - uint64 responder_session; - int i; + static const uint32 confirm_states[] + = { PCM_XT_QUEUED, PCM_XT_ACTIVE_PROBE, PCM_XT_ACTIVE_TRANSFER, + PCM_XT_COMPLETE, PCM_XT_CANCELLED, PCM_XT_RETIRE_CREDIT }; + static const uint32 complete_states[] = { PCM_XT_COMPLETE, PCM_XT_RETIRE_CREDIT }; + static const uint32 cancel_states[] = { PCM_XT_CANCELLED, PCM_XT_RETIRE_CREDIT }; + PcmXMasterAdmission admission; + PcmXEnqueuePayload request; + Size i; - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - for (i = 0; i < 3; i++) { - PcmXEnqueuePayload request - = make_enqueue(make_wait_identity(705, i, (uint32)(14 + i), UINT64_C(6001) + i), - UINT64_C(1001) + i, UINT64_C(1)); + for (i = 0; i < lengthof(confirm_states); i++) { + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(719, 0, 26, UINT64_C(24001)), UINT64_C(2501), + UINT64_C(1)); bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(200) + i), - PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(ClusterPcmXConvertShmem)[admission.tag_slot.slot_index]; + ticket->graph_generation = UINT64_C(701); + if (confirm_states[i] == PCM_XT_ACTIVE_TRANSFER || confirm_states[i] == PCM_XT_COMPLETE) + ticket->ref.grant_generation = UINT64_C(2701); + ticket->reliable.response_tombstone_mask = PCM_X_RESPONSE_TOMBSTONE_ADMIT_CONFIRM; + test_set_slot_state(&ticket->slot, confirm_states[i]); + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(701)), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(702)) + != PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); } - tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; - tickets = master_ticket_slots(header); - queue_state_before = tag_slot->queue_state_sequence; - stale = admission[1].ref; - stale.handle.queue_generation++; - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&stale), PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state_before); - UT_ASSERT_EQ(tag_slot->head_index, admission[0].ticket_slot.slot_index); - UT_ASSERT_EQ(tickets[admission[0].ticket_slot.slot_index].next_index, - admission[1].ticket_slot.slot_index); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state_before + 1); - queue_state_before = tag_slot->queue_state_sequence; - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_DUPLICATE); - UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state_before); - UT_ASSERT_EQ(test_slot_state(&tickets[admission[1].ticket_slot.slot_index].slot), - PCM_XT_CANCELLED); - UT_ASSERT_EQ(tickets[admission[0].ticket_slot.slot_index].next_index, - admission[2].ticket_slot.slot_index); - UT_ASSERT_EQ(tickets[admission[2].ticket_slot.slot_index].prev_index, - admission[0].ticket_slot.slot_index); - UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->queued_node_bitmap) & (UINT32_C(1) << 1), 0); - UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[1].ref), - PCM_X_QUEUE_NOT_READY); - responder_session = header->peer_frontiers[1].sender_session_incarnation; - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact(&admission[1].ref, - PCM_X_TERMINAL_LEG_DRAIN, 1, - responder_session, &drain_token), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact( - &admission[1].ref, PCM_X_TERMINAL_LEG_DRAIN, 0, - header->peer_frontiers[0].sender_session_incarnation), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact( - &admission[1].ref, PCM_X_TERMINAL_LEG_DRAIN, 1, responder_session), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(test_slot_state(&tickets[admission[1].ticket_slot.slot_index].slot), - PCM_XT_RETIRE_CREDIT); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact(&admission[1].ref, - PCM_X_TERMINAL_LEG_RETIRE, 1, - responder_session, &retire_token), - PCM_X_QUEUE_OK); - retire_ack0 = make_retire_ack(&admission[1].ref, 0); - retire_ack1 = make_retire_ack(&admission[1].ref, 1); - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact( - &retire_ack0, 0, header->peer_frontiers[0].sender_session_incarnation), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire_ack1, 1, responder_session), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[1].ref), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 3); - UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, - &admission[1].prehandle, &found), - PCM_X_DIRECTORY_OK); - UT_ASSERT(slot_refs_equal(found, admission[1].ticket_slot)); - - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[0].ref), PCM_X_QUEUE_OK); - drain_retire_and_detach_master(&admission[0]); - UT_ASSERT_EQ(header->fully_retired_ticket_id, 1); - UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[1].ref), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(header->fully_retired_ticket_id, 2); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[2].ref), PCM_X_QUEUE_OK); - drain_retire_and_detach_master(&admission[2]); - UT_ASSERT_EQ(header->fully_retired_ticket_id, 3); - assert_master_queue_baseline(header); -} - -UT_TEST(test_master_prehandle_cancel_replays_exactly_and_never_hits_reused_slot) -{ - PcmXShmemHeader *header; - PcmXMasterAdmission first; - PcmXMasterAdmission cancelled; - PcmXMasterAdmission replay; - PcmXMasterAdmission successor; - PcmXPrehandleCancelPayload cancel; - PcmXEnqueuePayload first_request; - PcmXEnqueuePayload successor_request; - PcmXMasterTicketSlot *successor_ticket; - PcmXMasterTicketSlot successor_before; - - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - first_request - = make_enqueue(make_wait_identity(706, 0, 10, UINT64_C(61001)), UINT64_C(6101), 1); - bind_enqueue_peer(&first_request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); - memset(&cancel, 0, sizeof(cancel)); - cancel.identity = first_request.identity; - cancel.prehandle = first_request.prehandle; - - UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&cancel, &cancelled), PCM_X_QUEUE_OK); - UT_ASSERT(ticket_refs_equal(&cancelled.ref, &first.ref)); - UT_ASSERT(memcmp(&cancelled.prehandle, &first.prehandle, sizeof(first.prehandle)) == 0); - UT_ASSERT_EQ(test_slot_state(&master_ticket_slots(header)[first.ticket_slot.slot_index].slot), - PCM_XT_CANCELLED); - UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&cancel, &replay), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(ticket_refs_equal(&replay.ref, &cancelled.ref)); - UT_ASSERT(memcmp(&replay.prehandle, &cancelled.prehandle, sizeof(replay.prehandle)) == 0); + for (i = 0; i < lengthof(complete_states); i++) { + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef completion_ref; + PcmXTicketRef stale_ref; - drain_retire_and_detach_master(&cancelled); + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(720, 0, 27, UINT64_C(24002)), UINT64_C(2502), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(ClusterPcmXConvertShmem)[admission.tag_slot.slot_index]; + ticket->graph_generation = UINT64_C(703); + ticket->ref.grant_generation = UINT64_C(2702); + ticket->reliable.response_tombstone_mask = PCM_X_RESPONSE_TOMBSTONE_COMPLETE; + completion_ref = ticket->ref; + test_set_slot_state(&ticket->slot, complete_states[i]); + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&completion_ref), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + stale_ref = completion_ref; + stale_ref.grant_generation++; + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&stale_ref), PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + } - successor_request - = make_enqueue(make_wait_identity(706, 0, 11, UINT64_C(61002)), UINT64_C(6101), 2); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&successor_request, &successor), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(successor.ticket_slot.slot_index, first.ticket_slot.slot_index); - UT_ASSERT(successor.ticket_slot.slot_generation != first.ticket_slot.slot_generation); - successor_ticket = &master_ticket_slots(header)[successor.ticket_slot.slot_index]; - successor_before = *successor_ticket; - UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&cancel, &replay), - PCM_X_QUEUE_RETIRED); - UT_ASSERT(memcmp(successor_ticket, &successor_before, sizeof(*successor_ticket)) == 0); -} + for (i = 0; i < lengthof(cancel_states); i++) { + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef stale_ref; -UT_TEST(test_master_prehandle_identity_alias_is_corruption_not_stale_cancel) -{ - PcmXMasterAdmission admission; - PcmXMasterAdmission output; - PcmXPrehandleCancelPayload alias; - PcmXEnqueuePayload request; + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(721, 0, 28, UINT64_C(24003)), UINT64_C(2503), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(ClusterPcmXConvertShmem)[admission.tag_slot.slot_index]; + ticket->graph_generation = UINT64_C(704); + ticket->reliable.response_tombstone_mask = PCM_X_RESPONSE_TOMBSTONE_CANCEL; + test_set_slot_state(&ticket->slot, cancel_states[i]); + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + stale_ref = admission.ref; + stale_ref.handle.queue_generation++; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&stale_ref), PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + } + /* RETIRE_CREDIT alone is not outcome authority. */ init_active_pcm_x(UINT64_C(77)); - request = make_enqueue(make_wait_identity(709, 0, 14, UINT64_C(64001)), UINT64_C(6401), 1); + request + = make_enqueue(make_wait_identity(723, 0, 4, UINT64_C(24004)), UINT64_C(2504), UINT64_C(1)); bind_enqueue_peer(&request); UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - memset(&alias, 0, sizeof(alias)); - alias.identity = request.identity; - alias.identity.wait_seq++; - alias.prehandle = request.prehandle; - UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&alias, &output), PCM_X_QUEUE_CORRUPT); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); - UT_ASSERT_EQ( - test_slot_state( - &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index].slot), - PCM_XT_ADMITTING); -} + { + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef cross_complete; -UT_TEST(test_master_prehandle_cancel_first_publishes_terminal_tombstone_before_late_enqueue) -{ - PcmXMasterAdmission cancelled; - PcmXMasterAdmission late_admission; - PcmXMasterAdmission replay; - PcmXPrehandleCancelPayload cancel; - PcmXEnqueuePayload request; - PcmXMasterTicketSlot *ticket; + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(ClusterPcmXConvertShmem)[admission.tag_slot.slot_index]; + ticket->ref.grant_generation = UINT64_C(2703); + ticket->reliable.response_tombstone_mask = PCM_X_RESPONSE_TOMBSTONE_CANCEL; + test_set_slot_state(&ticket->slot, PCM_XT_RETIRE_CREDIT); + cross_complete = ticket->ref; + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&cross_complete), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + } init_active_pcm_x(UINT64_C(77)); - request = make_enqueue(make_wait_identity(708, 0, 13, UINT64_C(63001)), UINT64_C(6301), 1); + request + = make_enqueue(make_wait_identity(724, 0, 5, UINT64_C(24005)), UINT64_C(2505), UINT64_C(1)); bind_enqueue_peer(&request); - memset(&cancel, 0, sizeof(cancel)); - cancel.identity = request.identity; - cancel.prehandle = request.prehandle; + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + { + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; - UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&cancel, &cancelled), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[cancelled.ticket_slot.slot_index]; - UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_CANCELLED); - UT_ASSERT_EQ(ClusterPcmXConvertShmem->peer_frontiers[0].next_expected_prehandle_sequence, 2); - UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); - UT_ASSERT_EQ( - master_tag_slots(ClusterPcmXConvertShmem)[cancelled.tag_slot.slot_index].head_index, - PCM_X_INVALID_SLOT_INDEX); - UT_ASSERT_EQ( - pg_atomic_read_u32(&master_tag_slots(ClusterPcmXConvertShmem)[cancelled.tag_slot.slot_index] - .queued_node_bitmap), - 0); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &late_admission), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(ticket_refs_equal(&late_admission.ref, &cancelled.ref)); - UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_CANCELLED); - UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&cancel, &replay), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(ticket_refs_equal(&replay.ref, &cancelled.ref)); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(ClusterPcmXConvertShmem)[admission.tag_slot.slot_index]; + ticket->reliable.response_tombstone_mask = PCM_X_RESPONSE_TOMBSTONE_COMPLETE; + test_set_slot_state(&ticket->slot, PCM_XT_RETIRE_CREDIT); + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + } } -UT_TEST(test_master_cancel_rejects_active_probe_without_mutation) +UT_TEST(test_delayed_pretransfer_cancel_fences_active_transfer) { + PcmXShmemHeader *header; PcmXMasterAdmission admission; PcmXEnqueuePayload request; - PcmXMasterTicketSlot before; PcmXMasterTicketSlot *ticket; + PcmXRuntimeSnapshot snapshot; PcmXTicketRef active; + PcmXTicketRef transfer; init_active_pcm_x(UINT64_C(77)); - request = make_enqueue(make_wait_identity(707, 0, 12, UINT64_C(62001)), UINT64_C(6201), 1); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(714, 0, 23, UINT64_C(16002)), UINT64_C(1702), + UINT64_C(1)); bind_enqueue_peer(&request); UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(6202)), + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(502)), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, request.identity.cluster_epoch, &active), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; - before = *ticket; - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_reversible_exact(&active), PCM_X_QUEUE_BAD_STATE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + commit_empty_blocker_graph(&active, UINT64_C(502)); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(502), &transfer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RECOVERY_BLOCKED); + UT_ASSERT_EQ(ticket->ref.grant_generation, transfer.grant_generation); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_RECOVERY_BLOCKED); } -UT_TEST(test_master_unlink_corruption_is_byte_stable_before_fail_closed) +UT_TEST(test_recovery_blocked_runtime_cannot_promote_or_begin_transfer) { PcmXShmemHeader *header; - PcmXMasterAdmission admission[3]; - PcmXMasterTagSlot tag_before; + PcmXMasterAdmission admission; + PcmXMasterAdmission queued_admission; + PcmXEnqueuePayload request; + PcmXEnqueuePayload queued_request; + PcmXMasterTicketSlot *ticket; PcmXMasterTagSlot *tag_slot; - PcmXMasterTicketSlot ticket_before[3]; - PcmXMasterTicketSlot *tickets; - int i; - - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - for (i = 0; i < 3; i++) { - PcmXEnqueuePayload request - = make_enqueue(make_wait_identity(748, i, (uint32)(20 + i), UINT64_C(61001) + i), - UINT64_C(6201) + i, UINT64_C(1)); + PcmXMasterTicketSlot ticket_before; + PcmXMasterTagSlot tag_before; + PcmXTicketRef active; + PcmXTicketRef transfer; + PcmXSlotRef found; + uint64 queue_state; - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(701) + i), - PCM_X_QUEUE_OK); - } - tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; - tickets = master_ticket_slots(header); - /* Corrupt only the successor backlink. Validation must not first splice - * the predecessor out of the queue and erase the original evidence. */ - tickets[admission[2].ticket_slot.slot_index].prev_index = PCM_X_INVALID_SLOT_INDEX; + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(716, 0, 25, UINT64_C(21001)), UINT64_C(2201), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(601)), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + queue_state = tag_slot->queue_state_sequence; + ticket_before = *ticket; tag_before = *tag_slot; - for (i = 0; i < 3; i++) - ticket_before[i] = tickets[admission[i].ticket_slot.slot_index]; - - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_CORRUPT); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish( + request.identity.node_id, request.identity.cluster_epoch, + request.prehandle.sender_session_incarnation + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_QUEUED); + UT_ASSERT_EQ(tag_slot->active_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - for (i = 0; i < 3; i++) - UT_ASSERT(memcmp(&tickets[admission[i].ticket_slot.slot_index], &ticket_before[i], - sizeof(ticket_before[i])) - == 0); -} - -UT_TEST(test_master_cancel_requires_state_exact_active_locator) -{ - PcmXShmemHeader *header; - PcmXMasterAdmission admission[2]; - PcmXMasterTagSlot tag_before; - PcmXMasterTagSlot *tag_slot; - PcmXMasterTicketSlot ticket_before[2]; - PcmXMasterTicketSlot *tickets; - PcmXEnqueuePayload request; - PcmXTicketRef active; - int i; - /* An ACTIVE_PROBE cannot be cancelled after its active locator is lost. */ init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - request = make_enqueue(make_wait_identity(750, 0, 2, UINT64_C(64001)), UINT64_C(6501), 1); + request = make_enqueue(make_wait_identity(717, 0, 25, UINT64_C(21002)), UINT64_C(2202), + UINT64_C(1)); bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[0]), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[0].ref, UINT64_C(801)), + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(602)), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, request.identity.cluster_epoch, &active), PCM_X_QUEUE_OK); - tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; - tickets = master_ticket_slots(header); - tag_slot->active_index = PCM_X_INVALID_SLOT_INDEX; - tag_slot->active_slot_generation = 0; + commit_empty_blocker_graph(&active, UINT64_C(602)); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + queue_state = tag_slot->queue_state_sequence; + ticket_before = *ticket; tag_before = *tag_slot; - ticket_before[0] = tickets[admission[0].ticket_slot.slot_index]; - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&active), PCM_X_QUEUE_CORRUPT); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish( + request.identity.node_id, request.identity.cluster_epoch, + request.prehandle.sender_session_incarnation + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(602), &transfer), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_ACTIVE_PROBE); + UT_ASSERT_EQ(ticket->ref.grant_generation, 0); + UT_ASSERT_EQ(tag_slot->active_index, admission.ticket_slot.slot_index); + UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - UT_ASSERT(memcmp(&tickets[admission[0].ticket_slot.slot_index], &ticket_before[0], - sizeof(ticket_before[0])) - == 0); - /* Conversely a QUEUED ticket cannot be named by the active locator. */ - init_active_pcm_x(UINT64_C(78)); + init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - for (i = 0; i < 2; i++) { - request = make_enqueue(make_wait_identity(751, i, (uint32)(2 + i), UINT64_C(64002) + i), - UINT64_C(6502) + i, 1); - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(802) + i), - PCM_X_QUEUE_OK); - } - tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; - tickets = master_ticket_slots(header); - tag_slot->active_index = admission[1].ticket_slot.slot_index; - tag_slot->active_slot_generation = admission[1].ticket_slot.slot_generation; + request = make_enqueue(make_wait_identity(718, 0, 25, UINT64_C(21003)), UINT64_C(2203), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, + admission.ref.identity.node_id); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, + admission.ref.identity.node_id); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + ticket_before = *ticket; tag_before = *tag_slot; - for (i = 0; i < 2; i++) - ticket_before[i] = tickets[admission[i].ticket_slot.slot_index]; - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_CORRUPT); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish( + request.identity.node_id, request.identity.cluster_epoch, + request.prehandle.sender_session_incarnation + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); - for (i = 0; i < 2; i++) - UT_ASSERT(memcmp(&tickets[admission[i].ticket_slot.slot_index], &ticket_before[i], - sizeof(ticket_before[i])) - == 0); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_HANDLE, &admission.ref.handle, &found), + PCM_X_DIRECTORY_OK); + + /* A blocked runtime has no FINAL_ACK proof and must retain transfer evidence. */ + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request + = make_enqueue(make_wait_identity(733, 0, 2, UINT64_C(40001)), UINT64_C(4101), UINT64_C(1)); + queued_request + = make_enqueue(make_wait_identity(733, 1, 3, UINT64_C(40002)), UINT64_C(4102), UINT64_C(1)); + bind_enqueue_peer(&request); + bind_enqueue_peer(&queued_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&queued_request, &queued_admission), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(801)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&queued_admission.ref, UINT64_C(802)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + commit_empty_blocker_graph(&active, UINT64_C(801)); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(801), &transfer), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish( + request.identity.node_id, request.identity.cluster_epoch, + request.prehandle.sender_session_incarnation + 1), + PCM_X_QUEUE_STALE); + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&transfer), PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT_EQ( + test_slot_state(&master_ticket_slots(header)[admission.ticket_slot.slot_index].slot), + PCM_XT_ACTIVE_TRANSFER); + UT_ASSERT_EQ( + test_slot_state(&master_ticket_slots(header)[queued_admission.ticket_slot.slot_index].slot), + PCM_XT_QUEUED); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), + PCM_X_QUEUE_NOT_READY); } -UT_TEST(test_terminal_ack_requires_prearmed_leg_and_is_byte_stable) +UT_TEST(test_recovery_blocked_runtime_refuses_ack_and_local_mutators) { PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXMasterTicketSlot before; + PcmXMasterTicketSlot ticket_before; PcmXMasterTicketSlot *ticket; PcmXEnqueuePayload request; PcmXRetirePayload retire_ack; + PcmXTerminalLegToken retire_token; + PcmXLocalCutoff cutoff; + PcmXLocalHandle leader; + PcmXLocalHandle output; + PcmXLocalMembershipSlot member_before; + PcmXLocalMembershipSlot *member; + PcmXLocalTagSlot tag_before; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity identity; init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; request - = make_enqueue(make_wait_identity(740, 0, 2, UINT64_C(50001)), UINT64_C(5101), UINT64_C(1)); + = make_enqueue(make_wait_identity(734, 0, 2, UINT64_C(43001)), UINT64_C(4401), UINT64_C(1)); bind_enqueue_peer(&request); UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - retire_ack = make_retire_ack(&admission.ref, request.identity.node_id); - before = *ticket; + ticket_before = *ticket; + UT_ASSERT_EQ( + cluster_pcm_x_peer_bind_ack_publish(0, request.identity.cluster_epoch, + request.prehandle.sender_session_incarnation + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(901)), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + ticket_before = *ticket; + UT_ASSERT_EQ( + cluster_pcm_x_peer_bind_ack_publish(0, request.identity.cluster_epoch, + request.prehandle.sender_session_incarnation + 1), + PCM_X_QUEUE_STALE); UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact( - &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, admission.ref.identity.node_id, request.prehandle.sender_session_incarnation), PCM_X_QUEUE_NOT_READY); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); - UT_ASSERT_EQ( - cluster_pcm_x_master_retire_ack_exact(&retire_ack, request.identity.node_id, - request.prehandle.sender_session_incarnation), - PCM_X_QUEUE_STALE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); -} - -UT_TEST(test_terminal_ack_wire_fields_are_exact_and_zero_side_effect) -{ - PcmXShmemHeader *header; - PcmXMasterAdmission admission; - PcmXMasterTicketSlot before; - PcmXMasterTicketSlot *ticket; - PcmXEnqueuePayload request; - PcmXRetirePayload retire_ack; - PcmXTicketRef resolved; - PcmXTicketRef work; - PcmXTerminalLegToken drain; - PcmXTerminalLegToken replay; - PcmXTerminalLegToken retire; + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - request - = make_enqueue(make_wait_identity(741, 0, 2, UINT64_C(50002)), UINT64_C(5102), UINT64_C(1)); bind_enqueue_peer(&request); UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_work_next(&work), PCM_X_QUEUE_OK); - UT_ASSERT(ticket_refs_equal(&work, &admission.ref)); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - retire_ack = make_retire_ack(&admission.ref, request.identity.node_id); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, - request.prehandle.sender_session_incarnation, &drain), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(drain.kind, PCM_X_TERMINAL_LEG_DRAIN); - UT_ASSERT_EQ(drain.expected_responder_node, request.identity.node_id); - UT_ASSERT_EQ(drain.expected_responder_session, request.prehandle.sender_session_incarnation); - UT_ASSERT(drain.state_sequence != 0); - UT_ASSERT(drain.drain_generation != 0); - UT_ASSERT_EQ(ticket->drain_generation, drain.drain_generation); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, - request.prehandle.sender_session_incarnation, &replay), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(memcmp(&replay, &drain, sizeof(replay)) == 0); - before = *ticket; - - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, 1, - drain.expected_responder_session), - PCM_X_QUEUE_STALE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, - drain.expected_responder_node, - drain.expected_responder_session + 1), - PCM_X_QUEUE_STALE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire_ack, drain.expected_responder_node, - drain.expected_responder_session), - PCM_X_QUEUE_STALE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); - - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, - drain.expected_responder_node, - drain.expected_responder_session), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); - UT_ASSERT_EQ(ticket->reliable.pending_opcode, 0); - UT_ASSERT_EQ(ticket->reliable.phase, 0); - UT_ASSERT_EQ(ticket->reliable.expected_responder_node, 0); - UT_ASSERT_EQ(ticket->reliable.expected_responder_session, 0); - + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, + admission.ref.identity.node_id); UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, request.identity.node_id, - request.prehandle.sender_session_incarnation, &retire), + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, admission.ref.identity.node_id, + request.prehandle.sender_session_incarnation, &retire_token), PCM_X_QUEUE_OK); - before = *ticket; - /* A closed DRAIN key replays without clearing the newly armed RETIRE leg. */ - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, - drain.expected_responder_node, - drain.expected_responder_session), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); - memset(&resolved, 0, sizeof(resolved)); + retire_ack = make_retire_ack(&admission.ref, admission.ref.identity.node_id); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + ticket_before = *ticket; UT_ASSERT_EQ( - cluster_pcm_x_master_retire_ack_resolve_exact(&retire_ack, retire.expected_responder_node, - retire.expected_responder_session, &resolved), - PCM_X_QUEUE_OK); - UT_ASSERT(ticket_refs_equal(&resolved, &admission.ref)); - before = *ticket; - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire_ack, retire.expected_responder_node, - retire.expected_responder_session), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); -} - -UT_TEST(test_retire_ack_uses_persistent_exact_key_index) -{ - PcmXShmemHeader *header; - PcmXMasterAdmission admission; - PcmXMasterTicketSlot before; - PcmXMasterTicketSlot *ticket; - PcmXEnqueuePayload request; - PcmXRetirePayload retire0; - PcmXRetirePayload retire1; - PcmXRetirePayload stale; - PcmXSlotRef found; - PcmXTerminalLegToken token; - uint64 session0; - uint64 session1 = UINT64_C(6102); + cluster_pcm_x_peer_bind_ack_publish(0, request.identity.cluster_epoch, + request.prehandle.sender_session_incarnation + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ( + cluster_pcm_x_master_retire_ack_exact(&retire_ack, admission.ref.identity.node_id, + request.prehandle.sender_session_incarnation), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - request - = make_enqueue(make_wait_identity(745, 0, 2, UINT64_C(50006)), UINT64_C(5106), UINT64_C(1)); - bind_enqueue_peer(&request); - session0 = request.prehandle.sender_session_incarnation; - UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(1, request.identity.cluster_epoch, session1), + identity = make_wait_identity(735, 0, 2, UINT64_C(43002)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(4501)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(4501), &leader), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - ticket->involved_nodes_bitmap |= UINT32_C(1) << 1; - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); - arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 0); - arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 1); - UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); - retire0 = make_retire_ack(&admission.ref, 0); - retire1 = make_retire_ack(&admission.ref, 1); - before = *ticket; - - /* A byte-exact ACK has no authority until arm publishes its shmem key. */ - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire0, 0, session0), PCM_X_QUEUE_STALE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire0, &found), - PCM_X_DIRECTORY_NOT_FOUND); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + member = &membership_slots(header)[leader.membership_slot.slot_index]; + tag_before = *tag_slot; + member_before = *member; + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(1, identity.cluster_epoch, UINT64_C(4502)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, &output), PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT(memcmp(member, &member_before, sizeof(*member)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0, session0, &token), + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, identity.cluster_epoch, UINT64_C(4501)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(4501), &leader), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire0, &found), - PCM_X_DIRECTORY_OK); - UT_ASSERT(slot_refs_equal(found, admission.ticket_slot)); - before = *ticket; - - stale = retire0; - stale.cluster_epoch++; - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&stale, 0, session0), PCM_X_QUEUE_STALE); - stale = retire0; - stale.master_session_incarnation++; - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&stale, 0, session0), PCM_X_QUEUE_STALE); - stale = retire0; - stale.retire_through_ticket_id++; - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&stale, 0, session0), PCM_X_QUEUE_STALE); - stale = retire0; - stale.sender_node = 1; - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&stale, 1, session1), PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire0, 1, session1), PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire0, 0, session0 + 1), + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + member = &membership_slots(header)[leader.membership_slot.slot_index]; + tag_before = *tag_slot; + member_before = *member; + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(1, identity.cluster_epoch, UINT64_C(4502)), PCM_X_QUEUE_STALE); - stale = retire0; - stale.flags = 1; - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&stale, 0, session0), PCM_X_QUEUE_INVALID); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&identity.tag, identity.cluster_epoch, + &cutoff, &output), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT(memcmp(member, &member_before, sizeof(*member)) == 0); +} - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire0, 0, session0), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 1, session1, &token), - PCM_X_QUEUE_OK); - before = *ticket; - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire0, 0, session0), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire1, 1, session1), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(ticket->retire_acked_nodes_bitmap, ticket->involved_nodes_bitmap); - UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire0, &found), - PCM_X_DIRECTORY_NOT_FOUND); - UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire1, &found), - PCM_X_DIRECTORY_NOT_FOUND); +UT_TEST(test_runtime_fail_closed_records_winning_site_only) +{ + char site[PCM_X_FAIL_CLOSED_SITE_LEN]; + char first[PCM_X_FAIL_CLOSED_SITE_LEN]; + + init_active_pcm_x(UINT64_C(77)); + /* Before any fuse the site is absent and the buffer is cleared. */ + site[0] = 'x'; + UT_ASSERT(!cluster_pcm_x_runtime_fail_closed_site(site, sizeof(site))); + UT_ASSERT_EQ(site[0], '\0'); + /* The macro records this file:line as the fusing arm. */ + cluster_pcm_x_runtime_fail_closed(); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT(cluster_pcm_x_runtime_fail_closed_site(first, sizeof(first))); + UT_ASSERT(strncmp(first, "test_cluster_pcm_x_convert.c:", 29) == 0); + /* A losing caller (runtime already blocked) must not clobber the site. */ + cluster_pcm_x_runtime_fail_closed(); + UT_ASSERT(cluster_pcm_x_runtime_fail_closed_site(site, sizeof(site))); + UT_ASSERT(strcmp(site, first) == 0); } -UT_TEST(test_early_retire_arm_does_not_publish_locator) +UT_TEST(test_master_debug_iterators_report_live_slots) { PcmXMasterAdmission admission; PcmXEnqueuePayload request; - PcmXRetirePayload retire; - PcmXSlotRef found; - PcmXTerminalLegToken token; + PcmXWaitIdentity local_identity; + PcmXLocalHandle local_leader; + Size cursor = 0; + Size index = 0; + char line[512]; init_active_pcm_x(UINT64_C(77)); - request = make_enqueue(make_wait_identity(749, 0, 13, UINT64_C(61001)), UINT64_C(6201), 1); + /* Nothing admitted yet: both iterators must report no occupied slots. */ + UT_ASSERT(!cluster_pcm_x_master_tag_debug_next(&cursor, &index, line, sizeof(line))); + cursor = 0; + UT_ASSERT(!cluster_pcm_x_master_ticket_debug_next(&cursor, &index, line, sizeof(line))); + request = make_enqueue(make_wait_identity(716, 0, 25, UINT64_C(21001)), UINT64_C(2201), + UINT64_C(1)); bind_enqueue_peer(&request); UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - retire = make_retire_ack(&admission.ref, 0); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0, - request.prehandle.sender_session_incarnation, &token), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire, &found), - PCM_X_DIRECTORY_NOT_FOUND); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + cursor = 0; + UT_ASSERT(cluster_pcm_x_master_tag_debug_next(&cursor, &index, line, sizeof(line))); + UT_ASSERT(strstr(line, "queued=") != NULL); + UT_ASSERT(!cluster_pcm_x_master_tag_debug_next(&cursor, &index, line, sizeof(line))); + cursor = 0; + UT_ASSERT(cluster_pcm_x_master_ticket_debug_next(&cursor, &index, line, sizeof(line))); + UT_ASSERT(strstr(line, "state=") != NULL); + UT_ASSERT(strstr(line, "leg_op=") != NULL); + + init_active_pcm_x(UINT64_C(78)); + local_identity = make_wait_identity(717, 0, 26, UINT64_C(21002)); + bind_local_master(1, local_identity.cluster_epoch, UINT64_C(4501)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&local_identity, 1, UINT64_C(4501), &local_leader), + PCM_X_QUEUE_OK); + cursor = 0; + UT_ASSERT(cluster_pcm_x_local_tag_debug_next(&cursor, &index, line, sizeof(line))); + UT_ASSERT(strstr(line, "blocker_gen=") != NULL); + UT_ASSERT(strstr(line, "blocker_op=") != NULL); + UT_ASSERT(strstr(line, "blocker_last=") != NULL); + UT_ASSERT(strstr(line, "blocker_count=") != NULL); + UT_ASSERT(strstr(line, "blocker_head=") != NULL); } -UT_TEST(test_terminal_outcome_mask_corruption_is_fail_closed) +UT_TEST(test_master_cancel_is_exact_and_unlinks_middle_without_fifo_damage) { - int mode; + PcmXShmemHeader *header; + PcmXMasterAdmission admission[3]; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *tickets; + PcmXTerminalLegToken drain_token; + PcmXTerminalLegToken retire_token; + PcmXRetirePayload retire_ack0; + PcmXRetirePayload retire_ack1; + PcmXTicketRef stale; + PcmXSlotRef found; + uint64 queue_state_before; + uint64 responder_session; + int i; - for (mode = 0; mode < 3; mode++) { - PcmXShmemHeader *header; - PcmXMasterAdmission admission; - PcmXMasterTicketSlot before; - PcmXMasterTicketSlot *ticket; - PcmXEnqueuePayload request; - PcmXTerminalLegToken token; - PcmXQueueResult result; + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + for (i = 0; i < 3; i++) { + PcmXEnqueuePayload request + = make_enqueue(make_wait_identity(705, i, (uint32)(14 + i), UINT64_C(6001) + i), + UINT64_C(1001) + i, UINT64_C(1)); - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - request = make_enqueue(make_wait_identity(742 + mode, 0, 2, UINT64_C(50003) + mode), - UINT64_C(5103) + mode, UINT64_C(1)); bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - if (mode == 0) - ticket->reliable.response_tombstone_mask &= ~PCM_X_RESPONSE_TOMBSTONE_CANCEL; - else if (mode == 1) - ticket->reliable.response_tombstone_mask |= PCM_X_RESPONSE_TOMBSTONE_COMPLETE; - else - ticket->reliable.response_tombstone_mask |= UINT64_C(1) << 63; - before = *ticket; - if (mode == 0) - result = cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, - request.prehandle.sender_session_incarnation, &token); - else - result = cluster_pcm_x_master_terminal_leg_ack_exact( - &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, - request.prehandle.sender_session_incarnation); - UT_ASSERT_EQ(result, PCM_X_QUEUE_CORRUPT); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(200) + i), + PCM_X_QUEUE_OK); } + tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; + tickets = master_ticket_slots(header); + queue_state_before = tag_slot->queue_state_sequence; + stale = admission[1].ref; + stale.handle.queue_generation++; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&stale), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state_before); + UT_ASSERT_EQ(tag_slot->head_index, admission[0].ticket_slot.slot_index); + UT_ASSERT_EQ(tickets[admission[0].ticket_slot.slot_index].next_index, + admission[1].ticket_slot.slot_index); + + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state_before + 1); + queue_state_before = tag_slot->queue_state_sequence; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state_before); + UT_ASSERT_EQ(test_slot_state(&tickets[admission[1].ticket_slot.slot_index].slot), + PCM_XT_CANCELLED); + UT_ASSERT_EQ(tickets[admission[0].ticket_slot.slot_index].next_index, + admission[2].ticket_slot.slot_index); + UT_ASSERT_EQ(tickets[admission[2].ticket_slot.slot_index].prev_index, + admission[0].ticket_slot.slot_index); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->queued_node_bitmap) & (UINT32_C(1) << 1), 0); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[1].ref), + PCM_X_QUEUE_NOT_READY); + responder_session = header->peer_frontiers[1].sender_session_incarnation; + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact(&admission[1].ref, + PCM_X_TERMINAL_LEG_DRAIN, 1, + responder_session, &drain_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact( + &admission[1].ref, PCM_X_TERMINAL_LEG_DRAIN, 0, + header->peer_frontiers[0].sender_session_incarnation), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact( + &admission[1].ref, PCM_X_TERMINAL_LEG_DRAIN, 1, responder_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&tickets[admission[1].ticket_slot.slot_index].slot), + PCM_XT_RETIRE_CREDIT); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact(&admission[1].ref, + PCM_X_TERMINAL_LEG_RETIRE, 1, + responder_session, &retire_token), + PCM_X_QUEUE_OK); + retire_ack0 = make_retire_ack(&admission[1].ref, 0); + retire_ack1 = make_retire_ack(&admission[1].ref, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact( + &retire_ack0, 0, header->peer_frontiers[0].sender_session_incarnation), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire_ack1, 1, responder_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[1].ref), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 3); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, + &admission[1].prehandle, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, admission[1].ticket_slot)); + + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[0].ref), PCM_X_QUEUE_OK); + drain_retire_and_detach_master(&admission[0]); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[1].ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 2); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[2].ref), PCM_X_QUEUE_OK); + drain_retire_and_detach_master(&admission[2]); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 3); + assert_master_queue_baseline(header); } -UT_TEST(test_terminal_bitmap_superset_is_fail_closed_before_arm_or_ack_mutation) +UT_TEST(test_master_terminal_work_scan_covers_younger_tickets) { - int mode; + PcmXMasterAdmission admission[3]; + PcmXTicketRef work; + int i; - for (mode = 0; mode < 2; mode++) { - PcmXShmemHeader *header; - PcmXMasterAdmission admission; - PcmXMasterTicketSlot before; - PcmXMasterTicketSlot *ticket; - PcmXEnqueuePayload request; - PcmXTerminalLegToken token; - PcmXQueueResult result; + init_active_pcm_x(UINT64_C(77)); + for (i = 0; i < 3; i++) { + PcmXEnqueuePayload request + = make_enqueue(make_wait_identity(707, i, (uint32)(24 + i), UINT64_C(6101) + i), + UINT64_C(1101) + i, UINT64_C(1)); - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - request = make_enqueue( - make_wait_identity(752 + mode, 0, (uint32)(2 + mode), UINT64_C(66001) + mode), - UINT64_C(6701) + mode, 1); bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - if (mode == 1) - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, - request.prehandle.sender_session_incarnation, &token), - PCM_X_QUEUE_OK); - /* Node 1 never participated. Neither arming nor consuming a valid - * node-0 ACK may let this impossible superset wedge in ACTIVE. */ - ticket->drained_nodes_bitmap = UINT32_C(1) << 1; - before = *ticket; - if (mode == 0) - result = cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, - request.prehandle.sender_session_incarnation, &token); - else - result = cluster_pcm_x_master_terminal_leg_ack_exact( - &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, token.expected_responder_node, - token.expected_responder_session); - UT_ASSERT_EQ(result, PCM_X_QUEUE_CORRUPT); - UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(300) + i), + PCM_X_QUEUE_OK); } + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[2].ref), PCM_X_QUEUE_OK); + /* Frontier priority: the plain scan still returns the oldest terminal. */ + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_work_next(&work), PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&work, &admission[1].ref)); + /* A stuck head must not hide younger terminal work from the retry tick: + * the head's own RETIRE preflight can be waiting for exactly the younger + * ticket's DRAIN evidence (cross-lane holder interlock). */ + UT_ASSERT_EQ( + cluster_pcm_x_master_terminal_work_next_after(admission[1].ref.handle.ticket_id, &work), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&work, &admission[2].ref)); + UT_ASSERT_EQ( + cluster_pcm_x_master_terminal_work_next_after(admission[2].ref.handle.ticket_id, &work), + PCM_X_QUEUE_NOT_FOUND); } -UT_TEST(test_terminal_detach_missing_retire_ack_is_retryable_not_ready) +UT_TEST(test_master_prehandle_cancel_replays_exactly_and_never_hits_reused_slot) { PcmXShmemHeader *header; - PcmXMasterAdmission admission; - PcmXMasterTagSlot tag_before; - PcmXMasterTagSlot *tag_slot; - PcmXMasterTicketSlot ticket_before; - PcmXMasterTicketSlot *ticket; - PcmXEnqueuePayload request; - uint64 session1 = UINT64_C(6802); + PcmXMasterAdmission first; + PcmXMasterAdmission cancelled; + PcmXMasterAdmission replay; + PcmXMasterAdmission successor; + PcmXPrehandleCancelPayload cancel; + PcmXEnqueuePayload first_request; + PcmXEnqueuePayload successor_request; + PcmXMasterTicketSlot *successor_ticket; + PcmXMasterTicketSlot successor_before; init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - request = make_enqueue(make_wait_identity(754, 0, 2, UINT64_C(66003)), UINT64_C(6801), 1); - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(1, request.identity.cluster_epoch, session1), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - ticket->involved_nodes_bitmap |= UINT32_C(1) << 1; - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); - arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 0); - arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 1); - arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0); - UT_ASSERT_EQ(ticket->retire_acked_nodes_bitmap, UINT32_C(1)); - tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; - ticket_before = *ticket; - tag_before = *tag_slot; + first_request + = make_enqueue(make_wait_identity(706, 0, 10, UINT64_C(61001)), UINT64_C(6101), 1); + bind_enqueue_peer(&first_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); + memset(&cancel, 0, sizeof(cancel)); + cancel.identity = first_request.identity; + cancel.prehandle = first_request.prehandle; - UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_NOT_READY); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&cancel, &cancelled), PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&cancelled.ref, &first.ref)); + UT_ASSERT(memcmp(&cancelled.prehandle, &first.prehandle, sizeof(first.prehandle)) == 0); + UT_ASSERT_EQ(test_slot_state(&master_ticket_slots(header)[first.ticket_slot.slot_index].slot), + PCM_XT_CANCELLED); + UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&cancel, &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(ticket_refs_equal(&replay.ref, &cancelled.ref)); + UT_ASSERT(memcmp(&replay.prehandle, &cancelled.prehandle, sizeof(replay.prehandle)) == 0); - arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 1); - UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_OK); - assert_master_queue_baseline(header); + drain_retire_and_detach_master(&cancelled); + + successor_request + = make_enqueue(make_wait_identity(706, 0, 11, UINT64_C(61002)), UINT64_C(6101), 2); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&successor_request, &successor), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(successor.ticket_slot.slot_index, first.ticket_slot.slot_index); + UT_ASSERT(successor.ticket_slot.slot_generation != first.ticket_slot.slot_generation); + successor_ticket = &master_ticket_slots(header)[successor.ticket_slot.slot_index]; + successor_before = *successor_ticket; + UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&cancel, &replay), + PCM_X_QUEUE_RETIRED); + UT_ASSERT(memcmp(successor_ticket, &successor_before, sizeof(*successor_ticket)) == 0); } -UT_TEST(test_terminal_retry_skips_drained_responder_while_next_leg_is_armed) +UT_TEST(test_master_prehandle_identity_alias_is_corruption_not_stale_cancel) { - PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXMasterTicketSlot *ticket; + PcmXMasterAdmission output; + PcmXPrehandleCancelPayload alias; PcmXEnqueuePayload request; - PcmXTerminalLegToken pending; - PcmXTerminalLegToken replay; - PcmXTerminalLegToken skipped; - PcmXRetirePayload retire_ack; - uint64 session0; - uint64 session1 = UINT64_C(6804); init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - request = make_enqueue(make_wait_identity(754, 0, 2, UINT64_C(66004)), UINT64_C(6803), 1); + request = make_enqueue(make_wait_identity(709, 0, 14, UINT64_C(64001)), UINT64_C(6401), 1); bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(1, request.identity.cluster_epoch, session1), - PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - ticket->involved_nodes_bitmap |= UINT32_C(1) << 1; - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); - session0 = header->peer_frontiers[0].sender_session_incarnation; - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 0, session0, &skipped), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, - PCM_X_TERMINAL_LEG_DRAIN, 0, session0), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 1, session1, &pending), - PCM_X_QUEUE_OK); - - /* The retry driver scans participants from node zero on every pass. A - * completed earlier responder must not hide the exact later leg behind - * BUSY, or the later DRAIN can never be resent. */ - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 0, session0, &skipped), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 1, session1, &replay), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(memcmp(&replay, &pending, sizeof(replay)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, - PCM_X_TERMINAL_LEG_DRAIN, 1, session1), - PCM_X_QUEUE_OK); + memset(&alias, 0, sizeof(alias)); + alias.identity = request.identity; + alias.identity.wait_seq++; + alias.prehandle = request.prehandle; + UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&alias, &output), PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ( + test_slot_state( + &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index].slot), + PCM_XT_ADMITTING); +} - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0, session0, &skipped), - PCM_X_QUEUE_OK); - retire_ack = make_retire_ack(&admission.ref, 0); - UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire_ack, 0, session0), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 1, session1, &pending), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0, session0, &skipped), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 1, session1, &replay), +UT_TEST(test_master_prehandle_cancel_first_publishes_terminal_tombstone_before_late_enqueue) +{ + PcmXMasterAdmission cancelled; + PcmXMasterAdmission late_admission; + PcmXMasterAdmission replay; + PcmXPrehandleCancelPayload cancel; + PcmXEnqueuePayload request; + PcmXMasterTicketSlot *ticket; + + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(708, 0, 13, UINT64_C(63001)), UINT64_C(6301), 1); + bind_enqueue_peer(&request); + memset(&cancel, 0, sizeof(cancel)); + cancel.identity = request.identity; + cancel.prehandle = request.prehandle; + + UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&cancel, &cancelled), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[cancelled.ticket_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_CANCELLED); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->peer_frontiers[0].next_expected_prehandle_sequence, 2); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); + UT_ASSERT_EQ( + master_tag_slots(ClusterPcmXConvertShmem)[cancelled.tag_slot.slot_index].head_index, + PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ( + pg_atomic_read_u32(&master_tag_slots(ClusterPcmXConvertShmem)[cancelled.tag_slot.slot_index] + .queued_node_bitmap), + 0); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &late_admission), PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(memcmp(&replay, &pending, sizeof(replay)) == 0); + UT_ASSERT(ticket_refs_equal(&late_admission.ref, &cancelled.ref)); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_CANCELLED); + UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&cancel, &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(ticket_refs_equal(&replay.ref, &cancelled.ref)); } -UT_TEST(test_terminal_detach_rejects_pending_leg) +UT_TEST(test_master_cancel_rejects_active_probe_without_mutation) { - PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXMasterTagSlot tag_before; - PcmXMasterTagSlot *tag_slot; - PcmXMasterTicketSlot ticket_before; - PcmXMasterTicketSlot *ticket; PcmXEnqueuePayload request; - PcmXTerminalLegToken drain; - PcmXTerminalLegToken retire; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef active; init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - request - = make_enqueue(make_wait_identity(744, 0, 2, UINT64_C(50005)), UINT64_C(5105), UINT64_C(1)); + request = make_enqueue(make_wait_identity(707, 0, 12, UINT64_C(62001)), UINT64_C(6201), 1); bind_enqueue_peer(&request); UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, - request.prehandle.sender_session_incarnation, &drain), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, - drain.expected_responder_node, - drain.expected_responder_session), + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(6202)), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( - &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, request.identity.node_id, - request.prehandle.sender_session_incarnation, &retire), + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), PCM_X_QUEUE_OK); - ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; - /* Even a corrupt early bitmap cannot bypass an armed application leg. */ - ticket->retire_acked_nodes_bitmap = ticket->involved_nodes_bitmap; - ticket_before = *ticket; - tag_before = *tag_slot; - UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_NOT_READY); - UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); - UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + before = *ticket; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_reversible_exact(&active), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); } -UT_TEST(test_master_terminal_detach_rejects_hot_link_and_drain_corruption) +UT_TEST(test_master_unlink_corruption_is_byte_stable_before_fail_closed) { PcmXShmemHeader *header; - PcmXMasterAdmission admission[2]; + PcmXMasterAdmission admission[3]; + PcmXMasterTagSlot tag_before; PcmXMasterTagSlot *tag_slot; - PcmXMasterTicketSlot *ticket; - PcmXTicketRef active; + PcmXMasterTicketSlot ticket_before[3]; + PcmXMasterTicketSlot *tickets; int i; init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - for (i = 0; i < 2; i++) { + for (i = 0; i < 3; i++) { PcmXEnqueuePayload request - = make_enqueue(make_wait_identity(745, i, (uint32)(7 + i), UINT64_C(55001) + i), - UINT64_C(5601) + i, 1); + = make_enqueue(make_wait_identity(748, i, (uint32)(20 + i), UINT64_C(61001) + i), + UINT64_C(6201) + i, UINT64_C(1)); bind_enqueue_peer(&request); UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(501) + i), + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(701) + i), PCM_X_QUEUE_OK); } - UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&admission[0].ref.identity.tag, - admission[0].ref.identity.cluster_epoch, - &active), - PCM_X_QUEUE_OK); - commit_empty_blocker_graph(&active, UINT64_C(501)); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&active), PCM_X_QUEUE_OK); - arm_and_ack_master_terminal_leg(&admission[0].ref, PCM_X_TERMINAL_LEG_DRAIN, 0); - arm_and_ack_master_terminal_leg(&admission[0].ref, PCM_X_TERMINAL_LEG_RETIRE, 0); tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; - ticket = &master_ticket_slots(header)[admission[0].ticket_slot.slot_index]; - UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 2); - UT_ASSERT_EQ(tag_slot->head_index, admission[1].ticket_slot.slot_index); - tag_slot->head_index = admission[0].ticket_slot.slot_index; - UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[0].ref), - PCM_X_QUEUE_CORRUPT); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); - UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 2); - UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); - - init_active_pcm_x(UINT64_C(78)); - header = ClusterPcmXConvertShmem; - { - PcmXEnqueuePayload request - = make_enqueue(make_wait_identity(746, 0, 9, UINT64_C(57001)), UINT64_C(5801), 1); + tickets = master_ticket_slots(header); + /* Corrupt only the successor backlink. Validation must not first splice + * the predecessor out of the queue and erase the original evidence. */ + tickets[admission[2].ticket_slot.slot_index].prev_index = PCM_X_INVALID_SLOT_INDEX; + tag_before = *tag_slot; + for (i = 0; i < 3; i++) + ticket_before[i] = tickets[admission[i].ticket_slot.slot_index]; - bind_enqueue_peer(&request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[0]), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[0].ref, UINT64_C(601)), - PCM_X_QUEUE_OK); - } - UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&admission[0].ref.identity.tag, - admission[0].ref.identity.cluster_epoch, - &active), - PCM_X_QUEUE_OK); - commit_empty_blocker_graph(&active, UINT64_C(601)); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&active), PCM_X_QUEUE_OK); - arm_and_ack_master_terminal_leg(&admission[0].ref, PCM_X_TERMINAL_LEG_DRAIN, 0); - arm_and_ack_master_terminal_leg(&admission[0].ref, PCM_X_TERMINAL_LEG_RETIRE, 0); - ticket = &master_ticket_slots(header)[admission[0].ticket_slot.slot_index]; - ticket->drained_nodes_bitmap = 0; - UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[0].ref), - PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_CORRUPT); UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); - UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); -} - -UT_TEST(test_master_terminal_detach_preserves_same_node_successor) -{ - PcmXShmemHeader *header; - PcmXMasterAdmission first; - PcmXMasterAdmission successor; - PcmXMasterTagSlot *tag_slot; - PcmXEnqueuePayload first_request; - PcmXEnqueuePayload successor_request; - uint32 node_bit = UINT32_C(1); - - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - first_request = make_enqueue(make_wait_identity(747, 0, 9, UINT64_C(59001)), UINT64_C(6001), 1); - successor_request - = make_enqueue(make_wait_identity(747, 0, 10, UINT64_C(59002)), UINT64_C(6001), 2); - bind_enqueue_peer(&first_request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&first.ref), PCM_X_QUEUE_OK); - arm_and_ack_master_terminal_leg(&first.ref, PCM_X_TERMINAL_LEG_DRAIN, - first.ref.identity.node_id); - arm_and_ack_master_terminal_leg(&first.ref, PCM_X_TERMINAL_LEG_RETIRE, - first.ref.identity.node_id); - - /* The node bitmap identifies the current hot ticket, not every resident - * terminal from that node. A successor may therefore reuse the node bit - * while the older ticket waits to detach in retirement order. */ - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&successor_request, &successor), PCM_X_QUEUE_OK); - tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; - UT_ASSERT(slot_refs_equal(first.tag_slot, successor.tag_slot)); - UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 2); - UT_ASSERT_EQ(tag_slot->head_index, successor.ticket_slot.slot_index); - UT_ASSERT_EQ(tag_slot->tail_index, successor.ticket_slot.slot_index); - UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->queued_node_bitmap), node_bit); - - UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&first.ref), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); - UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 1); - UT_ASSERT_EQ(tag_slot->head_index, successor.ticket_slot.slot_index); - UT_ASSERT_EQ(tag_slot->tail_index, successor.ticket_slot.slot_index); - UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->queued_node_bitmap), node_bit); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, 1); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); - - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&successor.ref), PCM_X_QUEUE_OK); - drain_retire_and_detach_master(&successor); - assert_master_queue_baseline(header); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + for (i = 0; i < 3; i++) + UT_ASSERT(memcmp(&tickets[admission[i].ticket_slot.slot_index], &ticket_before[i], + sizeof(ticket_before[i])) + == 0); } -UT_TEST(test_master_tag_survives_until_every_terminal_ticket_detaches) +UT_TEST(test_master_cancel_requires_state_exact_active_locator) { PcmXShmemHeader *header; - PcmXMasterAdmission admission[3]; + PcmXMasterAdmission admission[2]; + PcmXMasterTagSlot tag_before; PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before[2]; + PcmXMasterTicketSlot *tickets; + PcmXEnqueuePayload request; + PcmXTicketRef active; int i; + /* An ACTIVE_PROBE cannot be cancelled after its active locator is lost. */ init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - for (i = 0; i < 3; i++) { - PcmXEnqueuePayload request - = make_enqueue(make_wait_identity(712, i, (uint32)(18 + i), UINT64_C(14001) + i), - UINT64_C(1501) + i, UINT64_C(1)); + request = make_enqueue(make_wait_identity(750, 0, 2, UINT64_C(64001)), UINT64_C(6501), 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[0]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[0].ref, UINT64_C(801)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; + tickets = master_ticket_slots(header); + tag_slot->active_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->active_slot_generation = 0; + tag_before = *tag_slot; + ticket_before[0] = tickets[admission[0].ticket_slot.slot_index]; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&active), PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT(memcmp(&tickets[admission[0].ticket_slot.slot_index], &ticket_before[0], + sizeof(ticket_before[0])) + == 0); + /* Conversely a QUEUED ticket cannot be named by the active locator. */ + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + for (i = 0; i < 2; i++) { + request = make_enqueue(make_wait_identity(751, i, (uint32)(2 + i), UINT64_C(64002) + i), + UINT64_C(6502) + i, 1); bind_enqueue_peer(&request); UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(401) + i), + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(802) + i), PCM_X_QUEUE_OK); } tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; - UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 3); - for (i = 0; i < 3; i++) { - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[i].ref), PCM_X_QUEUE_OK); - arm_and_ack_master_terminal_leg(&admission[i].ref, PCM_X_TERMINAL_LEG_DRAIN, - admission[i].ref.identity.node_id); - arm_and_ack_master_terminal_leg(&admission[i].ref, PCM_X_TERMINAL_LEG_RETIRE, - admission[i].ref.identity.node_id); - } - UT_ASSERT_EQ(tag_slot->head_index, PCM_X_INVALID_SLOT_INDEX); - UT_ASSERT_EQ(tag_slot->tail_index, PCM_X_INVALID_SLOT_INDEX); - UT_ASSERT_EQ(tag_slot->active_index, PCM_X_INVALID_SLOT_INDEX); - UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->queued_node_bitmap), 0); - for (i = 0; i < 3; i++) { - UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[i].ref), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(header->fully_retired_ticket_id, (uint64)i + 1); - UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, (Size)(2 - i)); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, i == 2 ? 0 : 1); - } - assert_master_queue_baseline(header); + tickets = master_ticket_slots(header); + tag_slot->active_index = admission[1].ticket_slot.slot_index; + tag_slot->active_slot_generation = admission[1].ticket_slot.slot_generation; + tag_before = *tag_slot; + for (i = 0; i < 2; i++) + ticket_before[i] = tickets[admission[i].ticket_slot.slot_index]; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + for (i = 0; i < 2; i++) + UT_ASSERT(memcmp(&tickets[admission[i].ticket_slot.slot_index], &ticket_before[i], + sizeof(ticket_before[i])) + == 0); } -UT_TEST(test_last_terminal_detach_waits_for_staged_admission) +UT_TEST(test_terminal_ack_requires_prearmed_leg_and_is_byte_stable) { PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot before; PcmXMasterTicketSlot *ticket; - PcmXRuntimeSnapshot snapshot; PcmXEnqueuePayload request; + PcmXRetirePayload retire_ack; init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - request = make_enqueue(make_wait_identity(715, 0, 24, UINT64_C(19001)), UINT64_C(2001), - UINT64_C(1)); + request + = make_enqueue(make_wait_identity(740, 0, 2, UINT64_C(50001)), UINT64_C(5101), UINT64_C(1)); bind_enqueue_peer(&request); UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); - arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, - admission.ref.identity.node_id); - arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, - admission.ref.identity.node_id); - tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 1); - pg_atomic_write_u32(&tag_slot->admission_gate, 1); - pg_atomic_write_u32(&tag_slot->queued_node_bitmap, UINT32_C(1) << 1); - UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_BUSY); - UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 1); - UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); - UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); - snapshot = cluster_pcm_x_runtime_snapshot(); - UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_ACTIVE); + retire_ack = make_retire_ack(&admission.ref, request.identity.node_id); + before = *ticket; + + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ( + cluster_pcm_x_master_retire_ack_exact(&retire_ack, request.identity.node_id, + request.prehandle.sender_session_incarnation), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); } -UT_TEST(test_master_outstanding_counter_overflow_and_rollback_are_exact) +UT_TEST(test_terminal_ack_wire_fields_are_exact_and_zero_side_effect) { PcmXShmemHeader *header; - PcmXMasterAdmission first; - PcmXMasterAdmission second; - PcmXMasterTagSlot *tag_slot; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot before; PcmXMasterTicketSlot *ticket; - PcmXEnqueuePayload first_request; - PcmXEnqueuePayload second_request; - PcmXSlotRef found; - uint64 queue_state_before; + PcmXEnqueuePayload request; + PcmXRetirePayload retire_ack; + PcmXTicketRef resolved; + PcmXTicketRef work; + PcmXTerminalLegToken drain; + PcmXTerminalLegToken replay; + PcmXTerminalLegToken retire; init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - first_request - = make_enqueue(make_wait_identity(731, 0, 2, UINT64_C(36001)), UINT64_C(3701), UINT64_C(1)); - second_request - = make_enqueue(make_wait_identity(731, 1, 3, UINT64_C(36002)), UINT64_C(3702), UINT64_C(1)); - bind_enqueue_peer(&first_request); - bind_enqueue_peer(&second_request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); - tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; - queue_state_before = tag_slot->queue_state_sequence; - tag_slot->outstanding_ticket_count = SIZE_MAX; - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&second_request, &second), - PCM_X_QUEUE_COUNTER_EXHAUSTED); - UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, SIZE_MAX); - UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state_before); - UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->admission_gate), 1); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 2); - UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, - &second_request.prehandle, &found), - PCM_X_DIRECTORY_OK); - ticket = &master_ticket_slots(header)[found.slot_index]; - UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RESERVED_NONVISIBLE); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + request + = make_enqueue(make_wait_identity(741, 0, 2, UINT64_C(50002)), UINT64_C(5102), UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_work_next(&work), PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&work, &admission.ref)); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + retire_ack = make_retire_ack(&admission.ref, request.identity.node_id); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation, &drain), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(drain.kind, PCM_X_TERMINAL_LEG_DRAIN); + UT_ASSERT_EQ(drain.expected_responder_node, request.identity.node_id); + UT_ASSERT_EQ(drain.expected_responder_session, request.prehandle.sender_session_incarnation); + UT_ASSERT(drain.state_sequence != 0); + UT_ASSERT(drain.drain_generation != 0); + UT_ASSERT_EQ(ticket->drain_generation, drain.drain_generation); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation, &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&replay, &drain, sizeof(replay)) == 0); + /* Each exact re-arm is one retry-pump pass; leg_retry must count it. */ + UT_ASSERT_EQ(ticket->reliable.retry_count, 1); + before = *ticket; - /* Existing-tag allocator exhaustion is retryable and releases its gate. */ - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - bind_enqueue_peer(&first_request); - bind_enqueue_peer(&second_request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); - tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; - header->allocator[PCM_X_ALLOC_MASTER_TICKET].free_head = PCM_X_INVALID_SLOT_INDEX; - header->allocator[PCM_X_ALLOC_MASTER_TICKET].used - = header->layout.pools[PCM_X_POOL_MASTER_TICKET].capacity; - header->allocator[PCM_X_ALLOC_MASTER_TICKET].high_water - = header->layout.pools[PCM_X_POOL_MASTER_TICKET].capacity; - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&second_request, &second), - PCM_X_QUEUE_NO_CAPACITY); - UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 1); - UT_ASSERT_EQ(tag_slot->next_admission_sequence, 2); - UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->admission_gate), 0); - UT_ASSERT_EQ(header->next_ticket_id, 2); - UT_ASSERT_EQ(header->peer_frontiers[1].next_expected_prehandle_sequence, 1); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, 1, + drain.expected_responder_session), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, + drain.expected_responder_node, + drain.expected_responder_session + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire_ack, drain.expected_responder_node, + drain.expected_responder_session), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); - /* Existing-tag directory failure rolls its reserved ticket back exactly. */ - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - bind_enqueue_peer(&first_request); - bind_enqueue_peer(&second_request); - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); - tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; - { - PcmXDirectoryEntry *entries; - Size capacity; - Size i; - uint64 second_hash; + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, + drain.expected_responder_node, + drain.expected_responder_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); + UT_ASSERT_EQ(ticket->reliable.pending_opcode, 0); + UT_ASSERT_EQ(ticket->reliable.phase, 0); + UT_ASSERT_EQ(ticket->reliable.expected_responder_node, 0); + UT_ASSERT_EQ(ticket->reliable.expected_responder_session, 0); - entries = directory_entries(header, PCM_X_DIR_MASTER_TICKET_PREHANDLE, &capacity); - UT_ASSERT(cluster_pcm_x_directory_key_hash(PCM_X_DIR_MASTER_TICKET_PREHANDLE, - &second_request.prehandle, &second_hash)); - for (i = 0; i < capacity; i++) { - if (entries[i].state == PCM_X_DIRECTORY_OCCUPIED) - continue; - entries[i].state = PCM_X_DIRECTORY_OCCUPIED; - entries[i].key_hash = second_hash ^ UINT64_C(1); - entries[i].slot_index = first.ticket_slot.slot_index; - entries[i].slot_generation = first.ticket_slot.slot_generation; - } - } - UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&second_request, &second), - PCM_X_QUEUE_NO_CAPACITY); - UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 1); - UT_ASSERT_EQ(tag_slot->next_admission_sequence, 2); - UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->admission_gate), 0); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); - UT_ASSERT_EQ(header->next_ticket_id, 2); - UT_ASSERT_EQ(header->peer_frontiers[1].next_expected_prehandle_sequence, 1); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, request.identity.node_id, + request.prehandle.sender_session_incarnation, &retire), + PCM_X_QUEUE_OK); + before = *ticket; + /* A closed DRAIN key replays without clearing the newly armed RETIRE leg. */ + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, + drain.expected_responder_node, + drain.expected_responder_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + memset(&resolved, 0, sizeof(resolved)); + UT_ASSERT_EQ( + cluster_pcm_x_master_retire_ack_resolve_exact(&retire_ack, retire.expected_responder_node, + retire.expected_responder_session, &resolved), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&resolved, &admission.ref)); + before = *ticket; + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire_ack, retire.expected_responder_node, + retire.expected_responder_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); } -UT_TEST(test_master_detach_second_caller_is_stale_not_fail_closed) +UT_TEST(test_retire_ack_uses_persistent_exact_key_index) { PcmXShmemHeader *header; PcmXMasterAdmission admission; - PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot before; PcmXMasterTicketSlot *ticket; PcmXEnqueuePayload request; + PcmXRetirePayload retire0; + PcmXRetirePayload retire1; + PcmXRetirePayload stale; PcmXSlotRef found; + PcmXTerminalLegToken token; + uint64 session0; + uint64 session1 = UINT64_C(6102); init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; request - = make_enqueue(make_wait_identity(732, 0, 2, UINT64_C(38001)), UINT64_C(3901), UINT64_C(1)); + = make_enqueue(make_wait_identity(745, 0, 2, UINT64_C(50006)), UINT64_C(5106), UINT64_C(1)); bind_enqueue_peer(&request); + session0 = request.prehandle.sender_session_incarnation; + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(1, request.identity.cluster_epoch, session1), + PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); - arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, - admission.ref.identity.node_id); - arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, - admission.ref.identity.node_id); - tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; - detach_interlock_tag = tag_slot; - detach_interlock_ticket = ticket; - detach_interlock_armed = true; - UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_STALE); - UT_ASSERT(!detach_interlock_armed); - UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_DETACHING); - UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_DETACHING); - UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 0); - UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); + ticket->involved_nodes_bitmap |= UINT32_C(1) << 1; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 0); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 1); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); + retire0 = make_retire_ack(&admission.ref, 0); + retire1 = make_retire_ack(&admission.ref, 1); + before = *ticket; + + /* A byte-exact ACK has no authority until arm publishes its shmem key. */ + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire0, 0, session0), PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire0, &found), + PCM_X_DIRECTORY_NOT_FOUND); + + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0, session0, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire0, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, admission.ticket_slot)); + before = *ticket; + + stale = retire0; + stale.cluster_epoch++; + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&stale, 0, session0), PCM_X_QUEUE_STALE); + stale = retire0; + stale.master_session_incarnation++; + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&stale, 0, session0), PCM_X_QUEUE_STALE); + stale = retire0; + stale.retire_through_ticket_id++; + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&stale, 0, session0), PCM_X_QUEUE_STALE); + stale = retire0; + stale.sender_node = 1; + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&stale, 1, session1), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire0, 1, session1), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire0, 0, session0 + 1), + PCM_X_QUEUE_STALE); + stale = retire0; + stale.flags = 1; + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&stale, 0, session0), PCM_X_QUEUE_INVALID); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire0, 0, session0), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 1, session1, &token), + PCM_X_QUEUE_OK); + before = *ticket; + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire0, 0, session0), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire1, 1, session1), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ticket->retire_acked_nodes_bitmap, ticket->involved_nodes_bitmap); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire0, &found), + PCM_X_DIRECTORY_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire1, &found), + PCM_X_DIRECTORY_NOT_FOUND); +} + +UT_TEST(test_early_retire_arm_does_not_publish_locator) +{ + PcmXMasterAdmission admission; + PcmXEnqueuePayload request; + PcmXRetirePayload retire; + PcmXSlotRef found; + PcmXTerminalLegToken token; + + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(749, 0, 13, UINT64_C(61001)), UINT64_C(6201), 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + retire = make_retire_ack(&admission.ref, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0, + request.prehandle.sender_session_incarnation, &token), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire, &found), + PCM_X_DIRECTORY_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_terminal_outcome_mask_corruption_is_fail_closed) +{ + int mode; + + for (mode = 0; mode < 3; mode++) { + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXTerminalLegToken token; + PcmXQueueResult result; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(742 + mode, 0, 2, UINT64_C(50003) + mode), + UINT64_C(5103) + mode, UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + if (mode == 0) + ticket->reliable.response_tombstone_mask &= ~PCM_X_RESPONSE_TOMBSTONE_CANCEL; + else if (mode == 1) + ticket->reliable.response_tombstone_mask |= PCM_X_RESPONSE_TOMBSTONE_COMPLETE; + else + ticket->reliable.response_tombstone_mask |= UINT64_C(1) << 63; + before = *ticket; + if (mode == 0) + result = cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation, &token); + else + result = cluster_pcm_x_master_terminal_leg_ack_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation); + UT_ASSERT_EQ(result, PCM_X_QUEUE_CORRUPT); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + } +} + +UT_TEST(test_terminal_bitmap_superset_is_fail_closed_before_arm_or_ack_mutation) +{ + int mode; + + for (mode = 0; mode < 2; mode++) { + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXTerminalLegToken token; + PcmXQueueResult result; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue( + make_wait_identity(752 + mode, 0, (uint32)(2 + mode), UINT64_C(66001) + mode), + UINT64_C(6701) + mode, 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + if (mode == 1) + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation, &token), + PCM_X_QUEUE_OK); + /* Node 1 never participated. Neither arming nor consuming a valid + * node-0 ACK may let this impossible superset wedge in ACTIVE. */ + ticket->drained_nodes_bitmap = UINT32_C(1) << 1; + before = *ticket; + if (mode == 0) + result = cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation, &token); + else + result = cluster_pcm_x_master_terminal_leg_ack_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, token.expected_responder_node, + token.expected_responder_session); + UT_ASSERT_EQ(result, PCM_X_QUEUE_CORRUPT); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + } +} + +UT_TEST(test_terminal_detach_missing_retire_ack_is_retryable_not_ready) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + uint64 session1 = UINT64_C(6802); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(754, 0, 2, UINT64_C(66003)), UINT64_C(6801), 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(1, request.identity.cluster_epoch, session1), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + ticket->involved_nodes_bitmap |= UINT32_C(1) << 1; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 0); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 1); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0); + UT_ASSERT_EQ(ticket->retire_acked_nodes_bitmap, UINT32_C(1)); + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + ticket_before = *ticket; + tag_before = *tag_slot; + + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_OK); + assert_master_queue_baseline(header); +} + +UT_TEST(test_terminal_retry_skips_drained_responder_while_next_leg_is_armed) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXTerminalLegToken pending; + PcmXTerminalLegToken replay; + PcmXTerminalLegToken skipped; + PcmXRetirePayload retire_ack; + uint64 session0; + uint64 session1 = UINT64_C(6804); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(754, 0, 2, UINT64_C(66004)), UINT64_C(6803), 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(1, request.identity.cluster_epoch, session1), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + ticket->involved_nodes_bitmap |= UINT32_C(1) << 1; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + session0 = header->peer_frontiers[0].sender_session_incarnation; + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 0, session0, &skipped), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, + PCM_X_TERMINAL_LEG_DRAIN, 0, session0), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 1, session1, &pending), + PCM_X_QUEUE_OK); + + /* The retry driver scans participants from node zero on every pass. A + * completed earlier responder must not hide the exact later leg behind + * BUSY, or the later DRAIN can never be resent. */ + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 0, session0, &skipped), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 1, session1, &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&replay, &pending, sizeof(replay)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, + PCM_X_TERMINAL_LEG_DRAIN, 1, session1), + PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0, session0, &skipped), + PCM_X_QUEUE_OK); + retire_ack = make_retire_ack(&admission.ref, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire_ack, 0, session0), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 1, session1, &pending), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0, session0, &skipped), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 1, session1, &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&replay, &pending, sizeof(replay)) == 0); +} + +UT_TEST(test_terminal_detach_rejects_pending_leg) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXTerminalLegToken drain; + PcmXTerminalLegToken retire; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request + = make_enqueue(make_wait_identity(744, 0, 2, UINT64_C(50005)), UINT64_C(5105), UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation, &drain), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, + drain.expected_responder_node, + drain.expected_responder_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, request.identity.node_id, + request.prehandle.sender_session_incarnation, &retire), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + /* Even a corrupt early bitmap cannot bypass an armed application leg. */ + ticket->retire_acked_nodes_bitmap = ticket->involved_nodes_bitmap; + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); +} + +UT_TEST(test_master_terminal_detach_rejects_hot_link_and_drain_corruption) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission[2]; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef active; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + for (i = 0; i < 2; i++) { + PcmXEnqueuePayload request + = make_enqueue(make_wait_identity(745, i, (uint32)(7 + i), UINT64_C(55001) + i), + UINT64_C(5601) + i, 1); + + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(501) + i), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&admission[0].ref.identity.tag, + admission[0].ref.identity.cluster_epoch, + &active), + PCM_X_QUEUE_OK); + commit_empty_blocker_graph(&active, UINT64_C(501)); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&active), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission[0].ref, PCM_X_TERMINAL_LEG_DRAIN, 0); + arm_and_ack_master_terminal_leg(&admission[0].ref, PCM_X_TERMINAL_LEG_RETIRE, 0); + tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; + ticket = &master_ticket_slots(header)[admission[0].ticket_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 2); + UT_ASSERT_EQ(tag_slot->head_index, admission[1].ticket_slot.slot_index); + tag_slot->head_index = admission[0].ticket_slot.slot_index; + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[0].ref), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 2); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); + + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + { + PcmXEnqueuePayload request + = make_enqueue(make_wait_identity(746, 0, 9, UINT64_C(57001)), UINT64_C(5801), 1); + + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[0]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[0].ref, UINT64_C(601)), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&admission[0].ref.identity.tag, + admission[0].ref.identity.cluster_epoch, + &active), + PCM_X_QUEUE_OK); + commit_empty_blocker_graph(&active, UINT64_C(601)); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&active), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission[0].ref, PCM_X_TERMINAL_LEG_DRAIN, 0); + arm_and_ack_master_terminal_leg(&admission[0].ref, PCM_X_TERMINAL_LEG_RETIRE, 0); + ticket = &master_ticket_slots(header)[admission[0].ticket_slot.slot_index]; + ticket->drained_nodes_bitmap = 0; + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[0].ref), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); +} + +UT_TEST(test_master_terminal_detach_preserves_same_node_successor) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission first; + PcmXMasterAdmission successor; + PcmXMasterTagSlot *tag_slot; + PcmXEnqueuePayload first_request; + PcmXEnqueuePayload successor_request; + uint32 node_bit = UINT32_C(1); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + first_request = make_enqueue(make_wait_identity(747, 0, 9, UINT64_C(59001)), UINT64_C(6001), 1); + successor_request + = make_enqueue(make_wait_identity(747, 0, 10, UINT64_C(59002)), UINT64_C(6001), 2); + bind_enqueue_peer(&first_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&first.ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&first.ref, PCM_X_TERMINAL_LEG_DRAIN, + first.ref.identity.node_id); + arm_and_ack_master_terminal_leg(&first.ref, PCM_X_TERMINAL_LEG_RETIRE, + first.ref.identity.node_id); + + /* The node bitmap identifies the current hot ticket, not every resident + * terminal from that node. A successor may therefore reuse the node bit + * while the older ticket waits to detach in retirement order. */ + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&successor_request, &successor), PCM_X_QUEUE_OK); + tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; + UT_ASSERT(slot_refs_equal(first.tag_slot, successor.tag_slot)); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 2); + UT_ASSERT_EQ(tag_slot->head_index, successor.ticket_slot.slot_index); + UT_ASSERT_EQ(tag_slot->tail_index, successor.ticket_slot.slot_index); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->queued_node_bitmap), node_bit); + + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&first.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 1); + UT_ASSERT_EQ(tag_slot->head_index, successor.ticket_slot.slot_index); + UT_ASSERT_EQ(tag_slot->tail_index, successor.ticket_slot.slot_index); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->queued_node_bitmap), node_bit); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); + + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&successor.ref), PCM_X_QUEUE_OK); + drain_retire_and_detach_master(&successor); + assert_master_queue_baseline(header); +} + +UT_TEST(test_master_tag_survives_until_every_terminal_ticket_detaches) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission[3]; + PcmXMasterTagSlot *tag_slot; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + for (i = 0; i < 3; i++) { + PcmXEnqueuePayload request + = make_enqueue(make_wait_identity(712, i, (uint32)(18 + i), UINT64_C(14001) + i), + UINT64_C(1501) + i, UINT64_C(1)); + + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(401) + i), + PCM_X_QUEUE_OK); + } + tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 3); + for (i = 0; i < 3; i++) { + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[i].ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission[i].ref, PCM_X_TERMINAL_LEG_DRAIN, + admission[i].ref.identity.node_id); + arm_and_ack_master_terminal_leg(&admission[i].ref, PCM_X_TERMINAL_LEG_RETIRE, + admission[i].ref.identity.node_id); + } + UT_ASSERT_EQ(tag_slot->head_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(tag_slot->tail_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(tag_slot->active_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->queued_node_bitmap), 0); + for (i = 0; i < 3; i++) { + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[i].ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->fully_retired_ticket_id, (uint64)i + 1); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, (Size)(2 - i)); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, i == 2 ? 0 : 1); + } + assert_master_queue_baseline(header); +} + +UT_TEST(test_last_terminal_detach_waits_for_staged_admission) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXRuntimeSnapshot snapshot; + PcmXEnqueuePayload request; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(715, 0, 24, UINT64_C(19001)), UINT64_C(2001), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, + admission.ref.identity.node_id); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, + admission.ref.identity.node_id); + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 1); + pg_atomic_write_u32(&tag_slot->admission_gate, 1); + pg_atomic_write_u32(&tag_slot->queued_node_bitmap, UINT32_C(1) << 1); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 1); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_master_outstanding_counter_overflow_and_rollback_are_exact) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission first; + PcmXMasterAdmission second; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload first_request; + PcmXEnqueuePayload second_request; + PcmXSlotRef found; + uint64 queue_state_before; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + first_request + = make_enqueue(make_wait_identity(731, 0, 2, UINT64_C(36001)), UINT64_C(3701), UINT64_C(1)); + second_request + = make_enqueue(make_wait_identity(731, 1, 3, UINT64_C(36002)), UINT64_C(3702), UINT64_C(1)); + bind_enqueue_peer(&first_request); + bind_enqueue_peer(&second_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); + tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; + queue_state_before = tag_slot->queue_state_sequence; + tag_slot->outstanding_ticket_count = SIZE_MAX; + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&second_request, &second), + PCM_X_QUEUE_COUNTER_EXHAUSTED); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, SIZE_MAX); + UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state_before); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->admission_gate), 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 2); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, + &second_request.prehandle, &found), + PCM_X_DIRECTORY_OK); + ticket = &master_ticket_slots(header)[found.slot_index]; + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RESERVED_NONVISIBLE); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* Existing-tag allocator exhaustion is retryable and releases its gate. */ + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_enqueue_peer(&first_request); + bind_enqueue_peer(&second_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); + tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; + header->allocator[PCM_X_ALLOC_MASTER_TICKET].free_head = PCM_X_INVALID_SLOT_INDEX; + header->allocator[PCM_X_ALLOC_MASTER_TICKET].used + = header->layout.pools[PCM_X_POOL_MASTER_TICKET].capacity; + header->allocator[PCM_X_ALLOC_MASTER_TICKET].high_water + = header->layout.pools[PCM_X_POOL_MASTER_TICKET].capacity; + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&second_request, &second), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 1); + UT_ASSERT_EQ(tag_slot->next_admission_sequence, 2); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->admission_gate), 0); + UT_ASSERT_EQ(header->next_ticket_id, 2); + UT_ASSERT_EQ(header->peer_frontiers[1].next_expected_prehandle_sequence, 1); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + + /* Existing-tag directory failure rolls its reserved ticket back exactly. */ + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_enqueue_peer(&first_request); + bind_enqueue_peer(&second_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); + tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; + { + PcmXDirectoryEntry *entries; + Size capacity; + Size i; + uint64 second_hash; + + entries = directory_entries(header, PCM_X_DIR_MASTER_TICKET_PREHANDLE, &capacity); + UT_ASSERT(cluster_pcm_x_directory_key_hash(PCM_X_DIR_MASTER_TICKET_PREHANDLE, + &second_request.prehandle, &second_hash)); + for (i = 0; i < capacity; i++) { + if (entries[i].state == PCM_X_DIRECTORY_OCCUPIED) + continue; + entries[i].state = PCM_X_DIRECTORY_OCCUPIED; + entries[i].key_hash = second_hash ^ UINT64_C(1); + entries[i].slot_index = first.ticket_slot.slot_index; + entries[i].slot_generation = first.ticket_slot.slot_generation; + } + } + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&second_request, &second), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 1); + UT_ASSERT_EQ(tag_slot->next_admission_sequence, 2); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->admission_gate), 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); + UT_ASSERT_EQ(header->next_ticket_id, 2); + UT_ASSERT_EQ(header->peer_frontiers[1].next_expected_prehandle_sequence, 1); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_master_detach_second_caller_is_stale_not_fail_closed) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXSlotRef found; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request + = make_enqueue(make_wait_identity(732, 0, 2, UINT64_C(38001)), UINT64_C(3901), UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, + admission.ref.identity.node_id); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, + admission.ref.identity.node_id); + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + detach_interlock_tag = tag_slot; + detach_interlock_ticket = ticket; + detach_interlock_armed = true; + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_STALE); + UT_ASSERT(!detach_interlock_armed); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_DETACHING); + UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_DETACHING); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 0); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_HANDLE, &admission.ref.handle, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_local_composite_join_publishes_one_leader_and_ordered_followers) +{ + PcmXShmemHeader *header; + PcmXLocalHandle handles[3]; + PcmXLocalHandle duplicate; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *members; + PcmXSlotRef found; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, 0, UINT64_C(177)); + for (i = 0; i < 3; i++) { + PcmXWaitIdentity identity + = make_wait_identity(706, 0, (uint32)(17 + i), UINT64_C(7001) + i); + + identity.cluster_epoch = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &handles[i]), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(handles[i].local_sequence, (uint64)i + 1); + UT_ASSERT_EQ(handles[i].local_round, 1); + UT_ASSERT_EQ(handles[i].role, + i == 0 ? PCM_X_LOCAL_ROLE_NODE_LEADER : PCM_X_LOCAL_ROLE_FOLLOWER); + } + UT_ASSERT(slot_refs_equal(handles[0].tag_slot, handles[1].tag_slot)); + UT_ASSERT(slot_refs_equal(handles[0].tag_slot, handles[2].tag_slot)); + tag_slot = &local_tag_slots(header)[handles[0].tag_slot.slot_index]; + members = membership_slots(header); + UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_LIVE); + UT_ASSERT_EQ(tag_slot->head_index, handles[0].membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->tail_index, handles[2].membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->leader_index, handles[0].membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->membership_count, 3); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 0); + UT_ASSERT_EQ(test_slot_state(&members[handles[0].membership_slot.slot_index].slot), + PCM_XL_NODE_LEADER); + UT_ASSERT_EQ(test_slot_state(&members[handles[1].membership_slot.slot_index].slot), + PCM_XL_JOINED_NONWAITABLE); + UT_ASSERT_EQ(members[handles[0].membership_slot.slot_index].next_index, + handles[1].membership_slot.slot_index); + UT_ASSERT_EQ(members[handles[1].membership_slot.slot_index].next_index, + handles[2].membership_slot.slot_index); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 3); + + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &handles[0].identity.tag, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, handles[0].tag_slot)); + for (i = 0; i < 3; i++) { + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_WAIT, &handles[i].identity, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, handles[i].membership_slot)); + } + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&handles[1].identity, 1, UINT64_C(177), &duplicate), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(slot_refs_equal(duplicate.membership_slot, handles[1].membership_slot)); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 3); +} + +UT_TEST(test_local_follower_wfg_publish_is_nonwaitable_then_exact) +{ + PcmXLocalHandle leader; + PcmXLocalHandle follower; + PcmXLocalFollowerWfgSnapshot snapshot; + PcmXLocalFollowerWfgSnapshot stale; + PcmXLocalMembershipSlot *members; + PcmXWaitIdentity identity; + + init_active_pcm_x(UINT64_C(77)); + bind_local_master(1, 0, UINT64_C(177)); + identity = make_wait_identity(806, 0, 17, UINT64_C(80601)); + identity.cluster_epoch = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + PCM_X_QUEUE_OK); + identity = make_wait_identity(806, 0, 18, UINT64_C(80602)); + identity.cluster_epoch = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower), + PCM_X_QUEUE_OK); + members = membership_slots(ClusterPcmXConvertShmem); + UT_ASSERT_EQ(test_slot_state(&members[follower.membership_slot.slot_index].slot), + PCM_XL_JOINED_NONWAITABLE); + + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&snapshot.waiter, &follower, sizeof(follower)) == 0); + UT_ASSERT(memcmp(&snapshot.blocker, &leader.identity, sizeof(leader.identity)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&snapshot, UINT64_C(901)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&members[follower.membership_slot.slot_index].slot), + PCM_XL_WAITABLE_FOLLOWER); + UT_ASSERT_EQ(members[follower.membership_slot.slot_index].graph_generation, UINT64_C(901)); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&snapshot, UINT64_C(901)), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(902)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(901)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&members[follower.membership_slot.slot_index].slot), + PCM_XL_JOINED_NONWAITABLE); + UT_ASSERT_EQ(members[follower.membership_slot.slot_index].graph_generation, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(901)), + PCM_X_QUEUE_DUPLICATE); + + stale = snapshot; + stale.leader_slot_generation++; + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&stale, UINT64_C(902)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(members[follower.membership_slot.slot_index].graph_generation, 0); +} + +UT_TEST(test_local_writer_claim_runs_closed_cohort_and_blocks_next_round) +{ + PcmXLocalHandle leader; + PcmXLocalHandle follower; + PcmXLocalHandle late; + PcmXLocalFollowerWfgSnapshot wfg; + PcmXLocalWriterClaim claim; + PcmXLocalWriterClaim released; + PcmXLocalWriterClaim stale; + PcmXLocalCutoff cutoff; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity identity; + PcmXWaitIdentity successor; + + init_active_pcm_x(UINT64_C(77)); + bind_local_master(1, UINT64_C(9), UINT64_C(177)); + identity = make_wait_identity(807, 0, 19, UINT64_C(80701)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + PCM_X_QUEUE_OK); + identity = make_wait_identity(807, 0, 20, UINT64_C(80702)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(903)), + PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->active_writer_index, leader.membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->active_writer_slot_generation, leader.membership_slot.slot_generation); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &stale), PCM_X_QUEUE_BUSY); + + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + identity = make_wait_identity(807, 0, 21, UINT64_C(80703)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &late), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(late.local_round, cutoff.next_round); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&late, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(909)), + PCM_X_QUEUE_OK); + released = claim; + memset(&successor, 0x7f, sizeof(successor)); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&successor, &follower.identity, sizeof(successor)) == 0); + UT_ASSERT_EQ(tag_slot->active_writer_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &claim), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(903)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &claim), PCM_X_QUEUE_OK); + memset(&successor, 0x7f, sizeof(successor)); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&successor, &(PcmXWaitIdentity){ 0 }, sizeof(successor)) == 0); + + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &claim), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&late, &claim), PCM_X_QUEUE_BARRIER_CLOSED); + stale = released; + stale.claim_generation++; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&stale), PCM_X_QUEUE_STALE); +} + +UT_TEST(test_local_writer_claim_completion_is_fifo_and_one_shot) +{ + PcmXLocalHandle leader; + PcmXLocalHandle follower1; + PcmXLocalHandle follower2; + PcmXLocalFollowerWfgSnapshot wfg; + PcmXLocalWriterClaim aborted; + PcmXLocalWriterClaim busy; + PcmXLocalWriterClaim claim; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity identity; + PcmXWaitIdentity successor; + uint32 state_flags; + + init_active_pcm_x(UINT64_C(77)); + bind_local_master(1, UINT64_C(9), UINT64_C(177)); + identity = make_wait_identity(809, 0, 24, UINT64_C(80901)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + PCM_X_QUEUE_OK); + identity = make_wait_identity(809, 0, 25, UINT64_C(80902)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower1), + PCM_X_QUEUE_OK); + identity = make_wait_identity(809, 0, 26, UINT64_C(80903)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower2), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower1, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(904)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower2, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(905)), + PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower2, &busy), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + aborted = claim; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_abort_exact(&claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + UT_ASSERT(claim.claim_generation > aborted.claim_generation); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_abort_exact(&aborted), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower1, &busy), PCM_X_QUEUE_BUSY); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; + state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); + pg_atomic_write_u32(&tag_slot->slot.state_flags, + state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); + memset(&successor, 0x7f, sizeof(successor)); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_BUSY); + UT_ASSERT(memcmp(&successor, &(PcmXWaitIdentity){ 0 }, sizeof(successor)) == 0); + UT_ASSERT_EQ(tag_slot->active_writer_index, leader.membership_slot.slot_index); UT_ASSERT_EQ( - cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_HANDLE, &admission.ref.handle, &found), - PCM_X_DIRECTORY_OK); + test_slot_flags( + &membership_slots(ClusterPcmXConvertShmem)[leader.membership_slot.slot_index].slot) + & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE, + 0); UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + pg_atomic_write_u32(&tag_slot->slot.state_flags, state_flags); + memset(&successor, 0, sizeof(successor)); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&successor, &follower1.identity, sizeof(successor)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower2, UINT64_C(905)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower2, &busy), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower1, UINT64_C(904)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower1, &claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower2, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&wfg.blocker, &follower1.identity, sizeof(follower1.identity)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(906)), + PCM_X_QUEUE_OK); + memset(&successor, 0, sizeof(successor)); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&successor, &follower2.identity, sizeof(successor)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower1, &claim), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower1, NULL), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower2, UINT64_C(906)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower1, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower1), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower2, &claim), PCM_X_QUEUE_OK); + memset(&successor, 0x7f, sizeof(successor)); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&successor, &(PcmXWaitIdentity){ 0 }, sizeof(successor)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower2, &claim), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower2, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower2), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); } -UT_TEST(test_local_composite_join_publishes_one_leader_and_ordered_followers) +UT_TEST(test_writer_and_holder_owner_exit_retry_preserves_exact_evidence) { - PcmXShmemHeader *header; - PcmXLocalHandle handles[3]; - PcmXLocalHandle duplicate; + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHandle leader; + PcmXLocalWriterClaim claim; PcmXLocalTagSlot *tag_slot; - PcmXLocalMembershipSlot *members; - PcmXSlotRef found; - int i; + PcmXLocalTagSlot tag_before; + PcmXLocalMembershipSlot holder_before; + PcmXLocalMembershipSlot writer_before; + PcmXWaitIdentity identity; + PcmXWaitIdentity successor; + LWLock content_lock; + uint32 state_flags; + Size holder_used_before; + const uint64 master_session = UINT64_C(8177); + + init_active_pcm_x(UINT64_C(177)); + bind_local_master(1, UINT64_C(9), master_session); + holder_key = make_local_holder_key(811, 0, 27, UINT64_C(81101), 1); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + identity = make_wait_identity(811, 0, 28, UINT64_C(81102)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; + holder_used_before = ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used; + memset(&content_lock, 0, sizeof(content_lock)); + state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); + pg_atomic_write_u32(&tag_slot->slot.state_flags, + state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); + memset(&successor, 0x7f, sizeof(successor)); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_exceptional_detach_exact(&holder, &content_lock), + PCM_X_QUEUE_GATE_RETRY); + UT_ASSERT_EQ(tag_slot->active_writer_index, leader.membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->active_holder_head_index, holder.holder_slot.slot_index); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, + holder_used_before); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + pg_atomic_write_u32(&tag_slot->slot.state_flags, state_flags); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_exceptional_detach_exact(&holder, &content_lock), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(tag_slot->active_writer_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(tag_slot->active_holder_head_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, + holder_used_before - 1); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + + /* A different runtime may never consume either exact ledger. Both APIs + * preserve byte-identical shared evidence for deferred recovery. */ + init_active_pcm_x(UINT64_C(178)); + bind_local_master(1, UINT64_C(9), master_session + 1); + holder_key = make_local_holder_key(812, 0, 27, UINT64_C(81201), 1); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + identity = make_wait_identity(812, 0, 28, UINT64_C(81202)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session + 1, &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; + tag_before = *tag_slot; + holder_before = membership_slots(ClusterPcmXConvertShmem)[holder.holder_slot.slot_index]; + writer_before = membership_slots(ClusterPcmXConvertShmem)[leader.membership_slot.slot_index]; + cluster_pcm_x_runtime_fail_closed(); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_exceptional_detach_exact(&holder, &content_lock), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(&tag_before, tag_slot, sizeof(tag_before)) == 0); + UT_ASSERT(memcmp(&holder_before, + &membership_slots(ClusterPcmXConvertShmem)[holder.holder_slot.slot_index], + sizeof(holder_before)) + == 0); + UT_ASSERT(memcmp(&writer_before, + &membership_slots(ClusterPcmXConvertShmem)[leader.membership_slot.slot_index], + sizeof(writer_before)) + == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_local_wfg_rejects_completed_blocker_semantic_aba) +{ + PcmXLocalHandle leader; + PcmXLocalHandle follower; + PcmXLocalFollowerWfgSnapshot snapshot; + PcmXLocalWriterClaim claim; + PcmXWaitIdentity identity; init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - bind_local_master(1, 0, UINT64_C(177)); - for (i = 0; i < 3; i++) { - PcmXWaitIdentity identity - = make_wait_identity(706, 0, (uint32)(17 + i), UINT64_C(7001) + i); + bind_local_master(1, UINT64_C(9), UINT64_C(177)); + identity = make_wait_identity(810, 0, 27, UINT64_C(81001)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + PCM_X_QUEUE_OK); + identity = make_wait_identity(810, 0, 28, UINT64_C(81002)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&snapshot, UINT64_C(906)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &snapshot), + PCM_X_QUEUE_BUSY); +} - identity.cluster_epoch = 0; - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &handles[i]), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(handles[i].local_sequence, (uint64)i + 1); - UT_ASSERT_EQ(handles[i].local_round, 1); - UT_ASSERT_EQ(handles[i].role, - i == 0 ? PCM_X_LOCAL_ROLE_NODE_LEADER : PCM_X_LOCAL_ROLE_FOLLOWER); - } - UT_ASSERT(slot_refs_equal(handles[0].tag_slot, handles[1].tag_slot)); - UT_ASSERT(slot_refs_equal(handles[0].tag_slot, handles[2].tag_slot)); - tag_slot = &local_tag_slots(header)[handles[0].tag_slot.slot_index]; - members = membership_slots(header); - UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_LIVE); - UT_ASSERT_EQ(tag_slot->head_index, handles[0].membership_slot.slot_index); - UT_ASSERT_EQ(tag_slot->tail_index, handles[2].membership_slot.slot_index); - UT_ASSERT_EQ(tag_slot->leader_index, handles[0].membership_slot.slot_index); - UT_ASSERT_EQ(tag_slot->membership_count, 3); - UT_ASSERT_EQ(tag_slot->closed_round_member_count, 0); - UT_ASSERT_EQ(test_slot_state(&members[handles[0].membership_slot.slot_index].slot), - PCM_XL_NODE_LEADER); - UT_ASSERT_EQ(test_slot_state(&members[handles[1].membership_slot.slot_index].slot), - PCM_XL_JOINED_NONWAITABLE); - UT_ASSERT_EQ(members[handles[0].membership_slot.slot_index].next_index, - handles[1].membership_slot.slot_index); - UT_ASSERT_EQ(members[handles[1].membership_slot.slot_index].next_index, - handles[2].membership_slot.slot_index); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 1); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 3); +UT_TEST(test_local_cancel_never_unlinks_an_active_writer) +{ + PcmXLocalHandle leader; + PcmXLocalHandle follower; + PcmXLocalHandle follower2; + PcmXLocalFollowerWfgSnapshot wfg; + PcmXLocalWriterClaim claim; + PcmXWaitIdentity identity; - UT_ASSERT_EQ( - cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &handles[0].identity.tag, &found), - PCM_X_DIRECTORY_OK); - UT_ASSERT(slot_refs_equal(found, handles[0].tag_slot)); - for (i = 0; i < 3; i++) { - UT_ASSERT_EQ( - cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_WAIT, &handles[i].identity, &found), - PCM_X_DIRECTORY_OK); - UT_ASSERT(slot_refs_equal(found, handles[i].membership_slot)); - } + init_active_pcm_x(UINT64_C(77)); + bind_local_master(1, UINT64_C(9), UINT64_C(177)); + identity = make_wait_identity(811, 0, 4, UINT64_C(81101)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + PCM_X_QUEUE_OK); + identity = make_wait_identity(811, 0, 5, UINT64_C(81102)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(907)), + PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&handles[1].identity, 1, UINT64_C(177), &duplicate), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(slot_refs_equal(duplicate.membership_slot, handles[1].membership_slot)); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 3); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(907)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + assert_local_queue_baseline(ClusterPcmXConvertShmem); + + /* ABORT clears active_writer without setting WRITER_COMPLETE. A later + * follower can still hold an exact edge to that former active writer, so + * cancellation must retain the slot until that edge is removed. */ + init_active_pcm_x(UINT64_C(78)); + bind_local_master(1, UINT64_C(9), UINT64_C(178)); + identity = make_wait_identity(812, 0, 6, UINT64_C(81201)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(178), &leader), + PCM_X_QUEUE_OK); + identity = make_wait_identity(812, 0, 7, UINT64_C(81202)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(178), &follower), + PCM_X_QUEUE_OK); + identity = make_wait_identity(812, 0, 8, UINT64_C(81203)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(178), &follower2), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(917)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower2, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(918)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(917)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower2, UINT64_C(918)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower2, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&wfg.blocker, &follower.identity, sizeof(follower.identity)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(919)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_abort_exact(&claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower2, UINT64_C(919)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower2, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower2), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + assert_local_queue_baseline(ClusterPcmXConvertShmem); } -UT_TEST(test_local_follower_wfg_publish_is_nonwaitable_then_exact) +UT_TEST(test_local_closed_round_never_promotes_a_late_joiner_early) { PcmXLocalHandle leader; - PcmXLocalHandle follower; - PcmXLocalFollowerWfgSnapshot snapshot; - PcmXLocalFollowerWfgSnapshot stale; + PcmXLocalHandle late1; + PcmXLocalHandle late2; + PcmXLocalCutoff cutoff; PcmXLocalMembershipSlot *members; PcmXWaitIdentity identity; init_active_pcm_x(UINT64_C(77)); - bind_local_master(1, 0, UINT64_C(177)); - identity = make_wait_identity(806, 0, 17, UINT64_C(80601)); - identity.cluster_epoch = 0; + bind_local_master(1, UINT64_C(9), UINT64_C(177)); + identity = make_wait_identity(808, 0, 21, UINT64_C(80801)); UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), PCM_X_QUEUE_OK); - identity = make_wait_identity(806, 0, 18, UINT64_C(80602)); - identity.cluster_epoch = 0; - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower), + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + identity = make_wait_identity(808, 0, 22, UINT64_C(80802)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &late1), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(late1.role, PCM_X_LOCAL_ROLE_FOLLOWER); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + + identity = make_wait_identity(808, 0, 23, UINT64_C(80803)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &late2), PCM_X_QUEUE_OK); members = membership_slots(ClusterPcmXConvertShmem); - UT_ASSERT_EQ(test_slot_state(&members[follower.membership_slot.slot_index].slot), + UT_ASSERT_EQ(late2.role, PCM_X_LOCAL_ROLE_FOLLOWER); + UT_ASSERT_EQ(test_slot_state(&members[late2.membership_slot.slot_index].slot), PCM_XL_JOINED_NONWAITABLE); + UT_ASSERT_EQ(local_tag_slots(ClusterPcmXConvertShmem)[late2.tag_slot.slot_index].leader_index, + PCM_X_INVALID_SLOT_INDEX); +} - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &snapshot), - PCM_X_QUEUE_OK); - UT_ASSERT(memcmp(&snapshot.waiter, &follower, sizeof(follower)) == 0); - UT_ASSERT(memcmp(&snapshot.blocker, &leader.identity, sizeof(leader.identity)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&snapshot, UINT64_C(901)), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(test_slot_state(&members[follower.membership_slot.slot_index].slot), - PCM_XL_WAITABLE_FOLLOWER); - UT_ASSERT_EQ(members[follower.membership_slot.slot_index].graph_generation, UINT64_C(901)); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&snapshot, UINT64_C(901)), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(902)), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(901)), +UT_TEST(test_local_lookup_is_read_only_and_identity_exact) +{ + PcmXShmemHeader *header; + PcmXLocalHandle joined; + PcmXLocalHandle found; + PcmXLocalHandle cleared; + PcmXWaitIdentity identity; + PcmXWaitIdentity wrong; + Size tag_used; + Size wait_used; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(177)); + identity = make_wait_identity(706, 0, 27, UINT64_C(7027)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &joined), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(test_slot_state(&members[follower.membership_slot.slot_index].slot), - PCM_XL_JOINED_NONWAITABLE); - UT_ASSERT_EQ(members[follower.membership_slot.slot_index].graph_generation, 0); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(901)), - PCM_X_QUEUE_DUPLICATE); + tag_used = header->allocator[PCM_X_ALLOC_LOCAL_TAG].used; + wait_used = header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used; - stale = snapshot; - stale.leader_slot_generation++; - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&stale, UINT64_C(902)), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(members[follower.membership_slot.slot_index].graph_generation, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&identity, &found), PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&found, &joined, sizeof(found)) == 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, tag_used); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, wait_used); + + wrong = identity; + wrong.request_id++; + memset(&found, 0xa5, sizeof(found)); + memset(&cleared, 0, sizeof(cleared)); + cleared.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + cleared.membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&wrong, &found), PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT(memcmp(&found, &cleared, sizeof(found)) == 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, tag_used); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, wait_used); } -UT_TEST(test_local_writer_claim_runs_closed_cohort_and_blocks_next_round) +UT_TEST(test_local_progress_is_exact_and_exposes_remote_wait_ledger) { PcmXLocalHandle leader; - PcmXLocalHandle follower; - PcmXLocalHandle late; - PcmXLocalFollowerWfgSnapshot wfg; - PcmXLocalWriterClaim claim; - PcmXLocalWriterClaim released; - PcmXLocalWriterClaim stale; - PcmXLocalCutoff cutoff; - PcmXLocalTagSlot *tag_slot; + PcmXLocalHandle stale; + PcmXLocalProgress progress; + PcmXLocalProgress cleared; PcmXWaitIdentity identity; - PcmXWaitIdentity successor; + PcmXEnqueuePayload enqueue; + PcmXAdmitAckPayload ack; + PcmXPhasePayload confirm; + PcmXLocalReliableToken token; init_active_pcm_x(UINT64_C(77)); - bind_local_master(1, UINT64_C(9), UINT64_C(177)); - identity = make_wait_identity(807, 0, 19, UINT64_C(80701)); + identity = make_wait_identity(706, 0, 28, UINT64_C(7028)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(177)); UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), PCM_X_QUEUE_OK); - identity = make_wait_identity(807, 0, 20, UINT64_C(80702)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &wfg), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(903)), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); - tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; - UT_ASSERT_EQ(tag_slot->active_writer_index, leader.membership_slot.slot_index); - UT_ASSERT_EQ(tag_slot->active_writer_slot_generation, leader.membership_slot.slot_generation); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &stale), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&progress.identity, &identity, sizeof(identity)) == 0); + UT_ASSERT_EQ(progress.local_sequence, leader.local_sequence); + UT_ASSERT_EQ(progress.local_round, leader.local_round); + UT_ASSERT_EQ(progress.member_state, PCM_XL_NODE_LEADER); + UT_ASSERT_EQ(progress.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ(progress.pending_opcode, 0); + UT_ASSERT_EQ(progress.last_response_opcode, 0); - UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); - identity = make_wait_identity(807, 0, 21, UINT64_C(80703)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &late), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(late.local_round, cutoff.next_round); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&late, &wfg), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(909)), - PCM_X_QUEUE_OK); - released = claim; - memset(&successor, 0x7f, sizeof(successor)); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &token), PCM_X_QUEUE_OK); + memset(&ack, 0, sizeof(ack)); + ack.ref.identity = identity; + ack.ref.handle.ticket_id = UINT64_C(9008); + ack.ref.handle.queue_generation = UINT64_C(8); + ack.prehandle = enqueue.prehandle; + ack.result = PCM_X_QUEUE_OK; + ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&leader, &ack, 1, UINT64_C(177)), PCM_X_QUEUE_OK); - UT_ASSERT(memcmp(&successor, &follower.identity, sizeof(successor)) == 0); - UT_ASSERT_EQ(tag_slot->active_writer_index, PCM_X_INVALID_SLOT_INDEX); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_BAD_STATE); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &claim), PCM_X_QUEUE_BAD_STATE); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(903)), + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_arm_exact(&leader, &confirm, &token), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &claim), PCM_X_QUEUE_OK); - memset(&successor, 0x7f, sizeof(successor)); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&leader, &confirm, 1, UINT64_C(177)), PCM_X_QUEUE_OK); - UT_ASSERT(memcmp(&successor, &(PcmXWaitIdentity){ 0 }, sizeof(successor)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &claim), PCM_X_QUEUE_BAD_STATE); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&late, &claim), PCM_X_QUEUE_BARRIER_CLOSED); - stale = released; - stale.claim_generation++; - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&stale), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&progress.ref, &ack.ref)); + UT_ASSERT_EQ(progress.local_sequence, leader.local_sequence); + UT_ASSERT_EQ(progress.local_round, leader.local_round); + UT_ASSERT_EQ(progress.member_state, PCM_XL_REMOTE_WAIT); + UT_ASSERT_EQ(progress.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ(progress.reliable_state_sequence, 2); + UT_ASSERT_EQ(progress.pending_opcode, 0); + UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK); + UT_ASSERT_EQ(progress.phase, PCM_X_LOCAL_RELIABLE_PHASE_NONE); + + stale = leader; + stale.local_sequence++; + memset(&progress, 0xa5, sizeof(progress)); + memset(&cleared, 0, sizeof(cleared)); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&stale, &progress), PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(&progress, &cleared, sizeof(progress)) == 0); } -UT_TEST(test_local_writer_claim_completion_is_fifo_and_one_shot) +UT_TEST(test_local_transfer_prepare_commit_and_final_ack_are_exact) { + PcmXLocalTagSlot *tag_slot; PcmXLocalHandle leader; - PcmXLocalHandle follower1; - PcmXLocalHandle follower2; + PcmXLocalHandle follower; + PcmXLocalProgress progress; PcmXLocalFollowerWfgSnapshot wfg; - PcmXLocalWriterClaim aborted; - PcmXLocalWriterClaim busy; - PcmXLocalWriterClaim claim; - PcmXLocalTagSlot *tag_slot; PcmXWaitIdentity identity; - PcmXWaitIdentity successor; - uint32 state_flags; + PcmXEnqueuePayload enqueue; + PcmXAdmitAckPayload admit_ack; + PcmXPhasePayload admit_confirm; + PcmXLocalCutoff cutoff; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXTicketRef probe_ref; + PcmXGrantPayload prepare; + PcmXGrantPayload bad_prepare; + PcmXGrantPayload holder_replay; + PcmXRevokePayload revoke; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + PcmXPhasePayload duplicate_confirm; + PcmXDrainPollPayload poll; + PcmXLocalDrainCertificate drain_certificate; + PcmXRetirePayload retire; + PcmXLocalReliableToken token; + PcmXLocalReliableToken retry_token; + PcmXLocalWriterClaim writer_claim; + PcmXLocalWriterClaim stale_writer_claim; + PcmXLocalHolderKey writer_holder_key; + PcmXLocalHolderKey stale_holder_key; + PcmXLocalHolderHandle writer_holder; + PcmXLocalHolderHandle duplicate_holder; + uint64 committed_own_generation; + uint64 holder_set_generation; + Size holder_used; + const uint64 master_session = UINT64_C(178); init_active_pcm_x(UINT64_C(77)); - bind_local_master(1, UINT64_C(9), UINT64_C(177)); - identity = make_wait_identity(809, 0, 24, UINT64_C(80901)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + identity = make_wait_identity(707, 0, 27, UINT64_C(7029)); + bind_local_master(1, identity.cluster_epoch, master_session); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &leader), PCM_X_QUEUE_OK); - identity = make_wait_identity(809, 0, 25, UINT64_C(80902)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower1), + identity = make_wait_identity(707, 0, 28, UINT64_C(7030)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &follower), PCM_X_QUEUE_OK); - identity = make_wait_identity(809, 0, 26, UINT64_C(80903)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower2), + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(908)), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower1, &wfg), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(904)), + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &writer_claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + writer_holder_key = make_local_holder_key(707, 0, 27, UINT64_C(7031), 3); + committed_own_generation = UINT64_MAX; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( + &writer_holder_key, &writer_claim, &writer_holder, &committed_own_generation), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(committed_own_generation, 0); + UT_ASSERT_EQ(writer_holder.tag_slot.slot_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &token), PCM_X_QUEUE_OK); + memset(&admit_ack, 0, sizeof(admit_ack)); + admit_ack.ref.identity = leader.identity; + admit_ack.ref.handle.ticket_id = UINT64_C(9009); + admit_ack.ref.handle.queue_generation = UINT64_C(9); + admit_ack.prehandle = enqueue.prehandle; + admit_ack.result = PCM_X_QUEUE_OK; + admit_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&leader, &admit_ack, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower2, &wfg), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(905)), + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_arm_exact(&leader, &admit_confirm, &token), PCM_X_QUEUE_OK); - - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower2, &busy), PCM_X_QUEUE_BAD_STATE); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); - aborted = claim; - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_abort_exact(&claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); - UT_ASSERT(claim.claim_generation > aborted.claim_generation); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_abort_exact(&aborted), PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower1, &busy), PCM_X_QUEUE_BUSY); - tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; - state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); - pg_atomic_write_u32(&tag_slot->slot.state_flags, - state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); - memset(&successor, 0x7f, sizeof(successor)); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), - PCM_X_QUEUE_BUSY); - UT_ASSERT(memcmp(&successor, &(PcmXWaitIdentity){ 0 }, sizeof(successor)) == 0); - UT_ASSERT_EQ(tag_slot->active_writer_index, leader.membership_slot.slot_index); UT_ASSERT_EQ( - test_slot_flags( - &membership_slots(ClusterPcmXConvertShmem)[leader.membership_slot.slot_index].slot) - & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE, - 0); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); - pg_atomic_write_u32(&tag_slot->slot.state_flags, state_flags); - memset(&successor, 0, sizeof(successor)); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), - PCM_X_QUEUE_OK); - UT_ASSERT(memcmp(&successor, &follower1.identity, sizeof(successor)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_BAD_STATE); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower2, UINT64_C(905)), + cluster_pcm_x_local_admit_confirm_ack_exact(&leader, &admit_confirm, 1, master_session), + PCM_X_QUEUE_OK); + + memset(&prepare, 0, sizeof(prepare)); + prepare.ref = admit_ack.ref; + prepare.ref.grant_generation = UINT64_C(99); + UT_ASSERT(cluster_pcm_x_image_id_encode(1, UINT64_C(199), &prepare.image.image_id)); + prepare.image.source_own_generation = UINT64_C(41); + prepare.image.page_scn = UINT64_C(42); + prepare.image.page_lsn = UINT64_C(43); + prepare.image.source_node = 0; + prepare.image.page_checksum = UINT32_C(44); + probe_ref = prepare.ref; + probe_ref.grant_generation = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + NULL, 0, &holder_snapshot), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower2, &busy), PCM_X_QUEUE_BUSY); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower1, UINT64_C(904)), + UT_ASSERT_EQ(holder_snapshot.holder_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower1, &claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower2, &wfg), PCM_X_QUEUE_OK); - UT_ASSERT(memcmp(&wfg.blocker, &follower1.identity, sizeof(follower1.identity)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(906)), + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), PCM_X_QUEUE_OK); - memset(&successor, 0, sizeof(successor)); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + memset(&revoke, 0, sizeof(revoke)); + revoke.ref = prepare.ref; + revoke.image_id = prepare.image.image_id; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT(memcmp(&successor, &follower2.identity, sizeof(successor)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower1, &claim), PCM_X_QUEUE_BAD_STATE); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower1, NULL), PCM_X_QUEUE_BUSY); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower2, UINT64_C(906)), + UT_ASSERT_EQ(cluster_pcm_x_local_holder_image_ready_arm_exact(&prepare, 1, master_session, + &holder_replay), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower1, NULL), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower1), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower2, &claim), PCM_X_QUEUE_OK); - memset(&successor, 0x7f, sizeof(successor)); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + UT_ASSERT(memcmp(&holder_replay, &prepare, sizeof(prepare)) == 0); + bad_prepare = prepare; + bad_prepare.image.image_id = UINT64CONST(0xe0000000000000c7); + UT_ASSERT_EQ(cluster_pcm_x_local_prepare_grant_exact(&leader, &bad_prepare, 1, master_session), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_local_prepare_grant_exact(&leader, &prepare, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT(memcmp(&successor, &(PcmXWaitIdentity){ 0 }, sizeof(successor)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower2, &claim), PCM_X_QUEUE_BAD_STATE); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_BUSY); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower2, NULL), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower2), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); -} + UT_ASSERT_EQ(cluster_pcm_x_local_prepare_grant_exact(&leader, &prepare, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.member_state, PCM_XL_REMOTE_WAIT); + UT_ASSERT_EQ(progress.pending_opcode, 0); + UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT); + UT_ASSERT(ticket_refs_equal(&progress.ref, &prepare.ref)); + UT_ASSERT(memcmp(&progress.image, &prepare.image, sizeof(progress.image)) == 0); -UT_TEST(test_writer_and_holder_owner_exit_retry_preserves_exact_evidence) -{ - PcmXLocalHolderKey holder_key; - PcmXLocalHolderHandle holder; - PcmXLocalHandle leader; - PcmXLocalWriterClaim claim; - PcmXLocalTagSlot *tag_slot; - PcmXLocalTagSlot tag_before; - PcmXLocalMembershipSlot holder_before; - PcmXLocalMembershipSlot writer_before; - PcmXWaitIdentity identity; - PcmXWaitIdentity successor; - LWLock content_lock; - uint32 state_flags; - Size holder_used_before; - const uint64 master_session = UINT64_C(8177); + UT_ASSERT_EQ(cluster_pcm_x_local_install_ready_arm_exact(&leader, &prepare.ref, &prepare.image, + &install_ready, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(install_ready.image_id, prepare.image.image_id); + UT_ASSERT_EQ(install_ready.phase, PGRAC_IC_MSG_PCM_X_INSTALL_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.member_state, PCM_XL_CONTENT_ACTIVE); + UT_ASSERT_EQ(progress.pending_opcode, PGRAC_IC_MSG_PCM_X_INSTALL_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_reliable_snapshot_exact(&leader, &retry_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(retry_token.pending_opcode, PGRAC_IC_MSG_PCM_X_INSTALL_READY); + UT_ASSERT_EQ( + cluster_pcm_x_local_reliable_retry_exact(&leader, &token, UINT64_C(1001), &retry_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(retry_token.retry_count, 1); + UT_ASSERT_EQ(retry_token.retry_deadline_ms, UINT64_C(1001)); + memset(&commit, 0, sizeof(commit)); + commit.ref = prepare.ref; + commit.phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; + UT_ASSERT_EQ(cluster_pcm_x_local_commit_x_exact(&leader, &commit, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_commit_x_exact(&leader, &commit, 1, master_session), + PCM_X_QUEUE_DUPLICATE); - init_active_pcm_x(UINT64_C(177)); - bind_local_master(1, UINT64_C(9), master_session); - holder_key = make_local_holder_key(811, 0, 27, UINT64_C(81101), 1); - UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); - identity = make_wait_identity(811, 0, 28, UINT64_C(81102)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &leader), + UT_ASSERT_EQ(cluster_pcm_x_local_final_ack_arm_exact(&leader, 1, &final_ack, &token), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(final_ack.image_id, prepare.image.image_id); + UT_ASSERT_EQ(final_ack.committed_own_generation, 1); + UT_ASSERT_EQ(cluster_pcm_x_local_reliable_snapshot_exact(&leader, &retry_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(retry_token.pending_opcode, PGRAC_IC_MSG_PCM_X_FINAL_ACK); + UT_ASSERT_EQ( + cluster_pcm_x_local_reliable_retry_exact(&leader, &token, UINT64_C(1002), &retry_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(retry_token.retry_count, 1); + UT_ASSERT_EQ(retry_token.retry_deadline_ms, UINT64_C(1002)); + memset(&final_commit, 0, sizeof(final_commit)); + final_commit.ref = prepare.ref; + final_commit.phase = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; + UT_ASSERT_EQ(cluster_pcm_x_local_final_commit_ack_exact(&leader, &final_commit, 1, + master_session, &final_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(final_confirm.phase, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.member_state, PCM_XL_GRANTED); + UT_ASSERT_EQ(progress.pending_opcode, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM); + UT_ASSERT_EQ(progress.phase, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM); + UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK); + UT_ASSERT_EQ(cluster_pcm_x_local_reliable_snapshot_exact(&leader, &retry_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(retry_token.pending_opcode, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM); + UT_ASSERT_EQ( + cluster_pcm_x_local_reliable_retry_exact(&leader, &token, UINT64_C(1003), &retry_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(retry_token.retry_count, 1); + UT_ASSERT_EQ(retry_token.retry_deadline_ms, UINT64_C(1003)); + UT_ASSERT_EQ(local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index] + .committed_own_generation, + 1); + UT_ASSERT_EQ(cluster_pcm_x_local_final_commit_ack_exact( + &leader, &final_commit, 1, master_session, &duplicate_confirm, &token), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&duplicate_confirm, &final_confirm, sizeof(final_confirm)) == 0); tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; - holder_used_before = ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used; - memset(&content_lock, 0, sizeof(content_lock)); - state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); - pg_atomic_write_u32(&tag_slot->slot.state_flags, - state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); - memset(&successor, 0x7f, sizeof(successor)); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), - PCM_X_QUEUE_BUSY); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_exceptional_detach_exact(&holder, &content_lock), - PCM_X_QUEUE_GATE_RETRY); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) == 0); + UT_ASSERT_EQ(tag_slot->leader_index, leader.membership_slot.slot_index); UT_ASSERT_EQ(tag_slot->active_writer_index, leader.membership_slot.slot_index); - UT_ASSERT_EQ(tag_slot->active_holder_head_index, holder.holder_slot.slot_index); - UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, - holder_used_before); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM); + UT_ASSERT(ticket_refs_equal(&tag_slot->ref, &prepare.ref)); + UT_ASSERT_EQ(tag_slot->master_node, 1); + UT_ASSERT_EQ(tag_slot->master_session_incarnation, master_session); + UT_ASSERT_EQ(tag_slot->cluster_epoch, identity.cluster_epoch); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->peer_frontiers[1].cluster_epoch, identity.cluster_epoch); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->peer_frontiers[1].sender_session_incarnation, + master_session); + UT_ASSERT_EQ(membership_slots(ClusterPcmXConvertShmem)[leader.membership_slot.slot_index] + .handle.ticket_id, + prepare.ref.handle.ticket_id); + UT_ASSERT_EQ( + test_slot_state( + &membership_slots(ClusterPcmXConvertShmem)[leader.membership_slot.slot_index].slot), + PCM_XL_GRANTED); + + writer_holder_key.identity.base_own_generation = 1; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&writer_holder_key, &writer_holder), + PCM_X_QUEUE_BARRIER_CLOSED); + holder_used = ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used; + holder_set_generation = tag_slot->holder_set_generation; + stale_writer_claim = writer_claim; + stale_writer_claim.claim_generation++; + committed_own_generation = UINT64_MAX; + UT_ASSERT_EQ( + cluster_pcm_x_local_writer_holder_register_exact(&writer_holder_key, &stale_writer_claim, + &writer_holder, &committed_own_generation), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(committed_own_generation, 0); + stale_writer_claim = writer_claim; + stale_writer_claim.local_round++; + UT_ASSERT_EQ( + cluster_pcm_x_local_writer_holder_register_exact(&writer_holder_key, &stale_writer_claim, + &writer_holder, &committed_own_generation), + PCM_X_QUEUE_STALE); + stale_holder_key = writer_holder_key; + stale_holder_key.identity.base_own_generation = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( + &stale_holder_key, &writer_claim, &writer_holder, &committed_own_generation), + PCM_X_QUEUE_STALE); + stale_holder_key.identity.base_own_generation = 2; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( + &stale_holder_key, &writer_claim, &writer_holder, &committed_own_generation), + PCM_X_QUEUE_STALE); + stale_holder_key = writer_holder_key; + stale_holder_key.identity.procno++; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( + &stale_holder_key, &writer_claim, &writer_holder, &committed_own_generation), + PCM_X_QUEUE_STALE); + stale_holder_key = writer_holder_key; + stale_holder_key.identity.xid++; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( + &stale_holder_key, &writer_claim, &writer_holder, &committed_own_generation), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, holder_used); + UT_ASSERT_EQ(tag_slot->holder_set_generation, holder_set_generation); UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); - pg_atomic_write_u32(&tag_slot->slot.state_flags, state_flags); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_exceptional_detach_exact(&holder, &content_lock), + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( + &writer_holder_key, &writer_claim, &writer_holder, &committed_own_generation), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(tag_slot->active_writer_index, PCM_X_INVALID_SLOT_INDEX); - UT_ASSERT_EQ(tag_slot->active_holder_head_index, PCM_X_INVALID_SLOT_INDEX); - UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, - holder_used_before - 1); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(committed_own_generation, 1); + stale_writer_claim = writer_claim; + stale_writer_claim.claim_generation++; + UT_ASSERT_EQ( + cluster_pcm_x_local_writer_holder_register_exact( + &writer_holder_key, &stale_writer_claim, &duplicate_holder, &committed_own_generation), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact(&writer_holder_key, &writer_claim, + &duplicate_holder, + &committed_own_generation), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&duplicate_holder, &writer_holder, sizeof(writer_holder)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&writer_holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&writer_holder), PCM_X_QUEUE_OK); - /* A different runtime may never consume either exact ledger. Both APIs - * preserve byte-identical shared evidence for deferred recovery. */ - init_active_pcm_x(UINT64_C(178)); - bind_local_master(1, UINT64_C(9), master_session + 1); - holder_key = make_local_holder_key(812, 0, 27, UINT64_C(81201), 1); - UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); - identity = make_wait_identity(812, 0, 28, UINT64_C(81202)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session + 1, &leader), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); - tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; - tag_before = *tag_slot; - holder_before = membership_slots(ClusterPcmXConvertShmem)[holder.holder_slot.slot_index]; - writer_before = membership_slots(ClusterPcmXConvertShmem)[leader.membership_slot.slot_index]; - cluster_pcm_x_runtime_fail_closed(); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_exceptional_detach_exact(&holder, &content_lock), + memset(&poll, 0, sizeof(poll)); + poll.ref = prepare.ref; + poll.drain_generation = UINT64_C(43); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_NOT_READY); - UT_ASSERT(memcmp(&tag_before, tag_slot, sizeof(tag_before)) == 0); - UT_ASSERT(memcmp(&holder_before, - &membership_slots(ClusterPcmXConvertShmem)[holder.holder_slot.slot_index], - sizeof(holder_before)) - == 0); - UT_ASSERT(memcmp(&writer_before, - &membership_slots(ClusterPcmXConvertShmem)[leader.membership_slot.slot_index], - sizeof(writer_before)) - == 0); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); -} - -UT_TEST(test_local_wfg_rejects_completed_blocker_semantic_aba) -{ - PcmXLocalHandle leader; - PcmXLocalHandle follower; - PcmXLocalFollowerWfgSnapshot snapshot; - PcmXLocalWriterClaim claim; - PcmXWaitIdentity identity; - - init_active_pcm_x(UINT64_C(77)); - bind_local_master(1, UINT64_C(9), UINT64_C(177)); - identity = make_wait_identity(810, 0, 27, UINT64_C(81001)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + UT_ASSERT_EQ(tag_slot->leader_index, leader.membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->active_writer_index, leader.membership_slot.slot_index); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&writer_claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(908)), PCM_X_QUEUE_OK); - identity = make_wait_identity(810, 0, 28, UINT64_C(81002)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower), + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &writer_claim), PCM_X_QUEUE_OK); + writer_holder_key = make_local_holder_key(707, 0, 28, UINT64_C(7032), 4); + writer_holder_key.identity.base_own_generation = 1; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( + &writer_holder_key, &writer_claim, &writer_holder, &committed_own_generation), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &snapshot), + UT_ASSERT_EQ(committed_own_generation, 1); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&writer_holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&writer_holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&writer_claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower), PCM_X_QUEUE_OK); + /* First consumption captures the completion certificate under the same + * tag lock; a DUPLICATE replay must not fabricate one. */ + memset(&drain_certificate, 0xff, sizeof(drain_certificate)); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_certificate_exact(&poll, 1, master_session, + &drain_certificate), + PCM_X_QUEUE_OK); + UT_ASSERT(drain_certificate.valid); + UT_ASSERT(ticket_refs_equal(&drain_certificate.ref, &poll.ref)); + UT_ASSERT_EQ(drain_certificate.master_node, 1); + UT_ASSERT_EQ(drain_certificate.master_session_incarnation, master_session); + UT_ASSERT_EQ(drain_certificate.committed_own_generation, tag_slot->committed_own_generation); + UT_ASSERT_EQ(drain_certificate.image.image_id, tag_slot->image.image_id); + UT_ASSERT_EQ(drain_certificate.image.source_own_generation, + tag_slot->image.source_own_generation); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_certificate_exact(&poll, 1, master_session, + &drain_certificate), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(!drain_certificate.valid); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.member_state, PCM_XL_GRANTED); + UT_ASSERT_EQ(progress.pending_opcode, 0); + UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_DRAIN_POLL); + + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = prepare.ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&snapshot, UINT64_C(906)), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &snapshot), - PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_NOT_FOUND); + assert_local_queue_baseline(ClusterPcmXConvertShmem); } -UT_TEST(test_local_cancel_never_unlinks_an_active_writer) +UT_TEST(test_local_tag_only_holder_transfer_persists_until_exact_drain) { - PcmXLocalHandle leader; - PcmXLocalHandle follower; - PcmXLocalHandle follower2; + PcmXShmemHeader *header; + PcmXLocalHolderKey new_holder_key; + PcmXLocalHolderHandle new_holder; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXBlockerSetHeaderPayload blocker_header; + PcmXEnqueuePayload enqueue; PcmXLocalFollowerWfgSnapshot wfg; - PcmXLocalWriterClaim claim; + PcmXLocalCutoff cutoff; + PcmXLocalCutoff promoted_cutoff; + PcmXLocalImageReadyRefusal ready_refusal; + PcmXLocalHandle late; + PcmXLocalHandle next_leader; + PcmXLocalTagSlot *tag_slot; + PcmXLocalReliableToken token; + PcmXLocalWriterClaim writer_claim; + PcmXTicketRef probe_ref; + PcmXLocalHolderProgress progress; + PcmXRevokePayload revoke; + PcmXRevokePayload bad_revoke; + PcmXGrantPayload ready; + PcmXGrantPayload replay; + PcmXDrainPollPayload poll; + PcmXRetirePayload retire; PcmXWaitIdentity identity; + PcmXWaitIdentity late_identity; + PcmXWaitIdentity wake_items[1]; + PcmXLocalWakeBatch wake_batch; + const uint64 master_session = UINT64_C(179); + uint32 state_flags; init_active_pcm_x(UINT64_C(77)); - bind_local_master(1, UINT64_C(9), UINT64_C(177)); - identity = make_wait_identity(811, 0, 4, UINT64_C(81101)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + header = ClusterPcmXConvertShmem; + identity = make_wait_identity(708, 2, 28, UINT64_C(7030)); + bind_local_master(1, identity.cluster_epoch, master_session); + memset(&revoke, 0, sizeof(revoke)); + revoke.ref.identity = identity; + revoke.ref.handle.ticket_id = UINT64_C(9010); + revoke.ref.handle.queue_generation = UINT64_C(10); + revoke.ref.grant_generation = UINT64_C(100); + UT_ASSERT(cluster_pcm_x_image_id_encode(1, UINT64_C(200), &revoke.image_id)); + probe_ref = revoke.ref; + probe_ref.grant_generation = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&identity.tag, NULL, 0, &holder_snapshot), PCM_X_QUEUE_OK); - identity = make_wait_identity(811, 0, 5, UINT64_C(81102)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower), + UT_ASSERT_EQ(holder_snapshot.tag_slot.slot_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(holder_snapshot.holder_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + NULL, 0, &holder_snapshot), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &wfg), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(907)), + UT_ASSERT_EQ(holder_snapshot.holder_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), PCM_X_QUEUE_OK); - - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_BUSY); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_BUSY); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_BUSY); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(907)), + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_copy_exact(&blocker_snapshot, &blocker_header, + NULL, 0), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); - assert_local_queue_baseline(ClusterPcmXConvertShmem); - - /* ABORT clears active_writer without setting WRITER_COMPLETE. A later - * follower can still hold an exact edge to that former active writer, so - * cancellation must retain the slot until that edge is removed. */ - init_active_pcm_x(UINT64_C(78)); - bind_local_master(1, UINT64_C(9), UINT64_C(178)); - identity = make_wait_identity(812, 0, 6, UINT64_C(81201)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(178), &leader), + UT_ASSERT_EQ(blocker_header.nblockers, 0); + UT_ASSERT_EQ(blocker_header.set_crc32c, blocker_set_crc32c(NULL, 0)); + late_identity = make_wait_identity(708, 0, 6, UINT64_C(7031)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&late_identity, 1, master_session, &late), PCM_X_QUEUE_OK); - identity = make_wait_identity(812, 0, 7, UINT64_C(81202)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(178), &follower), + UT_ASSERT_EQ(late.role, PCM_X_LOCAL_ROLE_FOLLOWER); + UT_ASSERT_EQ(late.local_round, 2); + UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact( + &identity.tag, identity.cluster_epoch, 1, master_session, &cutoff), PCM_X_QUEUE_OK); - identity = make_wait_identity(812, 0, 8, UINT64_C(81203)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(178), &follower2), + UT_ASSERT_EQ(cutoff.cutoff_sequence, 0); + UT_ASSERT_EQ(cutoff.closed_round, 1); + UT_ASSERT_EQ(cutoff.next_round, 2); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&late, &wfg), + PCM_X_QUEUE_BARRIER_CLOSED); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&identity.tag, identity.cluster_epoch, + &cutoff, &next_leader), + PCM_X_QUEUE_NOT_READY); + bad_revoke = revoke; + bad_revoke.image_id = UINT64CONST(0xf0000000000000c8); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&bad_revoke, 1, master_session), + PCM_X_QUEUE_INVALID); + /* Complete generation 1 normally. The separate gate-contention test below + * covers an ACK frame that is delivered but cannot apply before REVOKE. */ + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &wfg), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(917)), + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + /* The master may consume the first ACK before a same-ticket re-PROBE + * publishes a later empty blocker COMMIT. Authenticated REVOKE proves the + * master crossed that ACK boundary, so the source must consume this exact + * empty replay instead of waiting forever for an ACK the master no longer + * needs to send. */ + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + NULL, 0, &holder_snapshot), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower2, &wfg), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(918)), + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(917)), + UT_ASSERT_EQ(blocker_snapshot.set_generation, 2); + /* The exact blocker ACK tombstone is durable evidence, but it cannot + * reopen the writer round before REVOKE/IMAGE_READY/DRAIN completes. */ + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&identity.tag, identity.cluster_epoch, + &cutoff, &next_leader), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower2, UINT64_C(918)), + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_progress_exact(&identity.tag, &progress), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower2, &wfg), PCM_X_QUEUE_OK); - UT_ASSERT(memcmp(&wfg.blocker, &follower.identity, sizeof(follower.identity)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(919)), + UT_ASSERT(ticket_refs_equal(&progress.ref, &revoke.ref)); + UT_ASSERT_EQ(progress.image.image_id, revoke.image_id); + UT_ASSERT_EQ(progress.master_session_incarnation, master_session); + UT_ASSERT_EQ(progress.master_node, 1); + UT_ASSERT_EQ(progress.pending_opcode, 0); + UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_REVOKE); + + memset(&ready, 0, sizeof(ready)); + ready.ref = revoke.ref; + ready.image.image_id = revoke.image_id; + ready.image.source_own_generation = 0; + ready.image.page_scn = UINT64_C(52); + ready.image.page_lsn = UINT64_C(53); + ready.image.source_node = 0; + ready.image.page_checksum = UINT32_C(54); + tag_slot = &local_tag_slots(header)[holder_snapshot.tag_slot.slot_index]; + state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); + pg_atomic_write_u32(&tag_slot->slot.state_flags, + state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); + ready_refusal = PCM_X_LOCAL_IMAGE_READY_REFUSAL_NONE; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( + &ready, 1, master_session, &replay, &ready_refusal), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(ready_refusal, PCM_X_LOCAL_IMAGE_READY_REFUSAL_LOCAL_GATE); + pg_atomic_write_u32(&tag_slot->slot.state_flags, state_flags); + tag_slot->active_holder_head_index = 0; + ready_refusal = PCM_X_LOCAL_IMAGE_READY_REFUSAL_NONE; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( + &ready, 1, master_session, &replay, &ready_refusal), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(ready_refusal, PCM_X_LOCAL_IMAGE_READY_REFUSAL_ACTIVE_HOLDER); + tag_slot->active_holder_head_index = PCM_X_INVALID_SLOT_INDEX; + ready_refusal = PCM_X_LOCAL_IMAGE_READY_REFUSAL_ACTIVE_HOLDER; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_image_ready_arm_exact_diagnosed( + &ready, 1, master_session, &replay, &ready_refusal), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_abort_exact(&claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_BUSY); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower2, UINT64_C(919)), + UT_ASSERT_EQ(ready_refusal, PCM_X_LOCAL_IMAGE_READY_REFUSAL_NONE); + UT_ASSERT(memcmp(&ready, &replay, sizeof(ready)) == 0); + memset(&replay, 0, sizeof(replay)); + UT_ASSERT_EQ( + cluster_pcm_x_local_holder_image_ready_arm_exact(&ready, 1, master_session, &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&ready, &replay, sizeof(ready)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_progress_exact(&identity.tag, &progress), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower2, NULL), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower2), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); - assert_local_queue_baseline(ClusterPcmXConvertShmem); -} - -UT_TEST(test_local_closed_round_never_promotes_a_late_joiner_early) -{ - PcmXLocalHandle leader; - PcmXLocalHandle late1; - PcmXLocalHandle late2; - PcmXLocalCutoff cutoff; - PcmXLocalMembershipSlot *members; - PcmXWaitIdentity identity; + UT_ASSERT_EQ(progress.pending_opcode, PGRAC_IC_MSG_PCM_X_IMAGE_READY); + UT_ASSERT_EQ(progress.image.source_own_generation, 0); + UT_ASSERT_EQ(progress.flags, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&identity.tag, identity.cluster_epoch, + &cutoff, &next_leader), + PCM_X_QUEUE_NOT_READY); - init_active_pcm_x(UINT64_C(77)); - bind_local_master(1, UINT64_C(9), UINT64_C(177)); - identity = make_wait_identity(808, 0, 21, UINT64_C(80801)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); - identity = make_wait_identity(808, 0, 22, UINT64_C(80802)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &late1), + memset(&poll, 0, sizeof(poll)); + poll.ref = revoke.ref; + poll.drain_generation = UINT64_C(42); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&late.identity, &next_leader), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(next_leader.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT(slot_refs_equal(next_leader.membership_slot, late.membership_slot)); + /* Promotion is visible for exact lookup, but holder cleanup is not complete + * until RETIRE consumes the terminal image lane. No timed wake or unrelated + * latch may let this successor claim, refreeze, or mint ENQUEUE early. */ + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&next_leader, &writer_claim), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&next_leader, &promoted_cutoff), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&next_leader, &enqueue, &token), + PCM_X_QUEUE_BUSY); + new_holder_key = make_local_holder_key(708, 0, 3, UINT64_C(7032), 3); + UT_ASSERT_EQ(register_active_local_holder(&new_holder_key, &new_holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_progress_exact(&identity.tag, &progress), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(late1.role, PCM_X_LOCAL_ROLE_FOLLOWER); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.pending_opcode, 0); + UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_DRAIN_POLL); + UT_ASSERT_EQ(progress.image.source_own_generation, 0); + UT_ASSERT_EQ(progress.flags, PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); - identity = make_wait_identity(808, 0, 23, UINT64_C(80803)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &late2), + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = revoke.ref.handle.ticket_id; + retire.sender_node = 0; + memset(wake_items, 0, sizeof(wake_items)); + wake_batch.items = wake_items; + wake_batch.capacity = lengthof(wake_items); + wake_batch.count = 0; + UT_ASSERT_EQ( + cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, master_session, &wake_batch), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(wake_batch.count, 1); + UT_ASSERT(memcmp(&wake_items[0], &next_leader.identity, sizeof(wake_items[0])) == 0); + /* A visible wake batch proves the full watermark and global admission gate + * committed before the caller can make the successor runnable. */ + UT_ASSERT_EQ(pg_atomic_read_u32(&header->local_retire_gate), 0); + UT_ASSERT_EQ(header->peer_frontiers[1].local_retired_ticket_id, + retire.retire_through_ticket_id); + wake_batch.count = 1; + UT_ASSERT_EQ( + cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, master_session, &wake_batch), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(wake_batch.count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_progress_exact(&identity.tag, &progress), + PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&next_leader, &writer_claim), PCM_X_QUEUE_OK); - members = membership_slots(ClusterPcmXConvertShmem); - UT_ASSERT_EQ(late2.role, PCM_X_LOCAL_ROLE_FOLLOWER); - UT_ASSERT_EQ(test_slot_state(&members[late2.membership_slot.slot_index].slot), - PCM_XL_JOINED_NONWAITABLE); - UT_ASSERT_EQ(local_tag_slots(ClusterPcmXConvertShmem)[late2.tag_slot.slot_index].leader_index, - PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_abort_exact(&writer_claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&new_holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&next_leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&next_leader), PCM_X_QUEUE_OK); + assert_local_queue_baseline(header); } -UT_TEST(test_local_lookup_is_read_only_and_identity_exact) +UT_TEST(test_local_non_source_blocker_participant_drains_and_retires_exactly) { PcmXShmemHeader *header; - PcmXLocalHandle joined; - PcmXLocalHandle found; - PcmXLocalHandle cleared; - PcmXWaitIdentity identity; - PcmXWaitIdentity wrong; - Size tag_used; - Size wait_used; + PcmXLocalHolderKey holder_keys[2]; + PcmXLocalHolderKey new_holder_key; + PcmXLocalHolderHandle holders[2]; + PcmXLocalHolderHandle new_holder; + PcmXLocalHolderHandle holder_copies[2]; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalCutoff cutoff; + PcmXLocalTagSlot *tag_slot; + PcmXTicketRef probe_ref; + PcmXDrainPollPayload poll; + PcmXDrainPollPayload stale_poll; + PcmXRetirePayload retire; + PcmXSlotRef found; + ClusterLmdVertex blocker; + const uint64 master_session = UINT64_C(1801); + int i; + /* Model the local evidence left on a non-source participant after a + * same-ticket re-PROBE races behind the ACK already consumed by the master: + * two independent S pins are captured, both are later released, and an empty + * blocker COMMIT is still staged locally. The exact terminal DRAIN must + * consume that empty generation without waiting forever for a replayed ACK. */ init_active_pcm_x(UINT64_C(77)); header = ClusterPcmXConvertShmem; - bind_local_master(1, UINT64_C(9), UINT64_C(177)); - identity = make_wait_identity(706, 0, 27, UINT64_C(7027)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &joined), + bind_local_master(1, UINT64_C(9), master_session); + for (i = 0; i < 2; i++) { + holder_keys[i] + = make_local_holder_key(7091, 0, (uint32)(21 + i), UINT64_C(71001) + i, 4 + i); + UT_ASSERT_EQ(register_active_local_holder(&holder_keys[i], &holders[i]), PCM_X_QUEUE_OK); + } + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(7091, 3, 23, UINT64_C(71003)); + probe_ref.handle.ticket_id = UINT64_C(91001); + probe_ref.handle.queue_generation = UINT64_C(11); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact( + &probe_ref, 1, master_session, holder_copies, 2, &holder_snapshot), PCM_X_QUEUE_OK); - tag_used = header->allocator[PCM_X_ALLOC_LOCAL_TAG].used; - wait_used = header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used; - - UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&identity, &found), PCM_X_QUEUE_OK); - UT_ASSERT(memcmp(&found, &joined, sizeof(found)) == 0); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, tag_used); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, wait_used); - - wrong = identity; - wrong.request_id++; - memset(&found, 0xa5, sizeof(found)); - memset(&cleared, 0, sizeof(cleared)); - cleared.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; - cleared.membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; - UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&wrong, &found), PCM_X_QUEUE_NOT_FOUND); - UT_ASSERT(memcmp(&found, &cleared, sizeof(found)) == 0); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, tag_used); - UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, wait_used); -} - -UT_TEST(test_local_progress_is_exact_and_exposes_remote_wait_ledger) -{ - PcmXLocalHandle leader; - PcmXLocalHandle stale; - PcmXLocalProgress progress; - PcmXLocalProgress cleared; - PcmXWaitIdentity identity; - PcmXEnqueuePayload enqueue; - PcmXAdmitAckPayload ack; - PcmXPhasePayload confirm; - PcmXLocalReliableToken token; - - init_active_pcm_x(UINT64_C(77)); - identity = make_wait_identity(706, 0, 28, UINT64_C(7028)); - bind_local_master(1, identity.cluster_epoch, UINT64_C(177)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + UT_ASSERT_EQ(holder_snapshot.holder_count, 2); + blocker = make_blocker(holder_keys[0].identity.node_id, holder_keys[0].identity.procno, + holder_keys[0].identity.request_id); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, holder_copies, 2, + &blocker, 1, &blocker_snapshot), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact(&probe_ref.identity.tag, + probe_ref.identity.cluster_epoch, + 1, master_session, &cutoff), + PCM_X_QUEUE_OK); + for (i = 0; i < 2; i++) + UT_ASSERT_EQ(release_active_local_holder(&holders[i]), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); - UT_ASSERT(memcmp(&progress.identity, &identity, sizeof(identity)) == 0); - UT_ASSERT_EQ(progress.local_sequence, leader.local_sequence); - UT_ASSERT_EQ(progress.local_round, leader.local_round); - UT_ASSERT_EQ(progress.member_state, PCM_XL_NODE_LEADER); - UT_ASSERT_EQ(progress.role, PCM_X_LOCAL_ROLE_NODE_LEADER); - UT_ASSERT_EQ(progress.pending_opcode, 0); - UT_ASSERT_EQ(progress.last_response_opcode, 0); - - UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &token), PCM_X_QUEUE_OK); - memset(&ack, 0, sizeof(ack)); - ack.ref.identity = identity; - ack.ref.handle.ticket_id = UINT64_C(9008); - ack.ref.handle.queue_generation = UINT64_C(8); - ack.prehandle = enqueue.prehandle; - ack.result = PCM_X_QUEUE_OK; - ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; - UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&leader, &ack, 1, UINT64_C(177)), + tag_slot = &local_tag_slots(header)[holder_snapshot.tag_slot.slot_index]; + UT_ASSERT(ticket_refs_equal(&tag_slot->holder_ref, &(PcmXTicketRef){ 0 })); + UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &probe_ref)); + UT_ASSERT_EQ(tag_slot->blocker_snapshot_reliable.pending_opcode, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT); + + memset(&poll, 0, sizeof(poll)); + poll.ref = probe_ref; + poll.ref.grant_generation = UINT64_C(101); + poll.drain_generation = UINT64_C(42); + UT_ASSERT_EQ(tag_slot->blocker_snapshot_count, 1); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_arm_exact(&leader, &confirm, &token), + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact( + &probe_ref, 1, master_session, holder_copies, 2, &holder_snapshot), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&leader, &confirm, 1, UINT64_C(177)), + UT_ASSERT_EQ(holder_snapshot.holder_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(blocker_snapshot.set_generation, 2); + UT_ASSERT_EQ(tag_slot->blocker_snapshot_count, 0); + UT_ASSERT_EQ(tag_slot->blocker_snapshot_reliable.pending_opcode, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(ticket_refs_equal(&tag_slot->holder_ref, &(PcmXTicketRef){ 0 })); + UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &poll.ref)); + UT_ASSERT_EQ(tag_slot->blocker_set_generation, 0); + UT_ASSERT_EQ(tag_slot->holder_terminal_drain_generation, poll.drain_generation); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, + PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); - UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); - UT_ASSERT(ticket_refs_equal(&progress.ref, &ack.ref)); - UT_ASSERT_EQ(progress.local_sequence, leader.local_sequence); - UT_ASSERT_EQ(progress.local_round, leader.local_round); - UT_ASSERT_EQ(progress.member_state, PCM_XL_REMOTE_WAIT); - UT_ASSERT_EQ(progress.role, PCM_X_LOCAL_ROLE_NODE_LEADER); - UT_ASSERT_EQ(progress.reliable_state_sequence, 2); - UT_ASSERT_EQ(progress.pending_opcode, 0); - UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK); - UT_ASSERT_EQ(progress.phase, PCM_X_LOCAL_RELIABLE_PHASE_NONE); + stale_poll = poll; + stale_poll.ref.grant_generation++; + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&stale_poll, 1, master_session), + PCM_X_QUEUE_STALE); + stale_poll = poll; + stale_poll.drain_generation++; + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&stale_poll, 1, master_session), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session + 1), + PCM_X_QUEUE_STALE); - stale = leader; - stale.local_sequence++; - memset(&progress, 0xa5, sizeof(progress)); - memset(&cleared, 0, sizeof(cleared)); - UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&stale, &progress), PCM_X_QUEUE_STALE); - UT_ASSERT(memcmp(&progress, &cleared, sizeof(progress)) == 0); + new_holder_key = make_local_holder_key(7091, 0, 24, UINT64_C(71004), 6); + UT_ASSERT_EQ(register_active_local_holder(&new_holder_key, &new_holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = probe_ref.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = poll.ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &probe_ref.identity.tag, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(release_active_local_holder(&new_holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &probe_ref.identity.tag, &found), + PCM_X_DIRECTORY_NOT_FOUND); + assert_local_queue_baseline(header); } -UT_TEST(test_local_transfer_prepare_commit_and_final_ack_are_exact) -{ - PcmXLocalTagSlot *tag_slot; - PcmXLocalHandle leader; - PcmXLocalHandle follower; - PcmXLocalProgress progress; - PcmXLocalFollowerWfgSnapshot wfg; - PcmXWaitIdentity identity; - PcmXEnqueuePayload enqueue; +typedef struct TestLocalCrossLaneWedge { + PcmXLocalHandle writer; + PcmXLocalWriterClaim writer_claim; + PcmXLocalCutoff writer_cutoff; PcmXAdmitAckPayload admit_ack; - PcmXPhasePayload admit_confirm; - PcmXLocalCutoff cutoff; + PcmXTicketRef external_ref; + PcmXLocalTagSlot *tag_slot; + uint64 master_session; +} TestLocalCrossLaneWedge; + +/* + * Build the exact t/400 cross-lane wedge shape through production paths only. + * + * Older external cohort (external_ticket): this node participates as two S + * holders; PROBE freezes them, the blocker ACK tombstones the set, and DRAIN + * later upgrades the locator in place to the completed grant ref, leaving + * HOLDER_TERMINAL_READY|DRAINED on the tag. + * + * Newer local writer (writer_ticket > external_ticket): joins the same tag + * through join -> claim -> revoke-cutoff (the production REVOKE_BARRIER set + * point, closing a round of one) -> ENQUEUE arm -> ADMIT_ACK (installs the + * ticketed writer ref into tag_slot->ref) -> ADMIT_CONFIRM arm/ack. + * + * Final flags shape is the t/400 dump's 0x31: REVOKE_BARRIER + + * HOLDER_TERMINAL_READY|DRAINED with no writer-terminal bits. + */ +static void +prepare_local_cross_lane_wedge(BlockNumber block, uint64 master_session, uint64 external_ticket, + uint64 writer_ticket, TestLocalCrossLaneWedge *fixture) +{ + PcmXShmemHeader *header; + PcmXLocalHolderKey holder_keys[2]; + PcmXLocalHolderHandle holders[2]; + PcmXLocalHolderHandle holder_copies[2]; PcmXLocalHolderSnapshot holder_snapshot; PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalCutoff participant_cutoff; + PcmXWaitIdentity writer_identity; + PcmXEnqueuePayload enqueue; + PcmXPhasePayload admit_confirm; + PcmXLocalReliableToken token; PcmXTicketRef probe_ref; - PcmXGrantPayload prepare; - PcmXGrantPayload bad_prepare; - PcmXGrantPayload holder_replay; - PcmXRevokePayload revoke; - PcmXInstallReadyPayload install_ready; - PcmXPhasePayload commit; - PcmXFinalAckPayload final_ack; - PcmXPhasePayload final_commit; - PcmXPhasePayload final_confirm; - PcmXPhasePayload duplicate_confirm; PcmXDrainPollPayload poll; - PcmXRetirePayload retire; - PcmXLocalReliableToken token; - PcmXLocalReliableToken retry_token; - PcmXLocalWriterClaim writer_claim; - PcmXLocalWriterClaim stale_writer_claim; - PcmXLocalHolderKey writer_holder_key; - PcmXLocalHolderKey stale_holder_key; - PcmXLocalHolderHandle writer_holder; - PcmXLocalHolderHandle duplicate_holder; - uint64 committed_own_generation; - uint64 holder_set_generation; - Size holder_used; - const uint64 master_session = UINT64_C(178); + uint32 flags; + int i; + UT_ASSERT_NOT_NULL(fixture); + memset(fixture, 0, sizeof(*fixture)); + fixture->master_session = master_session; init_active_pcm_x(UINT64_C(77)); - identity = make_wait_identity(707, 0, 27, UINT64_C(7029)); - bind_local_master(1, identity.cluster_epoch, master_session); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &leader), - PCM_X_QUEUE_OK); - identity = make_wait_identity(707, 0, 28, UINT64_C(7030)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &follower), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &wfg), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(908)), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &writer_claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); - writer_holder_key = make_local_holder_key(707, 0, 27, UINT64_C(7031), 3); - committed_own_generation = UINT64_MAX; - UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( - &writer_holder_key, &writer_claim, &writer_holder, &committed_own_generation), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT_EQ(committed_own_generation, 0); - UT_ASSERT_EQ(writer_holder.tag_slot.slot_index, PCM_X_INVALID_SLOT_INDEX); - UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 0); - UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &token), PCM_X_QUEUE_OK); - memset(&admit_ack, 0, sizeof(admit_ack)); - admit_ack.ref.identity = leader.identity; - admit_ack.ref.handle.ticket_id = UINT64_C(9009); - admit_ack.ref.handle.queue_generation = UINT64_C(9); - admit_ack.prehandle = enqueue.prehandle; - admit_ack.result = PCM_X_QUEUE_OK; - admit_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; - UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&leader, &admit_ack, 1, master_session), + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + for (i = 0; i < 2; i++) { + holder_keys[i] = make_local_holder_key(block, 0, (uint32)(21 + i), + master_session + UINT64_C(1) + (uint64)i, 4 + i); + UT_ASSERT_EQ(register_active_local_holder(&holder_keys[i], &holders[i]), PCM_X_QUEUE_OK); + } + /* The newer local writer queues first through the production chain (join + * -> claim -> revoke-cutoff -> ENQUEUE -> ADMIT). The older external + * cohort was enqueued at the master before it (external_ticket < + * writer_ticket), so its grant-time PROBE reaches this node only after + * the local writer's round has closed -- the t/400 interleaving. */ + writer_identity = make_wait_identity(block, 0, 3, master_session + UINT64_C(4)); + UT_ASSERT_EQ( + cluster_pcm_x_local_join_begin(&writer_identity, 1, master_session, &fixture->writer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(fixture->writer.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&fixture->writer, &fixture->writer_claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_arm_exact(&leader, &admit_confirm, &token), + UT_ASSERT_EQ( + cluster_pcm_x_local_begin_revoke_cutoff_exact(&fixture->writer, &fixture->writer_cutoff), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&fixture->writer, &enqueue, &token), + PCM_X_QUEUE_OK); + memset(&fixture->admit_ack, 0, sizeof(fixture->admit_ack)); + fixture->admit_ack.ref.identity = writer_identity; + fixture->admit_ack.ref.handle.ticket_id = writer_ticket; + fixture->admit_ack.ref.handle.queue_generation = UINT64_C(1); + fixture->admit_ack.prehandle = enqueue.prehandle; + fixture->admit_ack.result = PCM_X_QUEUE_OK; + fixture->admit_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&fixture->writer, &fixture->admit_ack, 1, + master_session), PCM_X_QUEUE_OK); UT_ASSERT_EQ( - cluster_pcm_x_local_admit_confirm_ack_exact(&leader, &admit_confirm, 1, master_session), + cluster_pcm_x_local_admit_confirm_arm_exact(&fixture->writer, &admit_confirm, &token), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&fixture->writer, &admit_confirm, 1, + master_session), + PCM_X_QUEUE_OK); - memset(&prepare, 0, sizeof(prepare)); - prepare.ref = admit_ack.ref; - prepare.ref.grant_generation = UINT64_C(99); - UT_ASSERT(cluster_pcm_x_image_id_encode(1, UINT64_C(199), &prepare.image.image_id)); - prepare.image.source_own_generation = UINT64_C(41); - prepare.image.page_scn = UINT64_C(42); - prepare.image.page_lsn = UINT64_C(43); - prepare.image.source_node = 0; - prepare.image.page_checksum = UINT32_C(44); - probe_ref = prepare.ref; - probe_ref.grant_generation = 0; - UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, - NULL, 0, &holder_snapshot), + /* The older cohort's grant-time PROBE now freezes this node's S holders; + * the blocker ACK tombstones the set. */ + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(block, 3, 23, master_session + UINT64_C(3)); + probe_ref.handle.ticket_id = external_ticket; + probe_ref.handle.queue_generation = UINT64_C(11); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact( + &probe_ref, 1, master_session, holder_copies, 2, &holder_snapshot), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(holder_snapshot.holder_count, 0); + UT_ASSERT_EQ(holder_snapshot.holder_count, 2); UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, - &holder_snapshot, NULL, 0, NULL, 0, - &blocker_snapshot), + &holder_snapshot, holder_copies, 2, + NULL, 0, &blocker_snapshot), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, 1, master_session), PCM_X_QUEUE_OK); - memset(&revoke, 0, sizeof(revoke)); - revoke.ref = prepare.ref; - revoke.image_id = prepare.image.image_id; - UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_image_ready_arm_exact(&prepare, 1, master_session, - &holder_replay), + UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact( + &probe_ref.identity.tag, probe_ref.identity.cluster_epoch, 1, master_session, + &participant_cutoff), PCM_X_QUEUE_OK); - UT_ASSERT(memcmp(&holder_replay, &prepare, sizeof(prepare)) == 0); - bad_prepare = prepare; - bad_prepare.image.image_id = UINT64CONST(0xe0000000000000c7); - UT_ASSERT_EQ(cluster_pcm_x_local_prepare_grant_exact(&leader, &bad_prepare, 1, master_session), - PCM_X_QUEUE_INVALID); - UT_ASSERT_EQ(cluster_pcm_x_local_prepare_grant_exact(&leader, &prepare, 1, master_session), + for (i = 0; i < 2; i++) + UT_ASSERT_EQ(release_active_local_holder(&holders[i]), PCM_X_QUEUE_OK); + + /* The older cohort's DRAIN closes after the newer writer queued. */ + memset(&poll, 0, sizeof(poll)); + poll.ref = probe_ref; + poll.ref.grant_generation = UINT64_C(101); + poll.drain_generation = UINT64_C(42); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); + fixture->external_ref = poll.ref; + + /* Pin the wedge shape: same tag slot for both cohorts, dump flags 0x31, + * a ticketed queued writer and a closed round of exactly one member. */ + UT_ASSERT_EQ(fixture->writer.tag_slot.slot_index, holder_snapshot.tag_slot.slot_index); + fixture->tag_slot = &local_tag_slots(header)[holder_snapshot.tag_slot.slot_index]; + flags = test_slot_flags(&fixture->tag_slot->slot); + UT_ASSERT((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, + PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + UT_ASSERT_EQ(flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK, 0); + UT_ASSERT_EQ(fixture->tag_slot->ref.handle.ticket_id, writer_ticket); + UT_ASSERT_EQ(fixture->tag_slot->ref.handle.queue_generation, UINT64_C(1)); + UT_ASSERT_EQ(fixture->tag_slot->ref.grant_generation, 0); + UT_ASSERT(memcmp(&fixture->tag_slot->ref.identity, &writer_identity, sizeof(writer_identity)) + == 0); + UT_ASSERT_EQ(fixture->tag_slot->membership_count, 1); + UT_ASSERT_EQ(fixture->tag_slot->closed_round_member_count, 1); + UT_ASSERT_EQ(fixture->tag_slot->head_index, fixture->writer.membership_slot.slot_index); + UT_ASSERT_EQ(fixture->tag_slot->tail_index, fixture->writer.membership_slot.slot_index); + UT_ASSERT_EQ(fixture->tag_slot->leader_index, fixture->writer.membership_slot.slot_index); + UT_ASSERT_EQ(fixture->tag_slot->active_writer_index, + fixture->writer.membership_slot.slot_index); + UT_ASSERT(ticket_refs_equal(&fixture->tag_slot->blocker_snapshot_ref, &fixture->external_ref)); + UT_ASSERT_EQ(fixture->tag_slot->holder_terminal_drain_generation, poll.drain_generation); +} + +/* + * RED for the t/400 cross-lane retirement deadlock (2026-07-18): a fully + * DRAINED external holder/blocker terminal (an older granted cohort) must be + * able to retire its own holder lane even while a newer QUEUED writer still + * holds the tag's REVOKE_BARRIER. Before the fix the (REVOKE_BARRIER && + * !same_ref_dual) guard at cluster_pcm_x_convert.c:19192 / :19301 returns + * NOT_READY, the RETIRE watermark never advances, and the FIFO wedges. + * + * Rebuilt 2026-07-19 (review follow-up): the newer writer is a real + * production cohort (join -> claim -> revoke-cutoff -> ENQUEUE -> ADMIT), + * so the barrier, the closed round of one and the ticketed writer ref are + * all genuine; the retire must consume exactly the holder lane, keep every + * writer/round/FIFO byte, and the writer must afterwards progress through + * its full grant chain to the queue baseline (no barrier or tag leak). + */ +UT_TEST(test_local_cross_lane_holder_terminal_retires_under_revoke_barrier) +{ + TestLocalCrossLaneWedge fixture; + PcmXShmemHeader *header; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalTagSlot *tag_slot; + PcmXLocalHandle refreshed; + PcmXLocalProgress progress; + PcmXGrantPayload prepare; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + PcmXLocalReliableToken token; + PcmXTicketRef writer_ref_before; + PcmXReliableLegState writer_reliable_before; + PcmXDrainPollPayload poll; + PcmXRetirePayload retire; + PcmXSlotRef found; + const uint64 master_session = UINT64_C(1801); + uint64 cutoff_sequence_before; + uint64 next_sequence_before; + Size membership_before; + Size closed_before; + Size head_before; + Size tail_before; + Size leader_before; + Size active_writer_before; + uint32 local_round_before; + + prepare_local_cross_lane_wedge(7115, master_session, UINT64_C(95001), UINT64_C(95002), + &fixture); + header = ClusterPcmXConvertShmem; + tag_slot = fixture.tag_slot; + writer_ref_before = tag_slot->ref; + writer_reliable_before = tag_slot->reliable; + cutoff_sequence_before = tag_slot->cutoff_sequence; + next_sequence_before = tag_slot->next_sequence; + membership_before = tag_slot->membership_count; + closed_before = tag_slot->closed_round_member_count; + head_before = tag_slot->head_index; + tail_before = tag_slot->tail_index; + leader_before = tag_slot->leader_index; + active_writer_before = tag_slot->active_writer_index; + local_round_before = tag_slot->local_round; + + /* RETIRE watermark covers only the older external ticket. */ + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.external_ref.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = fixture.external_ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_prepare_grant_exact(&leader, &prepare, 1, master_session), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); + + /* Exactly the holder lane was consumed: blocker locator, its reliable leg + * and the drain generation are gone and HOLDER_TERMINAL_MASK cleared. */ + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, 0); + UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &(PcmXTicketRef){ 0 })); + UT_ASSERT_EQ(tag_slot->blocker_set_generation, 0); + UT_ASSERT_EQ(tag_slot->holder_terminal_drain_generation, 0); + + /* The newer writer's round is byte-stable: barrier, ticketed ref, closed + * round, FIFO indexes and the writer reliable leg are all untouched. */ + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_TERMINAL_MASK, 0); + UT_ASSERT(ticket_refs_equal(&tag_slot->ref, &writer_ref_before)); + UT_ASSERT(memcmp(&tag_slot->reliable, &writer_reliable_before, sizeof(writer_reliable_before)) + == 0); + UT_ASSERT_EQ(tag_slot->cutoff_sequence, cutoff_sequence_before); + UT_ASSERT_EQ(tag_slot->next_sequence, next_sequence_before); + UT_ASSERT_EQ(tag_slot->membership_count, membership_before); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, closed_before); + UT_ASSERT_EQ(tag_slot->head_index, head_before); + UT_ASSERT_EQ(tag_slot->tail_index, tail_before); + UT_ASSERT_EQ(tag_slot->leader_index, leader_before); + UT_ASSERT_EQ(tag_slot->active_writer_index, active_writer_before); + UT_ASSERT_EQ(tag_slot->local_round, local_round_before); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, + &fixture.external_ref.identity.tag, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&fixture.writer.identity, &refreshed), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(refreshed.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&fixture.writer, &progress), PCM_X_QUEUE_OK); UT_ASSERT_EQ(progress.member_state, PCM_XL_REMOTE_WAIT); - UT_ASSERT_EQ(progress.pending_opcode, 0); - UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT); - UT_ASSERT(ticket_refs_equal(&progress.ref, &prepare.ref)); - UT_ASSERT(memcmp(&progress.image, &prepare.image, sizeof(progress.image)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_local_install_ready_arm_exact(&leader, &prepare.ref, &prepare.image, - &install_ready, &token), + /* The unwedged writer must now progress through its full production grant + * chain and drain to the queue baseline -- the deadlock did not merely + * move into a barrier or tag leak. Its closed revoke round has no S + * holders left (new pins are correctly fenced BARRIER_CLOSED), so the + * writer's own PROBE freezes an empty set. */ + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact( + &fixture.admit_ack.ref, 1, master_session, &holder_copy, 1, &holder_snapshot), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(install_ready.image_id, prepare.image.image_id); - UT_ASSERT_EQ(install_ready.phase, PGRAC_IC_MSG_PCM_X_INSTALL_READY); - UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(progress.member_state, PCM_XL_CONTENT_ACTIVE); - UT_ASSERT_EQ(progress.pending_opcode, PGRAC_IC_MSG_PCM_X_INSTALL_READY); - UT_ASSERT_EQ(cluster_pcm_x_local_reliable_snapshot_exact(&leader, &retry_token), + UT_ASSERT_EQ(holder_snapshot.holder_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact( + &fixture.admit_ack.ref, 1, master_session, &holder_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(retry_token.pending_opcode, PGRAC_IC_MSG_PCM_X_INSTALL_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact( + &fixture.admit_ack.ref, blocker_snapshot.set_generation, 1, master_session), + PCM_X_QUEUE_OK); + + memset(&prepare, 0, sizeof(prepare)); + prepare.ref = fixture.admit_ack.ref; + prepare.ref.grant_generation = UINT64_C(102); + UT_ASSERT( + cluster_pcm_x_image_id_encode(1, master_session + UINT64_C(200), &prepare.image.image_id)); + prepare.image.source_own_generation = UINT64_C(9); + prepare.image.page_scn = UINT64_C(10); + prepare.image.page_lsn = UINT64_C(11); + prepare.image.source_node = 2; + prepare.image.page_checksum = UINT32_C(12); UT_ASSERT_EQ( - cluster_pcm_x_local_reliable_retry_exact(&leader, &token, UINT64_C(1001), &retry_token), + cluster_pcm_x_local_prepare_grant_exact(&fixture.writer, &prepare, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(retry_token.retry_count, 1); - UT_ASSERT_EQ(retry_token.retry_deadline_ms, UINT64_C(1001)); + UT_ASSERT_EQ(cluster_pcm_x_local_install_ready_arm_exact( + &fixture.writer, &prepare.ref, &prepare.image, &install_ready, &token), + PCM_X_QUEUE_OK); memset(&commit, 0, sizeof(commit)); commit.ref = prepare.ref; commit.phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; - UT_ASSERT_EQ(cluster_pcm_x_local_commit_x_exact(&leader, &commit, 1, master_session), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_commit_x_exact(&leader, &commit, 1, master_session), - PCM_X_QUEUE_DUPLICATE); - - UT_ASSERT_EQ(cluster_pcm_x_local_final_ack_arm_exact(&leader, 1, &final_ack, &token), + UT_ASSERT_EQ(cluster_pcm_x_local_commit_x_exact(&fixture.writer, &commit, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(final_ack.image_id, prepare.image.image_id); - UT_ASSERT_EQ(final_ack.committed_own_generation, 1); - UT_ASSERT_EQ(cluster_pcm_x_local_reliable_snapshot_exact(&leader, &retry_token), + UT_ASSERT_EQ(cluster_pcm_x_local_final_ack_arm_exact(&fixture.writer, 1, &final_ack, &token), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(retry_token.pending_opcode, PGRAC_IC_MSG_PCM_X_FINAL_ACK); - UT_ASSERT_EQ( - cluster_pcm_x_local_reliable_retry_exact(&leader, &token, UINT64_C(1002), &retry_token), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(retry_token.retry_count, 1); - UT_ASSERT_EQ(retry_token.retry_deadline_ms, UINT64_C(1002)); memset(&final_commit, 0, sizeof(final_commit)); final_commit.ref = prepare.ref; final_commit.phase = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; - UT_ASSERT_EQ(cluster_pcm_x_local_final_commit_ack_exact(&leader, &final_commit, 1, + UT_ASSERT_EQ(cluster_pcm_x_local_final_commit_ack_exact(&fixture.writer, &final_commit, 1, master_session, &final_confirm, &token), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(final_confirm.phase, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM); - UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(progress.member_state, PCM_XL_GRANTED); - UT_ASSERT_EQ(progress.pending_opcode, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM); - UT_ASSERT_EQ(progress.phase, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM); - UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK); - UT_ASSERT_EQ(cluster_pcm_x_local_reliable_snapshot_exact(&leader, &retry_token), + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&fixture.writer_claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(retry_token.pending_opcode, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM); - UT_ASSERT_EQ( - cluster_pcm_x_local_reliable_retry_exact(&leader, &token, UINT64_C(1003), &retry_token), - PCM_X_QUEUE_OK); - UT_ASSERT_EQ(retry_token.retry_count, 1); - UT_ASSERT_EQ(retry_token.retry_deadline_ms, UINT64_C(1003)); - UT_ASSERT_EQ(local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index] - .committed_own_generation, - 1); - UT_ASSERT_EQ(cluster_pcm_x_local_final_commit_ack_exact( - &leader, &final_commit, 1, master_session, &duplicate_confirm, &token), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(memcmp(&duplicate_confirm, &final_confirm, sizeof(final_confirm)) == 0); - tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; + memset(&poll, 0, sizeof(poll)); + poll.ref = prepare.ref; + poll.drain_generation = UINT64_C(43); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = prepare.ref.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = prepare.ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&fixture.writer, &progress), + PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, + &fixture.external_ref.identity.tag, &found), + PCM_X_DIRECTORY_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + assert_local_queue_baseline(header); +} + +/* + * Defense probes for the cross-lane retirement exemption: it must stay + * strictly cross-lane. A holder lane that aliases the writer lane's ref (or + * collides with its ticket id) keeps the pre-exemption NOT_READY verdict. + * Both alias states are protocol-impossible (master tickets are unique and + * same-ref forms route through the same-ref-dual path), so they are built by + * direct slot pokes like the CORRUPT arms elsewhere in this file. A distinct + * OLDER writer-lane ref, however, is legal (a cancelled predecessor leader's + * duplicate-ACK tombstone lingers in tag_slot->ref until the successor's + * ENQUEUE arm retires it), so the exemption deliberately has no ticket-order + * requirement: refusing there could wedge the empty-frozen path permanently + * if the successor dies before arming. + */ +UT_TEST(test_local_cross_lane_retire_exemption_requires_distinct_ticket) +{ + TestLocalCrossLaneWedge fixture; + PcmXLocalTagSlot *tag_slot; + PcmXTicketRef saved_ref; + PcmXRetirePayload retire; + const uint64 master_session = UINT64_C(1809); + + prepare_local_cross_lane_wedge(7116, master_session, UINT64_C(95001), UINT64_C(95002), + &fixture); + tag_slot = fixture.tag_slot; + saved_ref = tag_slot->ref; + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.external_ref.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = fixture.external_ref.handle.ticket_id; + retire.sender_node = 0; + + /* Sub-case order matters: the two NOT_READY alias probes must run before + * the distinct-older-ticket case, which consumes the holder lane. */ + + /* Full alias: writer lane ref == external holder lane ref. */ + tag_slot->ref = fixture.external_ref; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_NOT_READY); + + /* Ticket-id collision under a distinct identity. */ + tag_slot->ref = saved_ref; + tag_slot->ref.handle.ticket_id = fixture.external_ref.handle.ticket_id; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_NOT_READY); + + /* Distinct older writer-lane ref (cancelled-predecessor tombstone shape): + * the exemption must still let the external holder lane retire. */ + tag_slot->ref = saved_ref; + tag_slot->ref.handle.ticket_id = fixture.external_ref.handle.ticket_id - 1; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, 0); UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); - UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) == 0); - UT_ASSERT_EQ(tag_slot->leader_index, leader.membership_slot.slot_index); - UT_ASSERT_EQ(tag_slot->active_writer_index, leader.membership_slot.slot_index); - UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM); - UT_ASSERT(ticket_refs_equal(&tag_slot->ref, &prepare.ref)); - UT_ASSERT_EQ(tag_slot->master_node, 1); - UT_ASSERT_EQ(tag_slot->master_session_incarnation, master_session); - UT_ASSERT_EQ(tag_slot->cluster_epoch, identity.cluster_epoch); - UT_ASSERT_EQ(ClusterPcmXConvertShmem->peer_frontiers[1].cluster_epoch, identity.cluster_epoch); - UT_ASSERT_EQ(ClusterPcmXConvertShmem->peer_frontiers[1].sender_session_incarnation, - master_session); - UT_ASSERT_EQ(membership_slots(ClusterPcmXConvertShmem)[leader.membership_slot.slot_index] - .handle.ticket_id, - prepare.ref.handle.ticket_id); - UT_ASSERT_EQ( - test_slot_state( - &membership_slots(ClusterPcmXConvertShmem)[leader.membership_slot.slot_index].slot), - PCM_XL_GRANTED); + tag_slot->ref = saved_ref; + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} - writer_holder_key.identity.base_own_generation = 1; - UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&writer_holder_key, &writer_holder), - PCM_X_QUEUE_BARRIER_CLOSED); - holder_used = ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used; - holder_set_generation = tag_slot->holder_set_generation; - stale_writer_claim = writer_claim; - stale_writer_claim.claim_generation++; - committed_own_generation = UINT64_MAX; - UT_ASSERT_EQ( - cluster_pcm_x_local_writer_holder_register_exact(&writer_holder_key, &stale_writer_claim, - &writer_holder, &committed_own_generation), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(committed_own_generation, 0); - stale_writer_claim = writer_claim; - stale_writer_claim.local_round++; +typedef struct TestLocalIndependentDualTerminal { + PcmXLocalHandle writer; + PcmXTicketRef writer_ref; + PcmXTicketRef holder_ref; + PcmXLocalTagSlot *tag_slot; + uint64 master_session; +} TestLocalIndependentDualTerminal; + +/* + * Build the exact t/400 independent dual-terminal shape (dump flags 0x3d) + * through production paths only, in the production interleaving: + * + * Older local writer (writer_ticket): full grant chain (join -> claim -> + * revoke-cutoff -> ENQUEUE -> ADMIT -> its own empty PROBE -> PREPARE_GRANT + * -> INSTALL_READY -> COMMIT_X -> FINAL_ACK -> FINAL_COMMIT_ACK), with its + * writer-lane DRAIN deliberately delayed. + * + * Newer external cohort (holder_ticket > writer_ticket): REVOKEs this node + * as its image source before that delayed DRAIN lands (empty PROBE -> + * blocker ACK -> REVOKE apply publishes holder_ref -> IMAGE_READY). Both + * lanes then DRAIN, leaving TERMINAL_READY|DRAINED on the writer lane and + * HOLDER_TERMINAL_READY|DRAINED on the holder lane under one REVOKE_BARRIER + * with two DISTINCT master tickets -- the 2026-07-19 t/400 first-fuse shape. + */ +static void +prepare_local_independent_dual_terminal(BlockNumber block, uint64 master_session, + uint64 writer_ticket, uint64 holder_ticket, + bool cancelled_external, + TestLocalIndependentDualTerminal *fixture) +{ + PcmXShmemHeader *header; + PcmXLocalWriterClaim writer_claim; + PcmXLocalCutoff writer_cutoff; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXWaitIdentity writer_identity; + PcmXAdmitAckPayload admit_ack; + PcmXEnqueuePayload enqueue; + PcmXPhasePayload admit_confirm; + PcmXGrantPayload prepare; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + PcmXLocalReliableToken token; + PcmXTicketRef probe_ref; + PcmXRevokePayload revoke; + PcmXGrantPayload ready; + PcmXGrantPayload replay; + PcmXDrainPollPayload poll; + uint32 flags; + + UT_ASSERT_NOT_NULL(fixture); + memset(fixture, 0, sizeof(*fixture)); + fixture->master_session = master_session; + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + + /* Older local writer through the full production grant chain. */ + writer_identity = make_wait_identity(block, 0, 3, master_session + UINT64_C(4)); UT_ASSERT_EQ( - cluster_pcm_x_local_writer_holder_register_exact(&writer_holder_key, &stale_writer_claim, - &writer_holder, &committed_own_generation), - PCM_X_QUEUE_STALE); - stale_holder_key = writer_holder_key; - stale_holder_key.identity.base_own_generation = 0; - UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( - &stale_holder_key, &writer_claim, &writer_holder, &committed_own_generation), - PCM_X_QUEUE_STALE); - stale_holder_key.identity.base_own_generation = 2; - UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( - &stale_holder_key, &writer_claim, &writer_holder, &committed_own_generation), - PCM_X_QUEUE_STALE); - stale_holder_key = writer_holder_key; - stale_holder_key.identity.procno++; - UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( - &stale_holder_key, &writer_claim, &writer_holder, &committed_own_generation), - PCM_X_QUEUE_STALE); - stale_holder_key = writer_holder_key; - stale_holder_key.identity.xid++; - UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( - &stale_holder_key, &writer_claim, &writer_holder, &committed_own_generation), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, holder_used); - UT_ASSERT_EQ(tag_slot->holder_set_generation, holder_set_generation); - UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( - &writer_holder_key, &writer_claim, &writer_holder, &committed_own_generation), + cluster_pcm_x_local_join_begin(&writer_identity, 1, master_session, &fixture->writer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(fixture->writer.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&fixture->writer, &writer_claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(committed_own_generation, 1); - stale_writer_claim = writer_claim; - stale_writer_claim.claim_generation++; + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&fixture->writer, &writer_cutoff), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&fixture->writer, &enqueue, &token), + PCM_X_QUEUE_OK); + memset(&admit_ack, 0, sizeof(admit_ack)); + admit_ack.ref.identity = writer_identity; + admit_ack.ref.handle.ticket_id = writer_ticket; + admit_ack.ref.handle.queue_generation = UINT64_C(1); + admit_ack.prehandle = enqueue.prehandle; + admit_ack.result = PCM_X_QUEUE_OK; + admit_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; UT_ASSERT_EQ( - cluster_pcm_x_local_writer_holder_register_exact( - &writer_holder_key, &stale_writer_claim, &duplicate_holder, &committed_own_generation), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact(&writer_holder_key, &writer_claim, - &duplicate_holder, - &committed_own_generation), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(memcmp(&duplicate_holder, &writer_holder, sizeof(writer_holder)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&writer_holder), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(release_active_local_holder(&writer_holder), PCM_X_QUEUE_OK); - - memset(&poll, 0, sizeof(poll)); - poll.ref = prepare.ref; - poll.drain_generation = UINT64_C(43); - UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT_EQ(tag_slot->leader_index, leader.membership_slot.slot_index); - UT_ASSERT_EQ(tag_slot->active_writer_index, leader.membership_slot.slot_index); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&writer_claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(908)), + cluster_pcm_x_local_apply_admit_ack_exact(&fixture->writer, &admit_ack, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_admit_confirm_arm_exact(&fixture->writer, &admit_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&fixture->writer, &admit_confirm, 1, + master_session), + PCM_X_QUEUE_OK); + /* The writer's grant-time PROBE targets only nodes holding S pins; this + * node has none, so no blocker tombstone is planted here (the production + * shape: the later external PROBE must not find a stale locator). */ + memset(&prepare, 0, sizeof(prepare)); + prepare.ref = admit_ack.ref; + prepare.ref.grant_generation = UINT64_C(2); + UT_ASSERT( + cluster_pcm_x_image_id_encode(1, master_session + UINT64_C(300), &prepare.image.image_id)); + prepare.image.source_own_generation = UINT64_C(9); + prepare.image.page_scn = UINT64_C(10); + prepare.image.page_lsn = UINT64_C(11); + prepare.image.source_node = 2; + prepare.image.page_checksum = UINT32_C(12); + UT_ASSERT_EQ( + cluster_pcm_x_local_prepare_grant_exact(&fixture->writer, &prepare, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_install_ready_arm_exact( + &fixture->writer, &prepare.ref, &prepare.image, &install_ready, &token), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &writer_claim), PCM_X_QUEUE_OK); - writer_holder_key = make_local_holder_key(707, 0, 28, UINT64_C(7032), 4); - writer_holder_key.identity.base_own_generation = 1; - UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( - &writer_holder_key, &writer_claim, &writer_holder, &committed_own_generation), + memset(&commit, 0, sizeof(commit)); + commit.ref = prepare.ref; + commit.phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; + UT_ASSERT_EQ(cluster_pcm_x_local_commit_x_exact(&fixture->writer, &commit, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_final_ack_arm_exact(&fixture->writer, 1, &final_ack, &token), + PCM_X_QUEUE_OK); + memset(&final_commit, 0, sizeof(final_commit)); + final_commit.ref = prepare.ref; + final_commit.phase = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; + UT_ASSERT_EQ(cluster_pcm_x_local_final_commit_ack_exact(&fixture->writer, &final_commit, 1, + master_session, &final_confirm, &token), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(committed_own_generation, 1); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&writer_holder), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(release_active_local_holder(&writer_holder), PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&writer_claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower), PCM_X_QUEUE_OK); + fixture->writer_ref = prepare.ref; + + /* The newer external cohort REVOKEs this node as image source before the + * writer-lane DRAIN lands (the production interleaving). */ + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(block, 3, 23, master_session + UINT64_C(3)); + probe_ref.handle.ticket_id = holder_ticket; + probe_ref.handle.queue_generation = UINT64_C(11); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + &holder_copy, 1, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holder_snapshot.holder_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + if (cancelled_external) { + /* The external cohort cancels: no REVOKE/IMAGE_READY; its DRAIN + * upgrades the blocker locator in place with a zero grant + * generation (the cancelled blocker-participant form). */ + fixture->holder_ref = probe_ref; + } else { + memset(&revoke, 0, sizeof(revoke)); + revoke.ref = probe_ref; + revoke.ref.grant_generation = UINT64_C(2); + UT_ASSERT( + cluster_pcm_x_image_id_encode(1, master_session + UINT64_C(400), &revoke.image_id)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + PCM_X_QUEUE_OK); + memset(&ready, 0, sizeof(ready)); + ready.ref = revoke.ref; + ready.image.image_id = revoke.image_id; + ready.image.source_own_generation = UINT64_C(61); + ready.image.page_scn = UINT64_C(62); + ready.image.page_lsn = UINT64_C(63); + ready.image.source_node = 0; + ready.image.page_checksum = UINT32_C(64); + UT_ASSERT_EQ( + cluster_pcm_x_local_holder_image_ready_arm_exact(&ready, 1, master_session, &replay), + PCM_X_QUEUE_OK); + fixture->holder_ref = revoke.ref; + } + + /* Both lanes now DRAIN: the delayed writer leg first, then the holder. */ + memset(&poll, 0, sizeof(poll)); + poll.ref = fixture->writer_ref; + poll.drain_generation = UINT64_C(7); UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(progress.member_state, PCM_XL_GRANTED); - UT_ASSERT_EQ(progress.pending_opcode, 0); - UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_DRAIN_POLL); + memset(&poll, 0, sizeof(poll)); + poll.ref = fixture->holder_ref; + poll.drain_generation = UINT64_C(7); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); + + /* Pin the t/400 dump shape byte-exactly: flags 0x3d, one closed round of + * one, two distinct tickets, one unlinked terminal membership, every + * FIFO index invalid. */ + fixture->tag_slot = &local_tag_slots(header)[fixture->writer.tag_slot.slot_index]; + flags = test_slot_flags(&fixture->tag_slot->slot); + UT_ASSERT_EQ(flags, PCM_X_LOCAL_TAG_F_REVOKE_BARRIER | PCM_X_LOCAL_TAG_F_TERMINAL_MASK + | PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + UT_ASSERT_EQ(fixture->tag_slot->closed_round_member_count, 1); + UT_ASSERT_EQ(fixture->tag_slot->ref.handle.ticket_id, writer_ticket); + UT_ASSERT_EQ(fixture->tag_slot->ref.grant_generation, UINT64_C(2)); + if (cancelled_external) { + UT_ASSERT(ticket_refs_equal(&fixture->tag_slot->holder_ref, &(PcmXTicketRef){ 0 })); + UT_ASSERT_EQ(fixture->tag_slot->blocker_snapshot_ref.handle.ticket_id, holder_ticket); + UT_ASSERT_EQ(fixture->tag_slot->blocker_snapshot_ref.grant_generation, 0); + } else { + UT_ASSERT_EQ(fixture->tag_slot->holder_ref.handle.ticket_id, holder_ticket); + } + UT_ASSERT_EQ(fixture->tag_slot->membership_count, 1); + UT_ASSERT_EQ(fixture->tag_slot->head_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(fixture->tag_slot->tail_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(fixture->tag_slot->leader_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(fixture->tag_slot->active_writer_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT(fixture->tag_slot->terminal_drain_generation != 0); + UT_ASSERT(fixture->tag_slot->holder_terminal_drain_generation != 0); +} + +/* + * RED for the t/400 independent dual-terminal false fuse (2026-07-19): a + * fully DRAINED older writer terminal (T) and a fully DRAINED newer holder + * terminal (H, distinct master ticket) legally share one tag under the + * writer round's REVOKE_BARRIER. Before the fix the same-ref-dual + * classifier coerces the distinct-ticket shape to CORRUPT (the ref-equality + * arm of pcm_x_local_same_ref_dual_retire_state), RETIRE_UP_TO(T) blocks the + * runtime, and the master's RETIRE leg retries forever -- the 20088-family + * first fuse in loop3/loop4b. After the fix RETIRE_UP_TO(T) must consume + * exactly the writer lane (holder lane byte-exact, tag LIVE) and a later + * RETIRE_UP_TO(H) must drain the tag to the queue baseline. + */ +UT_TEST(test_local_independent_dual_terminal_retires_writer_then_holder) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalTagSlot *tag_slot; + PcmXLocalProgress progress; + PcmXTicketRef holder_ref_before; + PcmXImageToken holder_image_before; + PcmXReliableLegState holder_reliable_before; + PcmXRetirePayload retire; + PcmXSlotRef found; + uint64 holder_drain_before; + uint64 reliable_sequence_before; + const uint64 master_session = UINT64_C(1811); + + prepare_local_independent_dual_terminal(7117, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + holder_ref_before = tag_slot->holder_ref; + holder_image_before = tag_slot->holder_image; + holder_reliable_before = tag_slot->holder_reliable; + holder_drain_before = tag_slot->holder_terminal_drain_generation; + reliable_sequence_before = tag_slot->reliable.state_sequence; + /* RETIRE watermark covers only the older writer ticket. */ memset(&retire, 0, sizeof(retire)); - retire.cluster_epoch = identity.cluster_epoch; + retire.cluster_epoch = fixture.writer_ref.identity.cluster_epoch; retire.master_session_incarnation = master_session; - retire.retire_through_ticket_id = prepare.ref.handle.ticket_id; + retire.retire_through_ticket_id = fixture.writer_ref.handle.ticket_id; retire.sender_node = 0; UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + + /* Exactly the writer lane was consumed... */ + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_TERMINAL_MASK, 0); + UT_ASSERT(ticket_refs_equal(&tag_slot->ref, &(PcmXTicketRef){ 0 })); + UT_ASSERT_EQ(tag_slot->terminal_drain_generation, 0); + UT_ASSERT_EQ(tag_slot->committed_own_generation, 0); + UT_ASSERT_EQ(tag_slot->membership_count, 0); + UT_ASSERT_EQ(tag_slot->reliable.state_sequence, reliable_sequence_before); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&fixture.writer, &progress), + PCM_X_QUEUE_NOT_FOUND); + + /* ...while the holder lane, its master binding and the barrier survive + * byte-exact on the still-LIVE tag. */ + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, + PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + UT_ASSERT(ticket_refs_equal(&tag_slot->holder_ref, &holder_ref_before)); + UT_ASSERT(memcmp(&tag_slot->holder_image, &holder_image_before, sizeof(holder_image_before)) + == 0); + UT_ASSERT( + memcmp(&tag_slot->holder_reliable, &holder_reliable_before, sizeof(holder_reliable_before)) + == 0); + UT_ASSERT_EQ(tag_slot->holder_terminal_drain_generation, holder_drain_before); + UT_ASSERT_EQ(tag_slot->master_node, 1); + UT_ASSERT_EQ(tag_slot->master_session_incarnation, master_session); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &fixture.writer_ref.identity.tag, &found), + PCM_X_DIRECTORY_OK); + + /* The later watermark drains the holder lane and the tag to baseline. */ + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.holder_ref.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = fixture.holder_ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &fixture.writer_ref.identity.tag, &found), + PCM_X_DIRECTORY_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); assert_local_queue_baseline(ClusterPcmXConvertShmem); } -UT_TEST(test_local_tag_only_holder_transfer_persists_until_exact_drain) +/* + * Defense probes for the independent dual-terminal classification: it must + * stay strictly distinct-ticket. A ticket-id collision under a different + * ref, a half-drained writer mask and a poisoned holder drain leg are all + * protocol-impossible shapes and must keep the CORRUPT fail-closed verdict + * (direct slot pokes, like the CORRUPT arms elsewhere in this file). + */ +UT_TEST(test_local_independent_dual_terminal_alias_and_malformed_lanes_fail_closed) { - PcmXShmemHeader *header; - PcmXLocalHolderKey new_holder_key; - PcmXLocalHolderHandle new_holder; - PcmXLocalHolderSnapshot holder_snapshot; - PcmXLocalBlockerSnapshot blocker_snapshot; - PcmXBlockerSetHeaderPayload blocker_header; - PcmXEnqueuePayload enqueue; - PcmXLocalFollowerWfgSnapshot wfg; - PcmXLocalCutoff cutoff; - PcmXLocalCutoff promoted_cutoff; - PcmXLocalHandle late; - PcmXLocalHandle next_leader; - PcmXLocalReliableToken token; - PcmXLocalWriterClaim writer_claim; - PcmXTicketRef probe_ref; - PcmXLocalHolderProgress progress; - PcmXRevokePayload revoke; - PcmXRevokePayload bad_revoke; - PcmXGrantPayload ready; - PcmXGrantPayload replay; - PcmXDrainPollPayload poll; + TestLocalIndependentDualTerminal fixture; + PcmXLocalTagSlot *tag_slot; PcmXRetirePayload retire; - PcmXWaitIdentity identity; - PcmXWaitIdentity late_identity; - PcmXWaitIdentity wake_items[1]; - PcmXLocalWakeBatch wake_batch; - const uint64 master_session = UINT64_C(179); + const uint64 master_session = UINT64_C(1813); - init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; - identity = make_wait_identity(708, 2, 28, UINT64_C(7030)); - bind_local_master(1, identity.cluster_epoch, master_session); - memset(&revoke, 0, sizeof(revoke)); - revoke.ref.identity = identity; - revoke.ref.handle.ticket_id = UINT64_C(9010); - revoke.ref.handle.queue_generation = UINT64_C(10); - revoke.ref.grant_generation = UINT64_C(100); - UT_ASSERT(cluster_pcm_x_image_id_encode(1, UINT64_C(200), &revoke.image_id)); - probe_ref = revoke.ref; - probe_ref.grant_generation = 0; - UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&identity.tag, NULL, 0, &holder_snapshot), + /* Ticket-id collision under a distinct holder ref. */ + prepare_local_independent_dual_terminal(7118, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + tag_slot->holder_ref.handle.ticket_id = fixture.writer_ref.handle.ticket_id; + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.writer_ref.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = fixture.writer_ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* Half-drained writer mask: READY without DRAINED is not independent. */ + prepare_local_independent_dual_terminal(7118, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + test_clear_slot_flags(&tag_slot->slot, PCM_X_LOCAL_TAG_F_TERMINAL_DRAINED); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* Poisoned holder drain leg: a response tombstone is never clean. */ + prepare_local_independent_dual_terminal(7118, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + tag_slot->holder_reliable.response_tombstone_mask = UINT32_C(1); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +/* Join one next-round follower on the fixture's frozen tag; production + * admission stays legal under the barrier and links it behind the round. */ +static void +join_independent_dual_follower(const TestLocalIndependentDualTerminal *fixture, uint32 procno, + uint64 request_id, PcmXLocalHandle *follower_out) +{ + PcmXWaitIdentity identity + = make_wait_identity(fixture->writer_ref.identity.tag.blockNum, 0, procno, request_id); + + UT_ASSERT_EQ( + cluster_pcm_x_local_join_begin(&identity, 1, fixture->master_session, follower_out), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(follower_out->role, PCM_X_LOCAL_ROLE_FOLLOWER); +} + +static void +retire_watermark(uint64 cluster_epoch, uint64 master_session, uint64 through_ticket, + PcmXRetirePayload *retire_out) +{ + memset(retire_out, 0, sizeof(*retire_out)); + retire_out->cluster_epoch = cluster_epoch; + retire_out->master_session_incarnation = master_session; + retire_out->retire_through_ticket_id = through_ticket; + retire_out->sender_node = 0; +} + +/* + * RED (2026-07-19 adversarial review, leg A1): a next-round follower that + * joined BEFORE the writer's RETIRE must not push the independent dual out + * of eligibility -- the follower waits for exactly this barrier to drop, so + * refusing the writer lane here is a deterministic deadlock. The holder + * ticket's later RETIRE must close the frozen round and promote it. + */ +UT_TEST(test_local_independent_dual_terminal_supports_pre_joined_follower) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalHandle follower; + PcmXLocalHandle refreshed; + PcmXLocalTagSlot *tag_slot; + PcmXRetirePayload retire; + PcmXSlotRef found; + const uint64 master_session = UINT64_C(1815); + + prepare_local_independent_dual_terminal(7119, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + join_independent_dual_follower(&fixture, 5, master_session + UINT64_C(10), &follower); + UT_ASSERT_EQ(tag_slot->membership_count, 2); + + retire_watermark(fixture.writer_ref.identity.cluster_epoch, master_session, + fixture.writer_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(holder_snapshot.tag_slot.slot_index, PCM_X_INVALID_SLOT_INDEX); - UT_ASSERT_EQ(holder_snapshot.holder_count, 0); - UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, - NULL, 0, &holder_snapshot), + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_TERMINAL_MASK, 0); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(tag_slot->membership_count, 1); + UT_ASSERT_EQ(tag_slot->holder_ref.handle.ticket_id, fixture.holder_ref.handle.ticket_id); + + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(holder_snapshot.holder_count, 0); - UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, - &holder_snapshot, NULL, 0, NULL, 0, - &blocker_snapshot), + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER, 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&follower.identity, &refreshed), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(refreshed.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &fixture.writer_ref.identity.tag, &found), + PCM_X_DIRECTORY_OK); +} + +/* + * RED (leg A2): the same promotion must hold when the follower joins in the + * window BETWEEN the writer's RETIRE and the holder's RETIRE -- before the + * fix the holder detach consumed its lane, answered OK and left the barrier + * standing over the follower with no terminal authority left to drop it. + */ +UT_TEST(test_local_independent_dual_terminal_promotes_follower_joined_between_watermarks) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalHandle follower; + PcmXLocalHandle refreshed; + PcmXLocalTagSlot *tag_slot; + PcmXRetirePayload retire; + const uint64 master_session = UINT64_C(1817); + + prepare_local_independent_dual_terminal(7120, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + retire_watermark(fixture.writer_ref.identity.cluster_epoch, master_session, + fixture.writer_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_copy_exact(&blocker_snapshot, &blocker_header, - NULL, 0), + join_independent_dual_follower(&fixture, 5, master_session + UINT64_C(10), &follower); + UT_ASSERT_EQ(tag_slot->membership_count, 1); + + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(blocker_header.nblockers, 0); - UT_ASSERT_EQ(blocker_header.set_crc32c, blocker_set_crc32c(NULL, 0)); - late_identity = make_wait_identity(708, 0, 6, UINT64_C(7031)); - UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&late_identity, 1, master_session, &late), + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&follower.identity, &refreshed), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(refreshed.role, PCM_X_LOCAL_ROLE_NODE_LEADER); +} + +/* + * RED (leg B): a cancelled external cohort (zero grant generation on the + * blocker locator) is the staged form of the independent dual -- the writer + * lane retires first, then the cancelled lane by its own watermark. Before + * the fix the writer RETIRE either fused (classifier) or wedged forever + * (asymmetric gen!=0 eligibility). + */ +UT_TEST(test_local_independent_dual_terminal_staged_retire_of_cancelled_external) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalTagSlot *tag_slot; + PcmXRetirePayload retire; + PcmXSlotRef found; + const uint64 master_session = UINT64_C(1819); + + prepare_local_independent_dual_terminal(7121, master_session, UINT64_C(96001), UINT64_C(96002), + true, &fixture); + tag_slot = fixture.tag_slot; + retire_watermark(fixture.writer_ref.identity.cluster_epoch, master_session, + fixture.writer_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(late.role, PCM_X_LOCAL_ROLE_FOLLOWER); - UT_ASSERT_EQ(late.local_round, 2); - UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact( - &identity.tag, identity.cluster_epoch, 1, master_session, &cutoff), + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_TERMINAL_MASK, 0); + UT_ASSERT_EQ(tag_slot->blocker_snapshot_ref.handle.ticket_id, + fixture.holder_ref.handle.ticket_id); + + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cutoff.cutoff_sequence, 0); - UT_ASSERT_EQ(cutoff.closed_round, 1); - UT_ASSERT_EQ(cutoff.next_round, 2); - UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&late, &wfg), - PCM_X_QUEUE_BARRIER_CLOSED); UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); - UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&identity.tag, identity.cluster_epoch, - &cutoff, &next_leader), - PCM_X_QUEUE_NOT_READY); - bad_revoke = revoke; - bad_revoke.image_id = UINT64CONST(0xf0000000000000c8); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&bad_revoke, 1, master_session), - PCM_X_QUEUE_INVALID); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), - PCM_X_QUEUE_NOT_READY); - UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, - 1, master_session), + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &fixture.writer_ref.identity.tag, &found), + PCM_X_DIRECTORY_NOT_FOUND); + assert_local_queue_baseline(ClusterPcmXConvertShmem); +} + +/* + * RED (leg: single watermark): one RETIRE watermark covering BOTH tickets + * must consume the writer lane and then the holder lane in the same + * episode's candidate loop and drain the tag to baseline. + */ +UT_TEST(test_local_independent_dual_terminal_single_watermark_covers_both_lanes) +{ + TestLocalIndependentDualTerminal fixture; + PcmXRetirePayload retire; + PcmXSlotRef found; + const uint64 master_session = UINT64_C(1821); + + prepare_local_independent_dual_terminal(7122, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, - 1, master_session), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_BLOCKER].used, 0); - /* The exact blocker ACK tombstone is durable evidence, but it cannot - * reopen the writer round before REVOKE/IMAGE_READY/DRAIN completes. */ - UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&identity.tag, identity.cluster_epoch, - &cutoff, &next_leader), + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &fixture.writer_ref.identity.tag, &found), + PCM_X_DIRECTORY_NOT_FOUND); + assert_local_queue_baseline(ClusterPcmXConvertShmem); +} + +/* + * RED (2026-07-20 adversarial review P0-12, leg 1): a next-round follower + * cancelled BEFORE the writer's watermark leaves an unlinked CANCELLED + * membership on the tag. Its own detach must release exactly that + * membership while both terminals, the barrier and the master binding stay + * byte-exact -- before the fix the detach answered NOT_READY under the + * writer terminal and the eligibility mismatch (members>1, empty FIFO) + * refused the writer lane forever. + */ +UT_TEST(test_local_independent_dual_terminal_cancel_before_writer_watermark_converges) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalHandle follower; + PcmXLocalTagSlot *tag_slot; + PcmXRetirePayload retire; + uint32 flags; + const uint64 master_session = UINT64_C(1825); + + prepare_local_independent_dual_terminal(7124, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + join_independent_dual_follower(&fixture, 5, master_session + UINT64_C(10), &follower); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(tag_slot->membership_count, 2); + UT_ASSERT_EQ(tag_slot->head_index, PCM_X_INVALID_SLOT_INDEX); + + /* Writer lane refuses retryably while the cancelled membership lingers. */ + retire_watermark(fixture.writer_ref.identity.cluster_epoch, master_session, + fixture.writer_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_NOT_READY); UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + + /* The cancelled member's own detach releases only its membership. */ + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower), PCM_X_QUEUE_OK); + flags = test_slot_flags(&tag_slot->slot); + UT_ASSERT_EQ(flags, PCM_X_LOCAL_TAG_F_REVOKE_BARRIER | PCM_X_LOCAL_TAG_F_TERMINAL_MASK + | PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + UT_ASSERT_EQ(tag_slot->membership_count, 1); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 1); + UT_ASSERT_EQ(tag_slot->ref.handle.ticket_id, fixture.writer_ref.handle.ticket_id); + UT_ASSERT_EQ(tag_slot->holder_ref.handle.ticket_id, fixture.holder_ref.handle.ticket_id); + + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_progress_exact(&identity.tag, &progress), + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT(ticket_refs_equal(&progress.ref, &revoke.ref)); - UT_ASSERT_EQ(progress.image.image_id, revoke.image_id); - UT_ASSERT_EQ(progress.master_session_incarnation, master_session); - UT_ASSERT_EQ(progress.master_node, 1); - UT_ASSERT_EQ(progress.pending_opcode, 0); - UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_REVOKE); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + assert_local_queue_baseline(ClusterPcmXConvertShmem); +} - memset(&ready, 0, sizeof(ready)); - ready.ref = revoke.ref; - ready.image.image_id = revoke.image_id; - ready.image.source_own_generation = 0; - ready.image.page_scn = UINT64_C(52); - ready.image.page_lsn = UINT64_C(53); - ready.image.source_node = 0; - ready.image.page_checksum = UINT32_C(54); - UT_ASSERT_EQ( - cluster_pcm_x_local_holder_image_ready_arm_exact(&ready, 1, master_session, &replay), - PCM_X_QUEUE_OK); - UT_ASSERT(memcmp(&ready, &replay, sizeof(ready)) == 0); - memset(&replay, 0, sizeof(replay)); - UT_ASSERT_EQ( - cluster_pcm_x_local_holder_image_ready_arm_exact(&ready, 1, master_session, &replay), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(memcmp(&ready, &replay, sizeof(ready)) == 0); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_progress_exact(&identity.tag, &progress), +/* + * RED (P0-12, leg 2): the follower cancelled BETWEEN the two watermarks. + * Before the fix the holder RETIRE's frozen-round close saw membership=1 + * with an empty FIFO and fused the runtime; it must instead wait retryably + * for the cancelled member's own detach, byte-exact, then close. + */ +UT_TEST(test_local_independent_dual_terminal_cancel_between_watermarks_converges) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalHandle follower; + PcmXLocalTagSlot *tag_slot; + PcmXRetirePayload retire; + uint64 holder_drain_before; + const uint64 master_session = UINT64_C(1827); + + prepare_local_independent_dual_terminal(7125, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + retire_watermark(fixture.writer_ref.identity.cluster_epoch, master_session, + fixture.writer_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(progress.pending_opcode, PGRAC_IC_MSG_PCM_X_IMAGE_READY); - UT_ASSERT_EQ(progress.image.source_own_generation, 0); - UT_ASSERT_EQ(progress.flags, 0); - UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&identity.tag, identity.cluster_epoch, - &cutoff, &next_leader), + join_independent_dual_follower(&fixture, 5, master_session + UINT64_C(10), &follower); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_OK); + holder_drain_before = tag_slot->holder_terminal_drain_generation; + + /* Holder RETIRE waits retryably; the holder lane stays byte-exact. */ + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, + PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(tag_slot->holder_terminal_drain_generation, holder_drain_before); + UT_ASSERT_EQ(tag_slot->holder_ref.handle.ticket_id, fixture.holder_ref.handle.ticket_id); - memset(&poll, 0, sizeof(poll)); - poll.ref = revoke.ref; - poll.drain_generation = UINT64_C(42); - UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session + 1), - PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&late.identity, &next_leader), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(next_leader.role, PCM_X_LOCAL_ROLE_NODE_LEADER); - UT_ASSERT(slot_refs_equal(next_leader.membership_slot, late.membership_slot)); - /* Promotion is visible for exact lookup, but holder cleanup is not complete - * until RETIRE consumes the terminal image lane. No timed wake or unrelated - * latch may let this successor claim, refreeze, or mint ENQUEUE early. */ - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&next_leader, &writer_claim), - PCM_X_QUEUE_BUSY); - UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&next_leader, &promoted_cutoff), - PCM_X_QUEUE_BUSY); - UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&next_leader, &enqueue, &token), - PCM_X_QUEUE_BUSY); - new_holder_key = make_local_holder_key(708, 0, 3, UINT64_C(7032), 3); - UT_ASSERT_EQ(register_active_local_holder(&new_holder_key, &new_holder), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_progress_exact(&identity.tag, &progress), + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(progress.pending_opcode, 0); - UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_DRAIN_POLL); - UT_ASSERT_EQ(progress.image.source_own_generation, 0); - UT_ASSERT_EQ(progress.flags, PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + assert_local_queue_baseline(ClusterPcmXConvertShmem); +} - memset(&retire, 0, sizeof(retire)); - retire.cluster_epoch = identity.cluster_epoch; - retire.master_session_incarnation = master_session; - retire.retire_through_ticket_id = revoke.ref.handle.ticket_id; - retire.sender_node = 0; +/* + * RED (2026-07-20 review P1): the frozen-round close's follower promotion + * must travel through the RETIRE wake batch -- capacity 0 refuses the whole + * episode with NO_CAPACITY and zero mutation, capacity 1 delivers exactly + * the promoted identity. Without it the promotion is only discovered by + * the waiter's poll interval (a 2-32ms hot-block regression). + */ +UT_TEST(test_local_independent_dual_terminal_collect_reports_promotion_wake) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalHandle follower; + PcmXLocalHandle refreshed; + PcmXLocalTagSlot saved; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity wake_items[2]; + PcmXLocalWakeBatch wake_batch; + PcmXRetirePayload retire; + const uint64 master_session = UINT64_C(1829); + + prepare_local_independent_dual_terminal(7126, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + retire_watermark(fixture.writer_ref.identity.cluster_epoch, master_session, + fixture.writer_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + join_independent_dual_follower(&fixture, 5, master_session + UINT64_C(10), &follower); + + /* Zero capacity refuses before any mutation. */ + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + wake_batch.items = NULL; + wake_batch.capacity = 0; + wake_batch.count = 0; + saved = *tag_slot; + UT_ASSERT_EQ( + cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, master_session, &wake_batch), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT(memcmp(&saved, tag_slot, sizeof(saved)) == 0); + + /* Capacity one delivers exactly the promoted follower. */ memset(wake_items, 0, sizeof(wake_items)); wake_batch.items = wake_items; - wake_batch.capacity = lengthof(wake_items); + wake_batch.capacity = 1; wake_batch.count = 0; UT_ASSERT_EQ( cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, master_session, &wake_batch), PCM_X_QUEUE_OK); UT_ASSERT_EQ(wake_batch.count, 1); - UT_ASSERT(memcmp(&wake_items[0], &next_leader.identity, sizeof(wake_items[0])) == 0); - /* A visible wake batch proves the full watermark and global admission gate - * committed before the caller can make the successor runnable. */ - UT_ASSERT_EQ(pg_atomic_read_u32(&header->local_retire_gate), 0); - UT_ASSERT_EQ(header->peer_frontiers[1].local_retired_ticket_id, - retire.retire_through_ticket_id); - wake_batch.count = 1; + UT_ASSERT(memcmp(&wake_items[0], &follower.identity, sizeof(wake_items[0])) == 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&follower.identity, &refreshed), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(refreshed.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +/* + * RED (review P1, single-watermark form): one watermark covering both lanes + * with a pre-joined follower must deliver the same promotion wake through + * the batch it validated in the writer arm's projected rehearsal. + */ +UT_TEST(test_local_independent_dual_terminal_single_watermark_reports_promotion_wake) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalHandle follower; + PcmXLocalHandle refreshed; + PcmXLocalTagSlot saved; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity wake_items[2]; + PcmXLocalWakeBatch wake_batch; + PcmXRetirePayload retire; + const uint64 master_session = UINT64_C(1831); + + prepare_local_independent_dual_terminal(7127, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + join_independent_dual_follower(&fixture, 5, master_session + UINT64_C(10), &follower); + + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + wake_batch.items = NULL; + wake_batch.capacity = 0; + wake_batch.count = 0; + saved = *tag_slot; UT_ASSERT_EQ( cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, master_session, &wake_batch), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT_EQ(wake_batch.count, 0); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_progress_exact(&identity.tag, &progress), - PCM_X_QUEUE_NOT_FOUND); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&next_leader, &writer_claim), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT(memcmp(&saved, tag_slot, sizeof(saved)) == 0); + + memset(wake_items, 0, sizeof(wake_items)); + wake_batch.items = wake_items; + wake_batch.capacity = 1; + wake_batch.count = 0; + UT_ASSERT_EQ( + cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, master_session, &wake_batch), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(wake_batch.count, 1); + UT_ASSERT(memcmp(&wake_items[0], &follower.identity, sizeof(wake_items[0])) == 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&follower.identity, &refreshed), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(refreshed.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +/* + * RED (leg: non-narrow refusal is byte-exact): breaking one eligibility + * invariant must leave the whole tag slot untouched under a retryable + * NOT_READY -- never a mutation, never a fuse -- and the retire must + * succeed once the invariant is restored. + */ +UT_TEST(test_local_independent_dual_terminal_non_narrow_refusal_is_byte_exact) +{ + TestLocalIndependentDualTerminal fixture; + PcmXLocalTagSlot saved; + PcmXLocalTagSlot *tag_slot; + PcmXRetirePayload retire; + uint64 cutoff_before; + const uint64 master_session = UINT64_C(1823); + + prepare_local_independent_dual_terminal(7123, master_session, UINT64_C(96001), UINT64_C(96002), + false, &fixture); + tag_slot = fixture.tag_slot; + cutoff_before = tag_slot->cutoff_sequence; + tag_slot->cutoff_sequence = tag_slot->next_sequence; + saved = *tag_slot; + retire_watermark(fixture.writer_ref.identity.cluster_epoch, master_session, + fixture.writer_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT(memcmp(&saved, tag_slot, sizeof(saved)) == 0); + + tag_slot->cutoff_sequence = cutoff_before; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_abort_exact(&writer_claim), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(release_active_local_holder(&new_holder), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&next_leader, NULL), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&next_leader), PCM_X_QUEUE_OK); - assert_local_queue_baseline(header); + retire_watermark(fixture.holder_ref.identity.cluster_epoch, master_session, + fixture.holder_ref.handle.ticket_id, &retire); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + assert_local_queue_baseline(ClusterPcmXConvertShmem); } -UT_TEST(test_local_non_source_blocker_participant_drains_and_retires_exactly) +typedef struct TestLocalRebaseFixture { + PcmXLocalHandle writer; + PcmXLocalWriterClaim writer_claim; + PcmXAdmitAckPayload admit_ack; + PcmXGrantPayload prepare; + PcmXLocalTagSlot *tag_slot; + uint64 master_session; +} TestLocalRebaseFixture; + +/* Drive a local leader with a nonzero enqueue-time base (=5) through the + * production chain to the exact PREPARE_GRANT-applied, pre-INSTALL window + * where an interleaved revoke would have moved the own generation. */ +static void prepare_local_rebase_fixture(BlockNumber block, uint64 master_session, + TestLocalRebaseFixture *fixture); + +static void +prepare_local_rebase_fixture_with_follower(BlockNumber block, uint64 master_session, + TestLocalRebaseFixture *fixture, + PcmXLocalHandle *same_round_follower_out) { - PcmXShmemHeader *header; - PcmXLocalHolderKey holder_keys[2]; - PcmXLocalHolderKey new_holder_key; - PcmXLocalHolderHandle holders[2]; - PcmXLocalHolderHandle new_holder; - PcmXLocalHolderHandle holder_copies[2]; + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle holder_copy; PcmXLocalHolderSnapshot holder_snapshot; PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXWaitIdentity identity; + PcmXEnqueuePayload enqueue; + PcmXPhasePayload admit_confirm; PcmXLocalCutoff cutoff; - PcmXLocalTagSlot *tag_slot; - PcmXTicketRef probe_ref; - PcmXDrainPollPayload poll; - PcmXDrainPollPayload stale_poll; - PcmXRetirePayload retire; - PcmXSlotRef found; - const uint64 master_session = UINT64_C(1801); - int i; + PcmXLocalReliableToken token; - /* Model the local evidence left on a non-source participant: two independent - * S pins are captured by PROBE, both are later released, and no holder_ref - * image-source lane is installed. DRAIN must therefore close from the exact - * type-48 blocker ACK tombstone alone. */ + UT_ASSERT_NOT_NULL(fixture); + memset(fixture, 0, sizeof(*fixture)); + fixture->master_session = master_session; init_active_pcm_x(UINT64_C(77)); - header = ClusterPcmXConvertShmem; bind_local_master(1, UINT64_C(9), master_session); - for (i = 0; i < 2; i++) { - holder_keys[i] - = make_local_holder_key(7091, 0, (uint32)(21 + i), UINT64_C(71001) + i, 4 + i); - UT_ASSERT_EQ(register_active_local_holder(&holder_keys[i], &holders[i]), PCM_X_QUEUE_OK); + holder_key = make_local_holder_key(block, 0, 2, master_session + UINT64_C(1), 3); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + identity = make_wait_identity(block, 0, 3, master_session + UINT64_C(2)); + identity.base_own_generation = UINT64_C(5); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &fixture->writer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&fixture->writer, &fixture->writer_claim), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(fixture->writer_claim.grant_base_own_generation, 0); + if (same_round_follower_out != NULL) { + /* Joined before the cutoff freezes the round: a same-round FIFO + * follower with its own enqueue-time identity base. */ + PcmXWaitIdentity follower_identity + = make_wait_identity(block, 0, 4, master_session + UINT64_C(3)); + + follower_identity.base_own_generation = UINT64_C(5); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&follower_identity, 1, master_session, + same_round_follower_out), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(same_round_follower_out->role, PCM_X_LOCAL_ROLE_FOLLOWER); } - memset(&probe_ref, 0, sizeof(probe_ref)); - probe_ref.identity = make_wait_identity(7091, 3, 23, UINT64_C(71003)); - probe_ref.handle.ticket_id = UINT64_C(91001); - probe_ref.handle.queue_generation = UINT64_C(11); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&fixture->writer, &cutoff), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&fixture->writer, &enqueue, &token), + PCM_X_QUEUE_OK); + memset(&fixture->admit_ack, 0, sizeof(fixture->admit_ack)); + fixture->admit_ack.ref.identity = identity; + fixture->admit_ack.ref.handle.ticket_id = master_session + UINT64_C(100); + fixture->admit_ack.ref.handle.queue_generation = UINT64_C(1); + fixture->admit_ack.prehandle = enqueue.prehandle; + fixture->admit_ack.result = PCM_X_QUEUE_OK; + fixture->admit_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&fixture->writer, &fixture->admit_ack, 1, + master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_admit_confirm_arm_exact(&fixture->writer, &admit_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&fixture->writer, &admit_confirm, 1, + master_session), + PCM_X_QUEUE_OK); UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact( - &probe_ref, 1, master_session, holder_copies, 2, &holder_snapshot), + &fixture->admit_ack.ref, 1, master_session, &holder_copy, 1, &holder_snapshot), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(holder_snapshot.holder_count, 2); - UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, - &holder_snapshot, holder_copies, 2, - NULL, 0, &blocker_snapshot), + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact( + &fixture->admit_ack.ref, 1, master_session, &holder_snapshot, &holder_copy, 1, + NULL, 0, &blocker_snapshot), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, - 1, master_session), + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact( + &fixture->admit_ack.ref, blocker_snapshot.set_generation, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact(&probe_ref.identity.tag, - probe_ref.identity.cluster_epoch, - 1, master_session, &cutoff), + UT_ASSERT_EQ(release_active_local_holder(&holder), PCM_X_QUEUE_OK); + + memset(&fixture->prepare, 0, sizeof(fixture->prepare)); + fixture->prepare.ref = fixture->admit_ack.ref; + fixture->prepare.ref.grant_generation = UINT64_C(1); + UT_ASSERT(cluster_pcm_x_image_id_encode(1, master_session + UINT64_C(200), + &fixture->prepare.image.image_id)); + fixture->prepare.image.source_own_generation = UINT64_C(9); + fixture->prepare.image.page_scn = UINT64_C(10); + fixture->prepare.image.page_lsn = UINT64_C(11); + fixture->prepare.image.source_node = 2; + fixture->prepare.image.page_checksum = UINT32_C(12); + UT_ASSERT_EQ(cluster_pcm_x_local_prepare_grant_exact(&fixture->writer, &fixture->prepare, 1, + master_session), PCM_X_QUEUE_OK); - for (i = 0; i < 2; i++) - UT_ASSERT_EQ(release_active_local_holder(&holders[i]), PCM_X_QUEUE_OK); + fixture->tag_slot + = &local_tag_slots(ClusterPcmXConvertShmem)[fixture->writer.tag_slot.slot_index]; + UT_ASSERT_EQ(fixture->tag_slot->grant_base_own_generation, 0); +} - tag_slot = &local_tag_slots(header)[holder_snapshot.tag_slot.slot_index]; - UT_ASSERT(ticket_refs_equal(&tag_slot->holder_ref, &(PcmXTicketRef){ 0 })); - UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &probe_ref)); - UT_ASSERT_EQ(tag_slot->blocker_snapshot_reliable.last_response_opcode, - PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK); +static void +prepare_local_rebase_fixture(BlockNumber block, uint64 master_session, + TestLocalRebaseFixture *fixture) +{ + prepare_local_rebase_fixture_with_follower(block, master_session, fixture, NULL); +} - memset(&poll, 0, sizeof(poll)); - poll.ref = probe_ref; - poll.ref.grant_generation = UINT64_C(101); - poll.drain_generation = UINT64_C(42); - UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), - PCM_X_QUEUE_DUPLICATE); - UT_ASSERT(ticket_refs_equal(&tag_slot->holder_ref, &(PcmXTicketRef){ 0 })); - UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &poll.ref)); - UT_ASSERT_EQ(tag_slot->blocker_set_generation, 0); - UT_ASSERT_EQ(tag_slot->holder_terminal_drain_generation, poll.drain_generation); - UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, - PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); +/* + * A' rebase, local plane: the one-shot pre-reservation publication of the + * effective grant base and the exact final-ack math that follows it. The + * wire V2 field is emitted from the persisted value, never from a caller + * argument, and the enqueue-time math is refused once a rebase is live. + */ +UT_TEST(test_local_grant_rebase_publish_is_one_shot_and_effective) +{ + TestLocalRebaseFixture fixture; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + PcmXLocalReliableToken token; + PcmXDrainPollPayload poll; + PcmXRetirePayload retire; + PcmXLocalProgress progress; + const uint64 master_session = UINT64_C(1811); - stale_poll = poll; - stale_poll.ref.grant_generation++; - UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&stale_poll, 1, master_session), - PCM_X_QUEUE_STALE); - stale_poll = poll; - stale_poll.drain_generation++; - UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&stale_poll, 1, master_session), + prepare_local_rebase_fixture(7120, master_session, &fixture); + + /* Shape and strictly-newer gates. */ + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, 0), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_MAX), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(4)), PCM_X_QUEUE_STALE); - UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session + 1), + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(5)), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(fixture.tag_slot->grant_base_own_generation, 0); - new_holder_key = make_local_holder_key(7091, 0, 24, UINT64_C(71004), 6); - UT_ASSERT_EQ(register_active_local_holder(&new_holder_key, &new_holder), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + /* One-shot publish of a >1 drift (5 -> 8), then a same-value replay. */ + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(8)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(fixture.tag_slot->grant_base_own_generation, UINT64_C(8)); + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(8)), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&fixture.writer, &progress), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.grant_base_own_generation, UINT64_C(8)); + + /* The armed INSTALL_READY emits the persisted value on the V2 field. */ + UT_ASSERT_EQ(cluster_pcm_x_local_install_ready_arm_exact(&fixture.writer, &fixture.prepare.ref, + &fixture.prepare.image, &install_ready, + &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(install_ready.rebased_own_generation, UINT64_C(8)); + /* Once the leg is armed the publish window is closed. */ + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(8)), + PCM_X_QUEUE_BAD_STATE); + + memset(&commit, 0, sizeof(commit)); + commit.ref = fixture.prepare.ref; + commit.phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; + UT_ASSERT_EQ(cluster_pcm_x_local_commit_x_exact(&fixture.writer, &commit, 1, master_session), + PCM_X_QUEUE_OK); + + /* The enqueue-time math (5+1) is refused; the rebased math (8+1) arms. */ + UT_ASSERT_EQ( + cluster_pcm_x_local_final_ack_arm_exact(&fixture.writer, UINT64_C(6), &final_ack, &token), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ( + cluster_pcm_x_local_final_ack_arm_exact(&fixture.writer, UINT64_C(9), &final_ack, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(final_ack.committed_own_generation, UINT64_C(9)); + memset(&final_commit, 0, sizeof(final_commit)); + final_commit.ref = fixture.prepare.ref; + final_commit.phase = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; + UT_ASSERT_EQ(cluster_pcm_x_local_final_commit_ack_exact(&fixture.writer, &final_commit, 1, + master_session, &final_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&fixture.writer_claim), + PCM_X_QUEUE_OK); + memset(&poll, 0, sizeof(poll)); + poll.ref = fixture.prepare.ref; + poll.drain_generation = UINT64_C(43); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); memset(&retire, 0, sizeof(retire)); - retire.cluster_epoch = probe_ref.identity.cluster_epoch; + retire.cluster_epoch = fixture.prepare.ref.identity.cluster_epoch; retire.master_session_incarnation = master_session; - retire.retire_through_ticket_id = poll.ref.handle.ticket_id; + retire.retire_through_ticket_id = fixture.prepare.ref.handle.ticket_id; retire.sender_node = 0; UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &probe_ref.identity.tag, &found), - PCM_X_DIRECTORY_OK); - UT_ASSERT_EQ(release_active_local_holder(&new_holder), PCM_X_QUEUE_OK); - UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &probe_ref.identity.tag, &found), - PCM_X_DIRECTORY_NOT_FOUND); - assert_local_queue_baseline(header); + assert_local_queue_baseline(ClusterPcmXConvertShmem); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +/* A second, different local rebase value is evidence divergence. */ +UT_TEST(test_local_grant_rebase_conflict_fails_closed) +{ + TestLocalRebaseFixture fixture; + const uint64 master_session = UINT64_C(1813); + + prepare_local_rebase_fixture(7121, master_session, &fixture); + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(8)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(9)), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(fixture.tag_slot->grant_base_own_generation, UINT64_C(8)); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +/* A' rebase, same-node FIFO: a follower claiming after the leader's one-shot + * publish must inherit the published effective grant base from the tag slot; + * with its enqueue-time identity base it would verify the new X against the + * stale base+1 and fail closed (t/400 L3 review P0-1). */ +UT_TEST(test_local_follower_claim_inherits_published_rebase) +{ + TestLocalRebaseFixture fixture; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + PcmXLocalReliableToken token; + PcmXLocalHandle follower; + PcmXLocalWriterClaim follower_claim; + ClusterPcmOwnSnapshot granted; + const uint64 master_session = UINT64_C(1815); + + prepare_local_rebase_fixture_with_follower(7122, master_session, &fixture, &follower); + UT_ASSERT_EQ(cluster_pcm_x_local_grant_rebase_publish_exact(&fixture.writer, UINT64_C(8)), + PCM_X_QUEUE_OK); + + /* Leader finishes its grant on the rebased math and releases the claim. */ + UT_ASSERT_EQ(cluster_pcm_x_local_install_ready_arm_exact(&fixture.writer, &fixture.prepare.ref, + &fixture.prepare.image, &install_ready, + &token), + PCM_X_QUEUE_OK); + memset(&commit, 0, sizeof(commit)); + commit.ref = fixture.prepare.ref; + commit.phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; + UT_ASSERT_EQ(cluster_pcm_x_local_commit_x_exact(&fixture.writer, &commit, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_final_ack_arm_exact(&fixture.writer, UINT64_C(9), &final_ack, &token), + PCM_X_QUEUE_OK); + memset(&final_commit, 0, sizeof(final_commit)); + final_commit.ref = fixture.prepare.ref; + final_commit.phase = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; + UT_ASSERT_EQ(cluster_pcm_x_local_final_commit_ack_exact(&fixture.writer, &final_commit, 1, + master_session, &final_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&fixture.writer_claim), + PCM_X_QUEUE_OK); + + /* The follower's claim carries the effective grant base, and the bufmgr + * grant-snapshot proof accepts exactly rebased+1. */ + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &follower_claim), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(follower_claim.grant_base_own_generation, UINT64_C(8)); + memset(&granted, 0, sizeof(granted)); + granted.tag = follower_claim.writer.identity.tag; + granted.generation = UINT64_C(9); + granted.reservation_token = UINT64_C(77); + granted.flags = 0; + granted.pcm_state = (uint8)PCM_STATE_X; + UT_ASSERT(cluster_pcm_x_writer_grant_snapshot_exact(&follower_claim, &granted, &granted)); } UT_TEST(test_local_cancelled_non_source_participant_gen0_drains_and_retires_exactly) @@ -10793,6 +13093,7 @@ UT_TEST(test_local_blocker_snapshot_replays_exact_holder_set_and_gates_revoke) PcmXBlockerChunkPayload edge; PcmXRevokePayload revoke; PcmXTicketRef probe_ref; + ClusterLmdVertex blocker; const uint64 master_session = UINT64_C(180); init_active_pcm_x(UINT64_C(77)); @@ -10829,8 +13130,6 @@ UT_TEST(test_local_blocker_snapshot_replays_exact_holder_set_and_gates_revoke) revoke.ref = probe_ref; revoke.ref.grant_generation = UINT64_C(101); UT_ASSERT(cluster_pcm_x_image_id_encode(1, UINT64_C(201), &revoke.image_id)); - UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), - PCM_X_QUEUE_NOT_READY); UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, 1, master_session), PCM_X_QUEUE_OK); @@ -10838,10 +13137,80 @@ UT_TEST(test_local_blocker_snapshot_replays_exact_holder_set_and_gates_revoke) UT_ASSERT_EQ( cluster_pcm_x_local_blocker_snapshot_copy_exact(&blocker_snapshot, &header, &edge, 1), PCM_X_QUEUE_NOT_READY); + /* A later nonempty generation never qualifies for the replay exception. */ + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + &holder_copy, 1, &holder_snapshot), + PCM_X_QUEUE_OK); + blocker = make_blocker(holder_key.identity.node_id, holder_key.identity.procno, + holder_key.identity.request_id); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, &holder_copy, 1, + &blocker, 1, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(blocker_snapshot.set_generation, 2); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + PCM_X_QUEUE_OK); +} + +UT_TEST(test_local_first_blocker_ack_busy_then_exact_revoke_consumes_empty_commit) +{ + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalTagSlot *tag_slot; + PcmXRevokePayload revoke; + PcmXTicketRef probe_ref; + uint32 state_flags; + const uint64 master_session = UINT64_C(1810); + + init_active_pcm_x(UINT64_C(77)); + holder_key = make_local_holder_key(7091, 0, 1, UINT64_C(70311), 2); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + bind_local_master(1, holder_key.identity.cluster_epoch, master_session); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&holder_key.identity.tag, &holder_copy, 1, + &holder_snapshot), + PCM_X_QUEUE_OK); + + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(7091, 2, 2, UINT64_C(70312)); + probe_ref.handle.ticket_id = UINT64_C(90111); + probe_ref.handle.queue_generation = UINT64_C(11); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, &holder_copy, 1, + NULL, 0, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(blocker_snapshot.set_generation, 1); + UT_ASSERT_EQ(blocker_snapshot.blocker_count, 0); + + /* The master stages this ACK before the same-tag REVOKE. Model a local + * mutator owning ADMISSION_GATE when the ACK handler runs: the frame was + * delivered in FIFO order, but its application is retryably BUSY. */ + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[holder.tag_slot.slot_index]; + state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); + pg_atomic_write_u32(&tag_slot->slot.state_flags, + state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_BUSY); + pg_atomic_write_u32(&tag_slot->slot.state_flags, state_flags); + + memset(&revoke, 0, sizeof(revoke)); + revoke.ref = probe_ref; + revoke.ref.grant_generation = UINT64_C(101); + UT_ASSERT(cluster_pcm_x_image_id_encode(1, UINT64_C(2011), &revoke.image_id)); UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); } + UT_TEST(test_local_blocker_snapshot_exact_lookup_replays_when_pool_is_full) { PcmXShmemHeader *header; @@ -13688,6 +16057,114 @@ UT_TEST(test_stats_initialize_zero_and_narrow_note_apis_are_exact) } +typedef struct RequesterWaitCapture { + bool revalidate_ok; + int revalidate_calls; + int wait_calls; + uint64 count_seen_at_wait; +} RequesterWaitCapture; + +static bool +requester_wait_revalidate(void *callback_arg) +{ + RequesterWaitCapture *capture = (RequesterWaitCapture *)callback_arg; + + capture->revalidate_calls++; + return capture->revalidate_ok; +} + +static PcmXQueueResult +requester_wait_revalidate_result(void *callback_arg) +{ + RequesterWaitCapture *capture = (RequesterWaitCapture *)callback_arg; + + capture->revalidate_calls++; + return capture->revalidate_ok ? PCM_X_QUEUE_OK : PCM_X_QUEUE_BARRIER_CLOSED; +} + +static void +requester_wait_block(void *callback_arg) +{ + RequesterWaitCapture *capture = (RequesterWaitCapture *)callback_arg; + PcmXStatsSnapshot stats; + + capture->wait_calls++; + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + capture->count_seen_at_wait = stats.wait_count; +} + +UT_TEST(test_queue_wait_count_tracks_only_linearized_requester_waits) +{ + RequesterWaitCapture capture = { 0 }; + PcmXStatsSnapshot stats; + + init_active_pcm_x(UINT64_C(81)); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.wait_count, 0); + + /* Fast-path and polling iterations never enter the wait producer. */ + UT_ASSERT_EQ(capture.revalidate_calls, 0); + UT_ASSERT_EQ(capture.wait_calls, 0); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.wait_count, 0); + + /* A failed pre-sleep authority check cannot bump or invoke the wait. */ + UT_ASSERT(!cluster_pcm_x_requester_wait_once(requester_wait_revalidate, requester_wait_block, + &capture)); + UT_ASSERT_EQ(capture.revalidate_calls, 1); + UT_ASSERT_EQ(capture.wait_calls, 0); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.wait_count, 0); + + /* Each actual wait callback observes its own count already linearized. */ + capture.revalidate_ok = true; + UT_ASSERT(cluster_pcm_x_requester_wait_once(requester_wait_revalidate, requester_wait_block, + &capture)); + UT_ASSERT_EQ(capture.wait_calls, 1); + UT_ASSERT_EQ(capture.count_seen_at_wait, 1); + UT_ASSERT(cluster_pcm_x_requester_wait_once(requester_wait_revalidate, requester_wait_block, + &capture)); + UT_ASSERT(cluster_pcm_x_requester_wait_once(requester_wait_revalidate, requester_wait_block, + &capture)); + UT_ASSERT_EQ(capture.revalidate_calls, 4); + UT_ASSERT_EQ(capture.wait_calls, 3); + UT_ASSERT_EQ(capture.count_seen_at_wait, 3); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.wait_count, 3); +} + + +UT_TEST(test_requester_wait_preserves_late_revoke_barrier_refusal) +{ + RequesterWaitCapture capture = { 0 }; + PcmXStatsSnapshot stats; + + init_active_pcm_x(UINT64_C(82)); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.wait_count, 0); + + /* A type-49 barrier that freezes after initial admission but before the + * next WaitLatch must escape as BARRIER_CLOSED. Sleeping here would keep + * the source holder registered and deadlock IMAGE_READY forever. */ + UT_ASSERT_EQ(cluster_pcm_x_requester_wait_once_result(requester_wait_revalidate_result, + requester_wait_block, &capture), + PCM_X_QUEUE_BARRIER_CLOSED); + UT_ASSERT_EQ(capture.revalidate_calls, 1); + UT_ASSERT_EQ(capture.wait_calls, 0); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.wait_count, 0); + + /* An exact OK proof still linearizes the count before the physical wait. */ + capture.revalidate_ok = true; + UT_ASSERT_EQ(cluster_pcm_x_requester_wait_once_result(requester_wait_revalidate_result, + requester_wait_block, &capture), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(capture.revalidate_calls, 2); + UT_ASSERT_EQ(capture.wait_calls, 1); + UT_ASSERT_EQ(capture.count_seen_at_wait, 1); +} + + UT_TEST(test_stats_follow_exact_master_and_local_success_transitions) { PcmXMasterAdmission admission[2]; @@ -14140,15 +16617,16 @@ UT_TEST(test_local_retire_episode_lock_errors_fail_closed) int main(void) { - UT_PLAN(246); + UT_PLAN(275); UT_RUN(test_image_id_domain_is_canonical_and_bounded); UT_RUN(test_wire_abi_sizes_are_exact); UT_RUN(test_wire_abi_offsets_are_exact); UT_RUN(test_runtime_layout_abi_and_offsets_are_exact); + UT_RUN(test_runtime_rebase_wire_active_is_activation_bound); UT_RUN(test_lwlock_held_limit_is_shared_200); UT_RUN(test_default_capacity_formulas_are_exact); UT_RUN(test_exactly_five_pools_and_bounded_directories); - UT_RUN(test_layout_v12_records_transfer_and_terminal_frontiers); + UT_RUN(test_layout_v13_records_transfer_and_terminal_frontiers); UT_RUN(test_offsets_are_aligned_ordered_and_bounded); UT_RUN(test_membership_wait_and_holder_partitions_do_not_overlap); UT_RUN(test_generation_zero_advances_without_being_a_sentinel); @@ -14259,6 +16737,12 @@ main(void) UT_RUN(test_master_drive_bitmap_replace_rejects_reliable_leg_drift); UT_RUN(test_master_drive_snapshot_fails_closed_on_bitmap_corruption); UT_RUN(test_master_transfer_wire_49_56_is_generation_exact); + UT_RUN(test_master_invalidate_busy_backs_off_exact_revoke_ticket); + UT_RUN(test_master_commit_retry_loses_race_to_terminal_progress_without_fail_closed); + UT_RUN(test_master_periodic_drive_replays_post_handoff_commit_x_exactly); + UT_RUN(test_master_install_ready_rebase_grant_base_is_one_shot); + UT_RUN(test_master_install_ready_rebase_conflict_fails_closed); + UT_RUN(test_master_install_ready_no_drift_keeps_identity_base_math); UT_RUN(test_master_grant_generation_exhaustion_never_wraps); UT_RUN(test_master_image_id_allocator_encodes_node31_and_never_wraps); UT_RUN(test_master_blocker_wire_stage_preserves_old_set_until_commit); @@ -14273,7 +16757,10 @@ main(void) UT_RUN(test_delayed_pretransfer_cancel_fences_active_transfer); UT_RUN(test_recovery_blocked_runtime_cannot_promote_or_begin_transfer); UT_RUN(test_recovery_blocked_runtime_refuses_ack_and_local_mutators); + UT_RUN(test_runtime_fail_closed_records_winning_site_only); + UT_RUN(test_master_debug_iterators_report_live_slots); UT_RUN(test_master_cancel_is_exact_and_unlinks_middle_without_fifo_damage); + UT_RUN(test_master_terminal_work_scan_covers_younger_tickets); UT_RUN(test_master_prehandle_cancel_replays_exactly_and_never_hits_reused_slot); UT_RUN(test_master_prehandle_identity_alias_is_corruption_not_stale_cancel); UT_RUN(test_master_prehandle_cancel_first_publishes_terminal_tombstone_before_late_enqueue); @@ -14308,6 +16795,22 @@ main(void) UT_RUN(test_local_transfer_prepare_commit_and_final_ack_are_exact); UT_RUN(test_local_tag_only_holder_transfer_persists_until_exact_drain); UT_RUN(test_local_non_source_blocker_participant_drains_and_retires_exactly); + UT_RUN(test_local_cross_lane_holder_terminal_retires_under_revoke_barrier); + UT_RUN(test_local_cross_lane_retire_exemption_requires_distinct_ticket); + UT_RUN(test_local_independent_dual_terminal_retires_writer_then_holder); + UT_RUN(test_local_independent_dual_terminal_alias_and_malformed_lanes_fail_closed); + UT_RUN(test_local_independent_dual_terminal_supports_pre_joined_follower); + UT_RUN(test_local_independent_dual_terminal_promotes_follower_joined_between_watermarks); + UT_RUN(test_local_independent_dual_terminal_cancel_before_writer_watermark_converges); + UT_RUN(test_local_independent_dual_terminal_cancel_between_watermarks_converges); + UT_RUN(test_local_independent_dual_terminal_collect_reports_promotion_wake); + UT_RUN(test_local_independent_dual_terminal_single_watermark_reports_promotion_wake); + UT_RUN(test_local_independent_dual_terminal_staged_retire_of_cancelled_external); + UT_RUN(test_local_independent_dual_terminal_single_watermark_covers_both_lanes); + UT_RUN(test_local_independent_dual_terminal_non_narrow_refusal_is_byte_exact); + UT_RUN(test_local_grant_rebase_publish_is_one_shot_and_effective); + UT_RUN(test_local_grant_rebase_conflict_fails_closed); + UT_RUN(test_local_follower_claim_inherits_published_rebase); UT_RUN(test_local_cancelled_non_source_participant_gen0_drains_and_retires_exactly); UT_RUN(test_local_cancelled_participant_gen0_drain_requires_frozen_round); UT_RUN(test_local_holder_drain_validates_frozen_round_before_terminal_publish); @@ -14327,6 +16830,7 @@ main(void) UT_RUN(test_local_retire_requires_exact_holder_and_blocker_evidence); UT_RUN(test_local_blocker_allocator_corruption_retains_new_tag_gate_evidence); UT_RUN(test_local_blocker_snapshot_replays_exact_holder_set_and_gates_revoke); + UT_RUN(test_local_first_blocker_ack_busy_then_exact_revoke_consumes_empty_commit); UT_RUN(test_local_blocker_snapshot_exact_lookup_replays_when_pool_is_full); UT_RUN(test_local_blocker_snapshot_accepts_only_holder_bound_nested_waits); UT_RUN(test_local_duplicate_generation_overlap_is_stale_not_fail_closed); @@ -14379,6 +16883,8 @@ main(void) UT_RUN(test_exec_backend_init_fails_closed_on_layout_mismatch); UT_RUN(test_registration_exposes_one_exact_region); UT_RUN(test_stats_initialize_zero_and_narrow_note_apis_are_exact); + UT_RUN(test_queue_wait_count_tracks_only_linearized_requester_waits); + UT_RUN(test_requester_wait_preserves_late_revoke_barrier_refusal); UT_RUN(test_stats_follow_exact_master_and_local_success_transitions); UT_RUN(test_stats_count_fail_closed_and_exact_activating_reset_once); UT_RUN(test_master_admit_lock_window_error_releases_gate_and_fails_closed); diff --git a/src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c b/src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c index db72c591b2..b1fa6341f1 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c @@ -151,6 +151,7 @@ UT_TEST(test_fetch_holder_authenticates_requester_master_and_record_generation) PcmXLocalHolderProgress holder; GcsBlockRequestPayload request; ClusterICEnvelope env; + PcmXImageFetchRequestRefusal refusal; UT_ASSERT(cluster_pcm_x_image_fetch_build_request(&requester, 1, 3, &request)); memset(&holder, 0, sizeof(holder)); @@ -167,15 +168,23 @@ UT_TEST(test_fetch_holder_authenticates_requester_master_and_record_generation) env.dest_node_id = 3; env.payload_length = sizeof(request); - UT_ASSERT(cluster_pcm_x_image_fetch_request_exact(&env, &request, &holder, 3, 2, 0)); + UT_ASSERT(cluster_pcm_x_image_fetch_request_exact_diagnosed(&env, &request, &holder, 3, 2, 0, + &refusal)); + UT_ASSERT_EQ((int)refusal, (int)PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_NONE); env.source_node_id = 0; - UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact(&env, &request, &holder, 3, 2, 0)); + UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact_diagnosed(&env, &request, &holder, 3, 2, 0, + &refusal)); + UT_ASSERT_EQ((int)refusal, (int)PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_ENVELOPE); env.source_node_id = 1; holder.image.image_id++; - UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact(&env, &request, &holder, 3, 2, 0)); + UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact_diagnosed(&env, &request, &holder, 3, 2, 0, + &refusal)); + UT_ASSERT_EQ((int)refusal, (int)PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_IMAGE); holder.image = requester.image; holder.master_node = 0; - UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact(&env, &request, &holder, 3, 2, 0)); + UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact_diagnosed(&env, &request, &holder, 3, 2, 0, + &refusal)); + UT_ASSERT_EQ((int)refusal, (int)PCM_X_IMAGE_FETCH_REQUEST_REFUSAL_HOLDER_MASTER); } diff --git a/src/test/cluster_unit/test_cluster_sf_dep.c b/src/test/cluster_unit/test_cluster_sf_dep.c index 148ffe4bb0..2a7bf2c0e2 100644 --- a/src/test/cluster_unit/test_cluster_sf_dep.c +++ b/src/test/cluster_unit/test_cluster_sf_dep.c @@ -260,6 +260,77 @@ UT_TEST(test_pcm_x_capability_generation_snapshot_is_exact) UT_ASSERT_EQ(generation, (uint32)0); } +/* review P0-2: the family sample returns required-bit support, the optional + * bit and the record generation from one record read; a reconnect renote is + * visible as a generation move together with its bits. */ +UT_TEST(test_pcm_x_capability_family_sample_is_record_coherent) +{ + ClusterSfPeerCap cap; + uint32 generation = UINT32_MAX; + bool rebase = true; + + memset(&cap, 0, sizeof(cap)); + UT_ASSERT(!cluster_sf_peer_cap_family_sample(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, &rebase, + &generation)); + UT_ASSERT(!rebase); + UT_ASSERT_EQ(generation, (uint32)0); + + /* CONVERT without REBASE: supported, optional bit false. */ + cluster_sf_peer_cap_note(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, 7); + rebase = true; + UT_ASSERT(cluster_sf_peer_cap_family_sample(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, &rebase, + &generation)); + UT_ASSERT(!rebase); + UT_ASSERT_EQ(generation, (uint32)7); + + /* Reconnect renote with both bits: the generation moves with the bits. */ + cluster_sf_peer_cap_note( + &cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, 8); + UT_ASSERT(cluster_sf_peer_cap_family_sample(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, &rebase, + &generation)); + UT_ASSERT(rebase); + UT_ASSERT_EQ(generation, (uint32)8); + + /* REBASE alone never satisfies the family requirement. */ + cluster_sf_peer_cap_note(&cap, PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, 9); + rebase = true; + UT_ASSERT(!cluster_sf_peer_cap_family_sample(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, &rebase, + &generation)); + UT_ASSERT(!rebase); + UT_ASSERT_EQ(generation, (uint32)0); + + /* Disconnect invalidation clears the whole family. */ + cluster_sf_peer_cap_note( + &cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, 10); + UT_ASSERT(cluster_sf_peer_cap_invalidate_gen(&cap, 10)); + rebase = true; + UT_ASSERT(!cluster_sf_peer_cap_family_sample(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + PGRAC_IC_HELLO_CAP_PCM_X_REBASE_V1, &rebase, + &generation)); + UT_ASSERT(!rebase); +} + +UT_TEST(test_pcm_x_source_floor_capability_guard_is_generation_exact) +{ + ClusterSfPeerCap cap; + const uint32 family + = PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 | PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1; + + memset(&cap, 0, sizeof(cap)); + cluster_sf_peer_cap_note(&cap, family, 41); + UT_ASSERT(cluster_sf_peer_cap_generation_matches_exact( + &cap, PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1, 41)); + UT_ASSERT(!cluster_sf_peer_cap_generation_matches_exact( + &cap, PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1, 42)); + cluster_sf_peer_cap_note(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, 42); + UT_ASSERT(!cluster_sf_peer_cap_generation_matches_exact( + &cap, PGRAC_IC_HELLO_CAP_PCM_X_SOURCE_FLOOR_V1, 42)); +} + int main(void) { @@ -274,5 +345,7 @@ main(void) UT_RUN(test_peer_cap_gen_renote_after_reconnect); UT_RUN(test_pcm_x_capability_does_not_alias_idle_horizon); UT_RUN(test_pcm_x_capability_generation_snapshot_is_exact); + UT_RUN(test_pcm_x_capability_family_sample_is_record_coherent); + UT_RUN(test_pcm_x_source_floor_capability_guard_is_generation_exact); UT_DONE(); } diff --git a/src/test/cluster_unit/test_cluster_visibility_fork.c b/src/test/cluster_unit/test_cluster_visibility_fork.c index 42c632310e..a39c7f5399 100644 --- a/src/test/cluster_unit/test_cluster_visibility_fork.c +++ b/src/test/cluster_unit/test_cluster_visibility_fork.c @@ -51,11 +51,13 @@ #include +#include "access/htup_details.h" #include "cluster/cluster_itl_slot.h" #include "cluster/cluster_scn.h" #include "cluster/cluster_tt_slot.h" #include "cluster/cluster_tt_status.h" #include "cluster/cluster_visibility_inject.h" +#include "cluster/cluster_visibility_resolve.h" #include "utils/errcodes.h" #undef printf @@ -74,6 +76,13 @@ UT_DEFINE_GLOBALS(); +#ifndef HEAPAM_SOURCE_PATH +#error "HEAPAM_SOURCE_PATH must identify production heapam.c" +#endif +#ifndef TT_LOCAL_SOURCE_PATH +#error "TT_LOCAL_SOURCE_PATH must identify production cluster_tt_local.c" +#endif + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), @@ -103,6 +112,69 @@ void cluster_visibility_inject_shmem_register(void) {} +static char * +read_source(const char *path) +{ + FILE *fp; + char *source; + long length; + + fp = fopen(path, "rb"); + UT_ASSERT(fp != NULL); + if (fp == NULL) + return NULL; + UT_ASSERT_EQ(fseek(fp, 0, SEEK_END), 0); + length = ftell(fp); + UT_ASSERT(length > 0); + UT_ASSERT_EQ(fseek(fp, 0, SEEK_SET), 0); + source = malloc((size_t)length + 1); + UT_ASSERT(source != NULL); + if (source == NULL) { + fclose(fp); + return NULL; + } + UT_ASSERT_EQ((long)fread(source, 1, (size_t)length, fp), length); + source[length] = '\0'; + fclose(fp); + return source; +} + +static void +assert_data_active_publish(const char *source, const char *start_marker, const char *end_marker, + const char *uba_name) +{ + const char *start = strstr(source, start_marker); + const char *end = start != NULL ? strstr(start + strlen(start_marker), end_marker) : NULL; + char publish_call[160]; + const char *publish; + const char *crit_end = start != NULL ? strstr(start, "END_CRIT_SECTION();") : NULL; + + snprintf(publish_call, sizeof(publish_call), "cluster_tt_local_record_data_active(xid, %s);", + uba_name); + publish = start != NULL ? strstr(start, publish_call) : NULL; + + if (publish != NULL && end != NULL && publish >= end) + publish = NULL; + + UT_ASSERT(start != NULL); + UT_ASSERT(end != NULL); + UT_ASSERT(publish != NULL); + UT_ASSERT(crit_end != NULL); + if (start == NULL || end == NULL || publish == NULL || crit_end == NULL) + return; + + /* The ACTIVE identity is published only after the tuple + ITL stamp is + * WAL-protected, and before the function can release its heap buffer. */ + while (true) { + const char *next = strstr(crit_end + 1, "END_CRIT_SECTION();"); + + if (next == NULL || next >= publish) + break; + crit_end = next; + } + UT_ASSERT(crit_end < publish); +} + /* ===== T1: ClusterUndoTTSlotRef size 32B ===== */ UT_TEST(test_t1_undo_tt_slot_ref_sizeof_32) @@ -244,6 +316,66 @@ UT_TEST(test_t12_status_enum_5_values) UT_ASSERT_EQ((int)CLUSTER_TT_STATUS_CLEANED_OUT, 4); } +/* ===== P0-33: every ordinary data-DML producer must publish the exact + * binding as IN_PROGRESS after its ACTIVE ITL stamp, while it still owns the + * buffer content lock. Lock-only already did this; missing data-DML calls + * made a fresh active remote ref miss the overlay and surface 53R97. ===== */ +UT_TEST(test_p033_data_dml_publishes_active_identity) +{ + char *heap_source = read_source(HEAPAM_SOURCE_PATH); + char *tt_source = read_source(TT_LOCAL_SOURCE_PATH); + + if (heap_source == NULL || tt_source == NULL) { + free(heap_source); + free(tt_source); + return; + } + assert_data_active_publish(heap_source, "\nheap_insert(Relation", + "\nheap_prepare_insert(Relation", "cluster_itl_uba"); + assert_data_active_publish(heap_source, "\nheap_multi_insert(Relation", + "\nsimple_heap_insert(Relation", "cluster_mi_uba"); + assert_data_active_publish(heap_source, "\nheap_delete(Relation", + "\nsimple_heap_delete(Relation", "cluster_itl_uba"); + assert_data_active_publish(heap_source, "\nheap_update(Relation", + "\nsimple_heap_update(Relation", "cluster_itl_uba"); + + /* A real undo-record UBA may live in a different record segment from the + * transaction's canonical TT segment after rollover. The producer must + * therefore remember and publish the exact page-ref alias, and every + * terminal install must converge those aliases to COMMITTED/ABORTED. */ + UT_ASSERT(strstr(tt_source, "cluster_tt_local_record_data_active(TransactionId xid, UBA uba)") + != NULL); + UT_ASSERT(strstr(tt_source, "active_alias_segments") != NULL); + UT_ASSERT(strstr(tt_source, "install_binding_aliases") != NULL); + UT_ASSERT(strstr(tt_source, "install_binding_aliases(binding, status, commit_scn)") != NULL); + + free(heap_source); + free(tt_source); +} + +/* P0-33 safety matrix: a proved remote ACTIVE status is non-terminal and + * follows each consumer's existing truth table. None of the authoritative + * terminal/FROZEN/stale boundaries are widened by the producer fix. */ +UT_TEST(test_p033_active_and_safety_boundary_matrix) +{ + UT_ASSERT_EQ((int)cluster_vis_evidence_route(CLUSTER_VIS_EVIDENCE_REMOTE, false), + (int)CLUSTER_VIS_ROUTE_REMOTE_VERDICT); + UT_ASSERT_EQ((int)cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_IN_PROGRESS), + (int)CVV_INVISIBLE); + UT_ASSERT_EQ((int)cluster_vis_update_xmax_verdict(CLUSTER_TT_STATUS_IN_PROGRESS, false), + (int)CVV_BEING_MODIFIED); + + UT_ASSERT_EQ((int)cluster_vis_xmin_needs_resolution(HEAP_XMIN_FROZEN), 0); + UT_ASSERT_EQ((int)cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_COMMITTED), + (int)CVV_VISIBLE); + UT_ASSERT_EQ((int)cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_ABORTED), + (int)CVV_INVISIBLE); + UT_ASSERT_EQ((int)cluster_vis_evidence_route(CLUSTER_VIS_EVIDENCE_STALE_OR_AMBIGUOUS, false), + (int)CLUSTER_VIS_ROUTE_FAILCLOSED_UNKNOWN); + UT_ASSERT_EQ((int)cluster_vis_update_xmin_verdict(CLUSTER_TT_STATUS_UNKNOWN), + (int)CVV_FAILCLOSED_UNKNOWN); +} + int main(void) @@ -260,5 +392,7 @@ main(void) UT_RUN(test_t10_itl_slot_unallocated_sentinel); UT_RUN(test_t11_no_is_xid_local_origin_in_this_unit); UT_RUN(test_t12_status_enum_5_values); + UT_RUN(test_p033_data_dml_publishes_active_identity); + UT_RUN(test_p033_active_and_safety_boundary_matrix); UT_DONE(); } diff --git a/src/test/cluster_unit/test_cluster_visibility_variants.c b/src/test/cluster_unit/test_cluster_visibility_variants.c index 794fb780d4..1a760d4b2a 100644 --- a/src/test/cluster_unit/test_cluster_visibility_variants.c +++ b/src/test/cluster_unit/test_cluster_visibility_variants.c @@ -25,6 +25,7 @@ */ #include "postgres.h" +#include "access/htup_details.h" #include "cluster/cluster_tt_status.h" #include "cluster/cluster_visibility_resolve.h" @@ -43,6 +44,43 @@ static const ClusterTTStatus all_states[] CLUSTER_TT_STATUS_ABORTED, CLUSTER_TT_STATUS_CLEANED_OUT, CLUSTER_TT_STATUS_SUBCOMMITTED }; +/* + * P0-27: VACUUM freeze is an authoritative xmin-committed proof. A frozen + * tuple may retain its historical raw xmin and ITL index after that page slot + * is legally recycled, so resolving the stale xmin identity is both needless + * and wrong. Ordinary COMMITTED/INVALID hints are deliberately NOT widened: + * only the exact FROZEN bit pair bypasses cluster resolution. + */ +UT_TEST(test_update_xmin_frozen_precheck) +{ + UT_ASSERT_EQ((int)cluster_vis_xmin_needs_resolution(HEAP_XMIN_FROZEN), 0); + UT_ASSERT_EQ((int)cluster_vis_xmin_needs_resolution(HEAP_XMIN_FROZEN | HEAP_XMAX_INVALID), 0); + UT_ASSERT_EQ((int)cluster_vis_xmin_needs_resolution(HEAP_XMIN_COMMITTED), 1); + UT_ASSERT_EQ((int)(cluster_vis_xmin_needs_resolution(HEAP_XMIN_COMMITTED) + && cluster_vis_evidence_route(CLUSTER_VIS_EVIDENCE_STALE_OR_AMBIGUOUS, false) + == CLUSTER_VIS_ROUTE_FAILCLOSED_UNKNOWN), + 1); + UT_ASSERT_EQ((int)cluster_vis_xmin_needs_resolution(HEAP_XMIN_INVALID), 1); + UT_ASSERT_EQ((int)cluster_vis_xmin_needs_resolution(0), 1); +} + + +/* + * P0-28: a TID selected on one node remains a legal statement target while + * PCM-X waits for a newer page image. Once an ITL slot is recycled, neither + * xmin hints nor remote-evidence lookup can prove that no peer still names + * that TID. Until a real cluster-wide horizon exists, shared-storage tuples + * must bypass every local prune verdict, including xmin-invalid/unknown forms. + */ +UT_TEST(test_prune_requires_cluster_wide_horizon) +{ + UT_ASSERT_EQ((int)cluster_vis_prune_must_defer(true, false), 1); + UT_ASSERT_EQ((int)cluster_vis_prune_must_defer(true, true), 0); + UT_ASSERT_EQ((int)cluster_vis_prune_must_defer(false, false), 0); + UT_ASSERT_EQ((int)cluster_vis_prune_must_defer(false, true), 0); +} + + /* ---- OBS-4 Self ---- */ UT_TEST(test_obs4_self_full_table) { @@ -237,6 +275,8 @@ UT_TEST(test_evidence_route_full_table) int main(void) { + UT_RUN(test_update_xmin_frozen_precheck); + UT_RUN(test_prune_requires_cluster_wide_horizon); UT_RUN(test_obs4_self_full_table); UT_RUN(test_obs5_toast_full_table); UT_RUN(test_obs2_update_xmin_full_table);