From 2f9b5da5f13793d35a86162b0b4a280444301f97 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Mon, 27 Jul 2026 12:28:51 +0500 Subject: [PATCH 01/13] spock_apply: advance forwarded origin using pre-created disabled subscription Any node that subscribes with forward_origins='all' receives transactions originally sourced from other peer nodes, forwarded through its immediate provider. Each forwarded transaction's ORIGIN wire message carries the original peer's Spock node OID, not the immediate provider's. A peer's transactions can reach this node forwarded this way well before -- or without -- a direct subscription to that peer ever being created here. If one has been (or later is) created, tracking the forwarded progress on its origin lets it start from the right position whenever it is enabled, instead of a full resync. A replication origin only exists where a subscription entry does, so this requires resolving the peer's node id to the local subscription (if any) that names it as provider. handle_origin() resolves the peer node ID to a matching local subscription, excluding invalid origins and the direct provider, and stores the result in forwarded_local_origin_id for the duration of the transaction. This is resolved fresh for every ORIGIN message rather than cached across transactions: a cached RepOriginId can be silently reassigned to an unrelated origin once its owning subscription is dropped (RepOriginId is a small, actively recycled space -- replorigin_create() always reuses the lowest free id, and replorigin_advance() does no catalog validation against it), and Spock's resolution has no cheap way to revalidate a stale value later the way a simple name lookup would. Resolving within the same transaction that uses it bounds that exposure to a single transaction's replay. Multiple matching subscriptions are rejected as ambiguous. At commit, maybe_advance_forwarded_origin() advances the resolved origin using XactLastCommitEnd as this node's local position, rather than the provider's end_lsn from a different WAL space. The WAL-logged origin becomes the durable resume position. Because the advance is logged separately after the data commit, a crash may replay the latest transaction, preserving at-least-once semantics; delta-apply idempotency remains a separate follow-up. t/015 redesigned to test this path: - n1->n2->n3 cascade with forward_origins='all' - disabled sub_n3_n1 pre-created on n3 before data arrives - verifies origin is advanced beyond 0/0 after n1 inserts propagate - verifies origin is stable after n2->n1 is disabled (gap detection) - verifies sub_enable('sub_n3_n1') starts the apply worker from the advanced position (not 0/0), so rows already received via the cascade are not re-sent, and new inserts arrive exactly once via the direct sub --- src/spock_apply.c | 194 ++++++------- tests/tap/t/015_forward_origin_advance.pl | 333 ++++++++++++++-------- 2 files changed, 316 insertions(+), 211 deletions(-) diff --git a/src/spock_apply.c b/src/spock_apply.c index a05dd86f..c202df76 100644 --- a/src/spock_apply.c +++ b/src/spock_apply.c @@ -102,12 +102,12 @@ static TimeOffset apply_delay = 0; static TimestampTz required_commit_ts = 0; /* - * Cache for forwarded origin lookup. The remote_origin_id (Spock node ID) - * is consistent across the cluster, so we can use it as a cache key to - * avoid repeated slot name generation and origin lookups. + * Local origin for the current forwarded transaction's peer, if any. + * Resolved fresh per ORIGIN message (handle_origin()), not cached -- + * a cached RepOriginId can be silently reassigned to an unrelated + * origin once its subscription is dropped. */ -static RepOriginId cached_forward_remote_id = InvalidRepOriginId; -static RepOriginId cached_forward_local_id = InvalidRepOriginId; +static RepOriginId forwarded_local_origin_id = InvalidRepOriginId; static Oid QueueRelid = InvalidOid; @@ -274,7 +274,9 @@ static void append_feedback_position(XLogRecPtr local_commit_lsn, static void get_feedback_position(XLogRecPtr *recvpos, XLogRecPtr *writepos, XLogRecPtr *flushpos, XLogRecPtr *max_recvpos); static void UpdateWorkerStats(XLogRecPtr last_received, XLogRecPtr last_inserted); -static void maybe_advance_forwarded_origin(XLogRecPtr end_lsn, bool xact_had_exception); +static RepOriginId resolve_forward_peer_origin(RepOriginId remote_origin_id); +static void resolve_forwarded_origin_for_transaction(RepOriginId remote_origin_id); +static void maybe_advance_forwarded_origin(XLogRecPtr local_lsn, bool xact_had_exception); static ApplyReplayEntry *apply_replay_queue_next_entry(void); static bool apply_replay_queue_append_entry(ApplyReplayEntry **entry_p, StringInfo *msg_p); @@ -836,6 +838,7 @@ handle_commit(StringInfo s) XLogRecPtr end_lsn; TimestampTz commit_time; XLogRecPtr remote_insert_lsn; + XLogRecPtr local_commit_lsn = InvalidXLogRecPtr; errcallback_arg.action_name = "COMMIT"; xact_action_counter++; @@ -1026,6 +1029,7 @@ handle_commit(StringInfo s) flushpos = (SPKFlushPosition *) palloc(sizeof(SPKFlushPosition)); flushpos->local_end = XactLastCommitEnd; flushpos->remote_end = end_lsn; + local_commit_lsn = XactLastCommitEnd; dlist_push_tail(&lsn_mapping, &flushpos->node); MemoryContextSwitchTo(MessageContext); @@ -1050,10 +1054,12 @@ handle_commit(StringInfo s) /* * For forwarded transactions, advance the replication origin for the - * original source node. This is done outside the IsTransactionState() - * block because it starts its own transaction. + * original source node. The local LSN passed here must be OUR commit + * position (XactLastCommitEnd), not end_lsn — end_lsn is the provider's + * WAL position, a different WAL space entirely. For an empty forwarded + * transaction there is no local commit, so InvalidXLogRecPtr is passed. */ - maybe_advance_forwarded_origin(end_lsn, xact_had_exception); + maybe_advance_forwarded_origin(local_commit_lsn, xact_had_exception); /* Update the entry in the progress table. */ elog(DEBUG1, "SPOCK %s: updating progress table for node_id %d" \ @@ -1175,6 +1181,39 @@ handle_commit(StringInfo s) pgstat_report_activity(STATE_IDLE, NULL); } +/* + * Resolve a forwarding peer to its local subscription origin. + * + * Forwarded transactions may arrive before, or without, a direct subscription + * to that peer. Return InvalidRepOriginId if none exists. If found, its origin + * records progress for a later enable without requiring a full resync. + * + * Multiple matches are ambiguous -- sub_create() doesn't prevent a second + * subscription to the same peer. This only affects pre-positioning for a + * later sub_enable(), not data apply, so warn and skip rather than erroring + * on every forwarded transaction from that peer. + */ +static RepOriginId +resolve_forward_peer_origin(RepOriginId remote_origin_id) +{ + List *subs; + + subs = get_node_subscriptions((Oid) remote_origin_id, true); + if (subs == NIL) + return InvalidRepOriginId; + + if (list_length(subs) > 1) + { + elog(WARNING, "SPOCK %s: ambiguous forwarded origin: more than one " + "local subscription matches peer node (origin id %u), " + "skipping forwarded-origin advance", + MySubscription->name, remote_origin_id); + return InvalidRepOriginId; + } + + return replorigin_by_name(((SpockSubscription *) linitial(subs))->slot_name, true); +} + /* * Handle ORIGIN message. */ @@ -1211,6 +1250,32 @@ handle_origin(StringInfo s) */ remote_origin_id = spock_read_origin(s, &remote_origin_lsn, &remote_origin_name); replorigin_session_origin = remote_origin_id; + + resolve_forwarded_origin_for_transaction(remote_origin_id); +} + +/* + * Resolve the local subscription origin for remote_origin_id, if one + * exists, for use by maybe_advance_forwarded_origin() at commit time. + * Called for every ORIGIN message. + * + * ORIGIN may also identify the direct provider's own transactions, so + * ignore invalid and direct-provider origins -- there's nothing to + * forward in that case. + */ +static void +resolve_forwarded_origin_for_transaction(RepOriginId remote_origin_id) +{ + forwarded_local_origin_id = InvalidRepOriginId; + + if (remote_origin_id == InvalidRepOriginId || + remote_origin_id == (RepOriginId) MySubscription->origin->id) + return; + + StartTransactionCommand(); + forwarded_local_origin_id = resolve_forward_peer_origin(remote_origin_id); + CommitTransactionCommand(); + MemoryContextSwitchTo(MessageContext); } /* @@ -4867,101 +4932,36 @@ apply_replay_queue_start_replay(void) /* * Advance the replication origin for forwarded transactions. * - * In cascade replication (A -> B -> C with forward_origins='all'), when C - * receives transactions that originated on A (forwarded through B), we track - * C's position relative to A by maintaining a separate replication origin. - * - * This enables seamless switchover: if C later subscribes directly to A, - * the origin will already exist with the correct LSN, so C knows where to - * start receiving from A. + * forwarded_local_origin_id was resolved fresh for this transaction in + * handle_origin(). If no matching subscription exists, no advance is + * needed. remote_origin_lsn is in the peer's WAL space; local_lsn is this + * node's commit position, or InvalidXLogRecPtr for an empty transaction. + * Never use the provider's end_lsn as the local value, as that breaks + * recovery ordering. * - * The origin is named using slot name format (spk___) - * for consistency with direct subscriptions. - * - * We cache the remote_origin_id -> local_origin_id mapping since the Spock - * node ID is stable across the cluster (set by commit f60484e). + * wal_log=true makes the resume position durable. Because the advance is + * logged separately after the data commit, a crash may lose the latest + * advance and cause one transaction to be replayed, preserving at-least-once + * semantics. last_update_wins handles the duplicate; delta-apply idempotency + * is tracked separately. */ static void -maybe_advance_forwarded_origin(XLogRecPtr end_lsn, bool xact_had_exception) +maybe_advance_forwarded_origin(XLogRecPtr local_lsn, bool xact_had_exception) { - RepOriginId forwarded_origin; - - /* - * Only advance for forwarded transactions (origin differs from our direct - * provider) that completed without exceptions. - */ if (xact_had_exception || remote_origin_id == InvalidRepOriginId || - remote_origin_id == MySubscription->origin->id || - remote_origin_name == NULL) + remote_origin_id == (RepOriginId) MySubscription->origin->id || + remote_origin_name == NULL || + forwarded_local_origin_id == InvalidRepOriginId) return; - /* - * Check cache first. The remote_origin_id (Spock node ID) is stable - * for a given source node, so we can reuse the local origin ID. - */ - if (remote_origin_id == cached_forward_remote_id && - cached_forward_local_id != InvalidRepOriginId) - { - forwarded_origin = cached_forward_local_id; - - elog(DEBUG2, "SPOCK %s: advancing forwarded origin (cached, oid %u) " - "remote_lsn %X/%X end_lsn %X/%X", - MySubscription->name, - forwarded_origin, - (uint32) (remote_origin_lsn >> 32), (uint32) remote_origin_lsn, - (uint32) (end_lsn >> 32), (uint32) end_lsn); - } - else - { - /* - * Cache miss - look up or create the origin. Use slot name format - * (spk___) for consistency with direct - * subscriptions. - */ - Relation replorigin_rel; - NameData slot_name; - char *dbname; - - StartTransactionCommand(); - - dbname = get_database_name(MyDatabaseId); - gen_slot_name(&slot_name, dbname, remote_origin_name, - MySubscription->name); - - elog(DEBUG2, "SPOCK %s: advancing forwarded origin '%s' (from node '%s') " - "remote_lsn %X/%X end_lsn %X/%X", - MySubscription->name, - NameStr(slot_name), - remote_origin_name, - (uint32) (remote_origin_lsn >> 32), (uint32) remote_origin_lsn, - (uint32) (end_lsn >> 32), (uint32) end_lsn); - - replorigin_rel = table_open(ReplicationOriginRelationId, RowExclusiveLock); - forwarded_origin = replorigin_by_name(NameStr(slot_name), true); - - if (forwarded_origin == InvalidRepOriginId) - { - forwarded_origin = replorigin_create(NameStr(slot_name)); - elog(DEBUG2, "SPOCK %s: created replication origin '%s' (oid %u) " - "for forwarded transactions from node '%s'", - MySubscription->name, NameStr(slot_name), forwarded_origin, - remote_origin_name); - } - - table_close(replorigin_rel, RowExclusiveLock); - CommitTransactionCommand(); - MemoryContextSwitchTo(MessageContext); - - /* Update cache */ - cached_forward_remote_id = remote_origin_id; - cached_forward_local_id = forwarded_origin; - } + elog(DEBUG2, "SPOCK %s: advancing forwarded origin (oid %u) " + "remote_lsn %X/%X local_lsn %X/%X", + MySubscription->name, + forwarded_local_origin_id, + (uint32) (remote_origin_lsn >> 32), (uint32) remote_origin_lsn, + (uint32) (local_lsn >> 32), (uint32) local_lsn); - /* Advance the origin */ - StartTransactionCommand(); - replorigin_advance(forwarded_origin, remote_origin_lsn, - end_lsn, false, false); - CommitTransactionCommand(); - MemoryContextSwitchTo(MessageContext); + replorigin_advance(forwarded_local_origin_id, remote_origin_lsn, + local_lsn, false, true); } diff --git a/tests/tap/t/015_forward_origin_advance.pl b/tests/tap/t/015_forward_origin_advance.pl index d2ceea04..cdedf8b1 100755 --- a/tests/tap/t/015_forward_origin_advance.pl +++ b/tests/tap/t/015_forward_origin_advance.pl @@ -2,29 +2,40 @@ # ============================================================================= # Test: 015_forward_origin_advance.pl - Verify Forward Origin Tracking # ============================================================================= -# This test verifies that when forward_origins='all' is set, the subscriber -# creates and advances a replication origin for the original source node. +# This test verifies that when forward_origins='all' is set and a disabled +# subscription is pre-created on the new node to a peer, the apply worker +# on the new node correctly advances that pre-created origin as the peer's +# forwarded transactions arrive during catchup. # # Topology: -# A (n1) -> B (n2) -> C (n3) -# forward_origins='all' on both subscriptions +# n2 -> n1 -> n3 +# forward_origins='all' only on n3's subscription to n1 -- that is +# what tells n1 to relay transactions it received from n2, rather +# than only its own # -# Expected behavior: -# - A inserts data -# - B receives from A and forwards to C -# - C should create a replication origin using slot name format: -# spk___ (e.g., "spk_regression_n1_n3") -# - C should advance that origin's LSN as it receives forwarded transactions +# Roles: +# n1 = source — the node n3 actively subscribes to for catchup (enabled, +# forward_origins='all'); forwards n2's changes to n3 +# n2 = peer — existing cluster member whose changes are forwarded +# through n1 to n3 +# n3 = new_node — the node being added to the cluster # -# This test FAILS on 'main' branch (origin not created) -# This test PASSES on 'task/SPOC-228/physical-to-logical-replica' branch +# Expected behavior: +# - Before catchup: n3 pre-creates a disabled subscription to n2 (sub_n3_n2). +# This creates a named replication origin at LSN 0/0. +# - n2 inserts data; n1 receives it (via sub_n1_n2) and forwards it to n3 +# (via the enabled sub_n3_n1, forward_origins='all') +# - n3 advances the pre-created sub_n3_n2 origin LSN as forwarded n2 +# transactions arrive +# - When sub_enable('sub_n3_n2') fires, the apply worker starts from +# the correct position — no duplicates, no gaps # ============================================================================= use strict; use warnings; -use Test::More tests => 22; +use Test::More tests => 34; use lib '.'; -use SpockTest qw(create_cluster destroy_cluster system_or_bail system_maybe command_ok get_test_config scalar_query psql_or_bail); +use SpockTest qw(create_cluster destroy_cluster system_or_bail get_test_config scalar_query psql_or_bail wait_for_sub_status); # ============================================================================= # SETUP: Create 3-node cluster @@ -32,15 +43,13 @@ create_cluster(3, 'Create 3-node cluster'); -my $config = get_test_config(); +my $config = get_test_config(); my $node_ports = $config->{node_ports}; -my $pg_bin = $config->{pg_bin}; -my $dbname = $config->{db_name}; -my $host = $config->{host}; +my $dbname = $config->{db_name}; +my $host = $config->{host}; -# Connection strings -my $conn_a = "host=$host port=$node_ports->[0] dbname=$dbname"; -my $conn_b = "host=$host port=$node_ports->[1] dbname=$dbname"; +my $conn_n1 = "host=$host port=$node_ports->[0] dbname=$dbname"; +my $conn_n2 = "host=$host port=$node_ports->[1] dbname=$dbname"; # ============================================================================= # TEST: Forward Origin Advance @@ -64,116 +73,212 @@ psql_or_bail(3, "SELECT spock.repset_add_table('cascade_set', 'test_origin')"); pass('Added table to replication sets'); -# Create cascade: A -> B -> C -# B subscribes to A with forward_origins='all' -psql_or_bail(2, "SELECT spock.sub_create('sub_a_to_b', '$conn_a', ARRAY['cascade_set'], false, false, ARRAY['all'])"); -pass('Created subscription B->A with forward_origins=all'); - -# C subscribes to B with forward_origins='all' -psql_or_bail(3, "SELECT spock.sub_create('sub_b_to_c', '$conn_b', ARRAY['cascade_set'], false, false, ARRAY['all'])"); -pass('Created subscription C->B with forward_origins=all'); - -# Wait for subscriptions to be ready -system_or_bail 'sleep', '5'; - -# Verify subscriptions are replicating -my $sub_b = scalar_query(2, "SELECT 1 FROM spock.sub_show_status() WHERE subscription_name = 'sub_a_to_b' AND status = 'replicating'"); -is($sub_b, '1', 'Subscription A->B is replicating'); - -my $sub_c = scalar_query(3, "SELECT 1 FROM spock.sub_show_status() WHERE subscription_name = 'sub_b_to_c' AND status = 'replicating'"); -is($sub_c, '1', 'Subscription B->C is replicating'); - -# Insert data on A - this will be forwarded through B to C -psql_or_bail(1, "INSERT INTO test_origin (val) VALUES ('from_node_a')"); -system_or_bail 'sleep', '5'; - -# Verify data reached C -my $count_c = scalar_query(3, "SELECT COUNT(*) FROM test_origin WHERE val = 'from_node_a'"); -is($count_c, '1', 'Data from A reached C via B'); +# n1 subscribes to n2, an ordinary subscription. n2's own commits carry no +# origin (InvalidRepOriginId, since n2 has no upstream peer of its own in +# this test) and are never filtered by forward_origins regardless of its +# setting, so this leg does not need it. +psql_or_bail(1, "SELECT spock.sub_create('sub_n1_n2', '$conn_n2', ARRAY['cascade_set'], false, false)"); +pass('Created subscription n1->n2'); + +# n3 subscribes to n1 (the source) for catchup. forward_origins='all' here +# is what tells n1's output plugin to also include transactions n1 itself +# received from n2 (tagged with n2's origin), rather than only n1's own. +psql_or_bail(3, "SELECT spock.sub_create('sub_n3_n1', '$conn_n1', ARRAY['cascade_set'], false, false, ARRAY['all'])"); +pass('Created subscription n3->n1 with forward_origins=all'); + +# Pre-create a disabled subscription on n3 to peer n2 before catchup begins. +# sub_create(enabled=false) creates a named replication origin on n3 at LSN 0/0 +# without starting an apply worker. track_forward_peer_origin()/ +# maybe_advance_forwarded_origin() advance it as forwarded n2 transactions +# arrive, so that sub_enable('sub_n3_n2') starts from the correct LSN. +psql_or_bail(3, "SELECT spock.sub_create( + subscription_name := 'sub_n3_n2', + provider_dsn := '$conn_n2', + replication_sets := ARRAY['cascade_set'], + synchronize_structure := false, + synchronize_data := false, + enabled := false +)"); +pass('Pre-created disabled subscription n3->n2'); + +# sub_create(enabled=false) creates the catalog entry and replication origin +# but not a slot on the provider. In the real join flow, spock_create_subscriber +# handles slot creation. Create it directly here so sub_enable() can start the +# apply worker. +my $sub_n3_n2_slot = scalar_query(3, + "SELECT spock.spock_gen_slot_name(current_database()::name, 'n2'::name, 'sub_n3_n2'::name)"); +psql_or_bail(2, "SELECT pg_create_logical_replication_slot('$sub_n3_n2_slot', 'spock_output')"); + +ok(wait_for_sub_status(1, 'sub_n1_n2', 'replicating', 30), + 'Subscription n1->n2 is replicating'); +ok(wait_for_sub_status(3, 'sub_n3_n1', 'replicating', 30), + 'Subscription n3->n1 is replicating'); + +my $sub_disabled = scalar_query(3, "SELECT 1 FROM spock.sub_show_status() WHERE subscription_name = 'sub_n3_n2' AND status = 'disabled'"); +is($sub_disabled, '1', 'Subscription sub_n3_n2 is disabled on n3'); + +# Insert data on n2 — will be forwarded through n1 to n3 +psql_or_bail(2, "INSERT INTO test_origin (val) VALUES ('from_n2')"); + +my $count_n3 = 0; +for my $attempt (1..30) { + $count_n3 = scalar_query(3, "SELECT COUNT(*) FROM test_origin WHERE val = 'from_n2'"); + last if defined $count_n3 && $count_n3 >= 1; + sleep 1; +} +is($count_n3, '1', 'Data from n2 reached n3 via n1'); # ============================================================================= -# KEY TEST: Check if C has created a replication origin for node A +# KEY TEST: Check that n3's pre-created origin for n2 has been advanced # ============================================================================= -# On main branch: This origin will NOT exist (test fails) -# On fix branch: This origin WILL exist (test passes) -# -# The origin uses slot name format: spk___ -# e.g., "spk_regression_n1_sub_b_to_c" for forwarded transactions from n1 via sub_b_to_c - -my $expected_origin = scalar_query(3, "SELECT spock.spock_gen_slot_name(current_database()::name, 'n1'::name, 'sub_b_to_c'::name)"); -diag("Expected forwarded origin name on C: $expected_origin"); - -my $origin_exists = scalar_query(3, "SELECT COUNT(*) FROM pg_replication_origin WHERE roname = '$expected_origin'"); -diag("Origin '$expected_origin' exists on C: $origin_exists"); -is($origin_exists, '1', "C has replication origin for forwarded source ($expected_origin)"); - -# Also verify the origin has a valid LSN (not 0/0) -my $origin_lsn = scalar_query(3, "SELECT COALESCE(s.remote_lsn::text, 'NULL') FROM pg_replication_origin o LEFT JOIN pg_replication_origin_status s ON o.roident = s.local_id WHERE o.roname = '$expected_origin'"); -diag("Origin '$expected_origin' LSN on C: $origin_lsn"); -ok($origin_lsn ne '0/0' && $origin_lsn ne 'NULL' && $origin_lsn ne '', "Origin $expected_origin has been advanced (LSN is valid)"); +# sub_create(enabled=false) created the origin at 0/0. +# As forwarded n2 transactions arrive on n3, maybe_advance_forwarded_origin() +# looks up sub_n3_n2 by n2's node OID and advances its named origin. +# When sub_enable('sub_n3_n2') fires, the apply worker reads this origin +# and starts replication from the already-advanced position. + +my $expected_origin = scalar_query(3, + "SELECT spock.spock_gen_slot_name(current_database()::name, 'n2'::name, 'sub_n3_n2'::name)"); +diag("Expected forwarded origin name on n3: $expected_origin"); + +my $origin_exists = scalar_query(3, + "SELECT COUNT(*) FROM pg_replication_origin WHERE roname = '$expected_origin'"); +diag("Origin '$expected_origin' exists on n3: $origin_exists"); +is($origin_exists, '1', "n3 has replication origin for forwarded n2 ($expected_origin)"); + +my $origin_lsn = scalar_query(3, + "SELECT COALESCE(s.remote_lsn::text, 'NULL') + FROM pg_replication_origin o + LEFT JOIN pg_replication_origin_status s ON o.roident = s.local_id + WHERE o.roname = '$expected_origin'"); +diag("Origin '$expected_origin' LSN on n3: $origin_lsn"); +ok($origin_lsn ne '0/0' && $origin_lsn ne 'NULL' && $origin_lsn ne '', + "Origin $expected_origin has been advanced (LSN is valid)"); # ============================================================================= # GAP DETECTION TEST: Demonstrate origin tracking enables gap detection # ============================================================================= -# This test demonstrates that forwarded origin tracking enables detection of -# unreplicated data during cascade switchover scenarios. -# -# With the fix (forwarded origin tracking): -# - We can query C's position relative to A via the forwarded origin LSN -# - We can query A's current position: pg_current_wal_lsn() -# - If A's LSN > C's tracked LSN, there's unreplicated data (a "gap") -# - This enables tooling to make informed switchover decisions -# -# Without the fix (main branch): -# - C has no forwarded origin, so we cannot detect gaps -# - Switchover is "blind" - we don't know what C has received from A diag("=== GAP DETECTION TEST: Origin tracking enables gap detection ==="); -# Insert more data and let it propagate -psql_or_bail(1, "INSERT INTO test_origin (val) VALUES ('batch2_row1')"); -psql_or_bail(1, "INSERT INTO test_origin (val) VALUES ('batch2_row2')"); -system_or_bail 'sleep', '3'; - -# Verify data reached C -my $count_after_batch2 = scalar_query(3, "SELECT COUNT(*) FROM test_origin"); -diag("Row count on C after batch 2: $count_after_batch2"); -is($count_after_batch2, '3', 'C has 3 rows after batch 2'); - -# KEY TEST: Query C's tracked position for forwarded origin (only works with fix) -my $c_origin_lsn = scalar_query(3, "SELECT COALESCE(s.remote_lsn::text, 'not_tracked') FROM pg_replication_origin o LEFT JOIN pg_replication_origin_status s ON o.roident = s.local_id WHERE o.roname = '$expected_origin'"); -diag("C's tracked LSN for A (origin '$expected_origin'): $c_origin_lsn"); +psql_or_bail(2, "INSERT INTO test_origin (val) VALUES ('batch2_row1')"); +psql_or_bail(2, "INSERT INTO test_origin (val) VALUES ('batch2_row2')"); + +my $count_after_batch2 = 0; +for my $attempt (1..30) { + $count_after_batch2 = scalar_query(3, "SELECT COUNT(*) FROM test_origin"); + last if defined $count_after_batch2 && $count_after_batch2 >= 3; + sleep 1; +} +diag("Row count on n3 after batch 2: $count_after_batch2"); +is($count_after_batch2, '3', 'n3 has 3 rows after batch 2'); + +my $c_origin_lsn = scalar_query(3, + "SELECT COALESCE(s.remote_lsn::text, 'not_tracked') + FROM pg_replication_origin o + LEFT JOIN pg_replication_origin_status s ON o.roident = s.local_id + WHERE o.roname = '$expected_origin'"); +diag("n3's tracked LSN for n2 (origin '$expected_origin'): $c_origin_lsn"); -# Query A's current WAL position -my $a_wal_lsn = scalar_query(1, "SELECT pg_current_wal_lsn()::text"); -diag("A's current WAL LSN: $a_wal_lsn"); - -# On fix branch: c_origin_lsn should be a valid LSN (not 'not_tracked') -# On main branch: c_origin_lsn would be 'not_tracked' (query returns nothing) ok($c_origin_lsn ne 'not_tracked' && $c_origin_lsn ne '', - 'C can track its position relative to A (gap detection enabled)'); + 'n3 can track its position relative to n2 (gap detection enabled)'); + +# Simulate gap: disable n1->n2, insert on n2, check origin does not advance +diag("Creating gap: disabling n1->n2 subscription..."); +psql_or_bail(1, "SELECT spock.sub_disable('sub_n1_n2')"); +ok(wait_for_sub_status(1, 'sub_n1_n2', 'disabled', 30), + 'Subscription n1->n2 disabled to create gap'); + +# This is a negative check (proving gap_data does NOT reach n3): there is no +# status transition or count change to poll for, so a bounded sleep is the +# right tool -- it gives propagation every chance to happen before we assert +# that it didn't. +psql_or_bail(2, "INSERT INTO test_origin (val) VALUES ('gap_data')"); +system_or_bail 'sleep', '5'; -# Simulate gap: disable B->A, insert on A, check that we can detect the gap -diag("Creating gap: disabling B->A subscription..."); -psql_or_bail(2, "SELECT spock.sub_disable('sub_a_to_b')"); -system_or_bail 'sleep', '2'; +my $c_origin_after_gap = scalar_query(3, + "SELECT COALESCE(s.remote_lsn::text, 'not_tracked') + FROM pg_replication_origin o + LEFT JOIN pg_replication_origin_status s ON o.roident = s.local_id + WHERE o.roname = '$expected_origin'"); +diag("n3's origin LSN (unchanged, gap detected): $c_origin_after_gap"); -# Insert data on A that creates a gap (can't reach C) -psql_or_bail(1, "INSERT INTO test_origin (val) VALUES ('gap_data')"); -my $a_wal_after_gap = scalar_query(1, "SELECT pg_current_wal_lsn()::text"); -diag("A's WAL LSN after gap data: $a_wal_after_gap"); +is($c_origin_after_gap, $c_origin_lsn, 'Gap detected: n3 origin unchanged while n2 advanced'); -# C's origin LSN should still be at the old position (gap exists) -my $c_origin_after_gap = scalar_query(3, "SELECT COALESCE(s.remote_lsn::text, 'not_tracked') FROM pg_replication_origin o LEFT JOIN pg_replication_origin_status s ON o.roident = s.local_id WHERE o.roname = '$expected_origin'"); -diag("C's origin LSN (unchanged, gap detected): $c_origin_after_gap"); +my $count_with_gap = scalar_query(3, "SELECT COUNT(*) FROM test_origin"); +is($count_with_gap, '3', 'n3 still has 3 rows (gap data not received)'); -# Verify the LSNs show a gap (A advanced, C's tracking hasn't) -# This is the key insight: with origin tracking, we KNOW there's unreplicated data -is($c_origin_after_gap, $c_origin_lsn, 'Gap detected: C origin unchanged while A advanced'); +# ============================================================================= +# GAP RECOVERY TEST: Re-enable n1->n2 and verify gap data arrives on n3 +# ============================================================================= -# Clean test: verify C still has 3 rows (gap data didn't arrive) -my $count_with_gap = scalar_query(3, "SELECT COUNT(*) FROM test_origin"); -is($count_with_gap, '3', 'C still has 3 rows (gap data not received)'); +psql_or_bail(1, "SELECT spock.sub_enable('sub_n1_n2')"); +ok(wait_for_sub_status(1, 'sub_n1_n2', 'replicating', 30), + 'Subscription n1->n2 re-enabled'); + +my $count_after_reenable = 0; +for my $attempt (1..30) { + $count_after_reenable = scalar_query(3, "SELECT COUNT(*) FROM test_origin"); + last if defined $count_after_reenable && $count_after_reenable >= 4; + sleep 1; +} +is($count_after_reenable, '4', 'n3 received gap_data after n1->n2 re-enabled'); + +my $c_origin_after_reenable = scalar_query(3, + "SELECT COALESCE(s.remote_lsn::text, 'not_tracked') + FROM pg_replication_origin o + LEFT JOIN pg_replication_origin_status s ON o.roident = s.local_id + WHERE o.roname = '$expected_origin'"); +diag("n3's origin LSN after re-enable: $c_origin_after_reenable"); +ok($c_origin_after_reenable ne $c_origin_lsn, + 'n3 forwarded origin LSN advanced after gap closed'); + +# ============================================================================= +# SUB_ENABLE TEST: Verify no duplicates when sub_n3_n2 is enabled +# ============================================================================= +# The forwarded origin on n3 for n2 has been advanced during cascade catchup. +# When sub_enable('sub_n3_n2') fires, the apply worker calls +# replorigin_session_get_progress() on the pre-created origin and receives +# the forwarded LSN — so it starts replication from that position, skipping +# rows n3 already received via the n1 cascade. This is the end-to-end proof +# that maybe_advance_forwarded_origin() prevents duplicates. + +# Disable the cascade leg so that new rows on n2 can only reach n3 via the +# direct subscription we are about to enable. Without this, data arriving +# via sub_n3_n1 would mask whether sub_n3_n2 is actually running. +psql_or_bail(3, "SELECT spock.sub_disable('sub_n3_n1')"); +ok(wait_for_sub_status(3, 'sub_n3_n1', 'disabled', 30), + 'Cascade leg sub_n3_n1 fully disabled before enabling direct subscription'); + +psql_or_bail(3, "SELECT spock.sub_enable('sub_n3_n2')"); +ok(wait_for_sub_status(3, 'sub_n3_n2', 'replicating', 30), + 'Enabled direct subscription sub_n3_n2 on n3'); + +# No-duplicates check: if the apply worker started from 0/0 instead of the +# forwarded LSN, it would re-send all 4 rows already on n3, making the count +# jump to 8. A stable count of 4 proves the correct start position was used. +my $count_no_dup = scalar_query(3, "SELECT COUNT(*) FROM test_origin"); +is($count_no_dup, '4', 'No duplicates: existing rows not re-sent after sub_enable'); + +# Insert a new row on n2 and poll for it to arrive on n3. With sub_n3_n1 +# disabled, the only path is sub_n3_n2, so arrival proves that subscription +# is live and replicating. +psql_or_bail(2, "INSERT INTO test_origin (val) VALUES ('via_direct_sub')"); +my $direct_arrived = 0; +for my $attempt (1..60) { + my $cnt = scalar_query(3, + "SELECT COUNT(*) FROM test_origin WHERE val = 'via_direct_sub'"); + if (defined $cnt && $cnt >= 1) { $direct_arrived = 1; last; } + sleep 1; +} +ok($direct_arrived, 'sub_n3_n2 is replicating directly from n2 (cascade disabled)'); + +my $count_after_direct = scalar_query(3, "SELECT COUNT(*) FROM test_origin"); +is($count_after_direct, '5', 'n3 has exactly 5 rows via direct sub_n3_n2'); + +my $direct_row = scalar_query(3, + "SELECT COUNT(*) FROM test_origin WHERE val = 'via_direct_sub'"); +is($direct_row, '1', 'Direct row arrived exactly once on n3'); # ============================================================================= # CLEANUP From 43fdf9e650faadab81511786757a4189750c020d Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Thu, 23 Jul 2026 21:34:32 +0500 Subject: [PATCH 02/13] Reject forwarding/enabled-subscription coexistence A forwarding worker advances the replication origin of any local subscription matching a forwarded peer. If that subscription is enabled concurrently, its apply worker owns the same origin, causing replorigin_advance() to raise ERRCODE_OBJECT_IN_USE. Normal apply error handling then disables the entire forwarding subscription, including any unrelated traffic it carries. Add enforce_forwarding_exclusivity() to reject configurations where an enabled forwarding subscription coexists with another enabled subscription on the same node. Enforce this for sub_create(enabled=true), sub_enable(), and sub_alter_options() when enabling forward_origins, covering both possible state-change orders. --- src/spock_functions.c | 76 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/src/spock_functions.c b/src/spock_functions.c index b79bd8fd..4eeab6f7 100644 --- a/src/spock_functions.c +++ b/src/spock_functions.c @@ -462,6 +462,70 @@ spock_alter_node_drop_interface(PG_FUNCTION_ARGS) PG_RETURN_BOOL(true); } +/* + * Reject changes that enable forwarding alongside another enabled + * subscription on this node. + * + * A forwarding worker advances matching local subscription origins. If that + * subscription is enabled, its apply worker owns the origin and a subsequent + * advance fails with ERRCODE_OBJECT_IN_USE, disabling the forwarder. + * + * Check creates, enables, and non-empty forward_origins changes. The target's + * pending state is passed explicitly because its catalog state may not yet + * reflect the change. The check covers all subscriptions because forwarding + * may relay any origin. + * + * This catalog check cannot prevent worker races or direct catalog changes. + * maybe_advance_forwarded_origin() leaves ownership errors uncaught so they + * remain visible. + */ +static void +enforce_forwarding_exclusivity(Oid target_sub_id, const char *target_sub_name, + bool target_enabled, List *target_forward_origins) +{ + SpockLocalNode *localnode = get_local_node(true, false); + List *subs; + ListCell *lc; + bool target_forwards = target_enabled && list_length(target_forward_origins) > 0; + + if (!target_enabled) + return; /* a disabled target can never conflict with anything */ + + subs = get_node_subscriptions(localnode->node->id, false); + foreach(lc, subs) + { + SpockSubscription *sub = (SpockSubscription *) lfirst(lc); + bool other_forwards; + + if (sub->id == target_sub_id || !sub->enabled) + continue; + + other_forwards = list_length(sub->forward_origins) > 0; + + if (!target_forwards && !other_forwards) + continue; + + if (target_forwards) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot activate forwarding on subscription \"%s\" " + "while subscription \"%s\" is enabled on this node", + target_sub_name, sub->name), + errhint("clear forward_origins on \"%s\", or disable " + "\"%s\" first", + target_sub_name, sub->name))); + else + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot enable subscription \"%s\" while " + "subscription \"%s\" has forwarding active", + target_sub_name, sub->name), + errhint("clear forward_origins on \"%s\" " + "(spock.sub_alter_options) before enabling \"%s\"", + sub->name, target_sub_name))); + } +} + /* * Connect two existing nodes. */ @@ -488,12 +552,17 @@ spock_create_subscription(PG_FUNCTION_ARGS) SpockInterface targetif; List *replication_sets; List *other_subs; + List *new_forward_origins; ListCell *lc; NameData slot_name; /* Check that this is actually a node. */ localnode = get_local_node(true, false); + new_forward_origins = textarray_to_list(forward_origin_names); + if (enabled) + enforce_forwarding_exclusivity(InvalidOid, sub_name, true, new_forward_origins); + /* Now, fetch info about remote node. */ conn = spock_connect(provider_dsn, sub_name, "create"); @@ -608,7 +677,7 @@ spock_create_subscription(PG_FUNCTION_ARGS) sub.origin_if = &originif; sub.target_if = &targetif; sub.replication_sets = replication_sets; - sub.forward_origins = textarray_to_list(forward_origin_names); + sub.forward_origins = new_forward_origins; sub.enabled = enabled; gen_slot_name(&slot_name, get_database_name(MyDatabaseId), origin->name, sub_name); @@ -833,6 +902,8 @@ spock_alter_subscription_enable(PG_FUNCTION_ARGS) /* XXX: Only used for locking purposes. */ (void) get_local_node(true, false); + enforce_forwarding_exclusivity(sub->id, sub->name, true, sub->forward_origins); + sub->enabled = true; alter_subscription(sub); @@ -1030,7 +1101,10 @@ spock_alter_subscription_options(PG_FUNCTION_ARGS) } if (strcmp(key, "forward_origins") == 0) + { + enforce_forwarding_exclusivity(sub->id, sub->name, sub->enabled, result); sub->forward_origins = result; + } else sub->skip_schema = result; changed = true; From 09d0eba247eddf5b442d84032fa80de7488bb2eb Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Mon, 3 Aug 2026 21:56:48 +0500 Subject: [PATCH 03/13] docs: document forward_origins exclusivity restriction --- .../functions/spock_sub_alter_options.md | 3 ++ .../functions/spock_sub_create.md | 3 ++ .../functions/spock_sub_enable.md | 4 ++ docs/spock_functions/sub_mgmt.md | 49 +++++++++++++++---- docs/spock_release_notes.md | 7 +++ 5 files changed, 56 insertions(+), 10 deletions(-) diff --git a/docs/spock_functions/functions/spock_sub_alter_options.md b/docs/spock_functions/functions/spock_sub_alter_options.md index 72dea4f0..e1841cd3 100644 --- a/docs/spock_functions/functions/spock_sub_alter_options.md +++ b/docs/spock_functions/functions/spock_sub_alter_options.md @@ -52,6 +52,9 @@ options element is the string "all". Pass an empty array ([]) to disable origin forwarding. + See [Origin Forwarding](../sub_mgmt.md#origin-forwarding) for the + restriction on combining this with other enabled subscriptions. + apply_delay A PostgreSQL interval string (e.g. "2 seconds", "500ms", "0") diff --git a/docs/spock_functions/functions/spock_sub_create.md b/docs/spock_functions/functions/spock_sub_create.md index 75414be6..3d70b245 100644 --- a/docs/spock_functions/functions/spock_sub_create.md +++ b/docs/spock_functions/functions/spock_sub_create.md @@ -71,6 +71,9 @@ forward_origins replication to avoid forwarding changes in a loop). Use {all} to replicate all changes regardless of origin. The default is `{}` (local-origin changes only). + See [Origin Forwarding](../sub_mgmt.md#origin-forwarding) for the + restriction on combining this with other enabled subscriptions. + apply_delay An interval specifying how long to delay applying changes from the diff --git a/docs/spock_functions/functions/spock_sub_enable.md b/docs/spock_functions/functions/spock_sub_enable.md index f24966c0..2bcbd84f 100644 --- a/docs/spock_functions/functions/spock_sub_enable.md +++ b/docs/spock_functions/functions/spock_sub_enable.md @@ -13,6 +13,10 @@ spock.sub_enable(subscription_name name, immediate boolean) The `spock.sub_enable()` function enables a subscription. +This fails if another subscription on this node already has origin +forwarding active; see +[Origin Forwarding](../sub_mgmt.md#origin-forwarding). + ## Arguments The function accepts the following arguments: diff --git a/docs/spock_functions/sub_mgmt.md b/docs/spock_functions/sub_mgmt.md index 367f1ed2..7d8aa9f9 100644 --- a/docs/spock_functions/sub_mgmt.md +++ b/docs/spock_functions/sub_mgmt.md @@ -45,11 +45,9 @@ Parameters include: `false`. - `synchronize_data` specifies if Spock should synchronize data from provider to the subscriber; the default is `false`. -- `forward_origins` is an array of origin names to forward. Currently, the - only supported values are an empty array meaning don't forward any changes - that didn't originate on provider node (this is useful for two-way - replication between the nodes), or `{all}` which means replicate all changes - no matter what is their origin. The default is `{}` (an empty array, meaning only local changes are forwarded). +- `forward_origins` is an array of origin names to forward. See + [Origin Forwarding](#origin-forwarding) below for supported values and the + restriction on combining this with other enabled subscriptions. - `apply_delay` specifies how long to delay replication; the default is `0` seconds. - set `force_text_transfer` to `true` to force the provider to replicate all @@ -100,6 +98,36 @@ Drops a subscription named `accts`; if the subscription does not exist, an error message will be suppressed by the `true` trailing parameter (`ifexists = true`). +## Origin Forwarding + +`forward_origins` controls whether a subscription's apply worker also +processes transactions that did not originate on the immediate provider, but +were relayed through it from a peer further upstream (a cascade or multi-hop +topology). `{}` (the default) forwards only the provider's own +locally-originated changes — the setting used for ordinary bidirectional +replication between two nodes, since forwarding a peer's changes back to +itself would loop. `{all}` forwards every change regardless of origin. + +A forwarding worker advances the replication origin of the local subscription +matching the peer it's relaying (for example, a subscription to that peer +created disabled ahead of time, so it can later start from the right +position instead of a full resync). At most one local subscription may match +a given peer; if more than one does, replication fails with an ambiguous-match +error. If the matching subscription is also enabled, its own apply worker +owns the same origin, and the two would conflict. Spock avoids this by +requiring that a subscription with forwarding active be the only enabled +subscription on the node: + +- Enabling forwarding (`{all}`) on a subscription — via `spock.sub_create` + with `enabled := true`, `spock.sub_enable`, or `spock.sub_alter_options` + on an already-enabled subscription — fails if another subscription is + already enabled on this node. +- Enabling a subscription (`spock.sub_create` with `enabled := true`, or + `spock.sub_enable`) fails if another subscription on this node already has + forwarding active. +- Changing `forward_origins` on a subscription that stays disabled is always + allowed; the restriction is only checked once the subscription is actually + enabled. ## Subscription Management Functions @@ -127,11 +155,9 @@ Parameters: structure from the provider to the subscriber; the default is `false`. - `synchronize_data` tells Spock to synchronize data from provider to the subscriber; the default is `false`. -- `forward_origins` is an array of origin names to forward. Currently, the - only supported values are an empty array meaning don't forward any changes - that didn't originate on the provider node (this is useful for two-way - replication between the nodes), or `{all}` which means replicate all - changes regardless of their origin. The default is `{}` (an empty array, meaning only local changes are forwarded). +- `forward_origins` is an array of origin names to forward. See + [Origin Forwarding](#origin-forwarding) above for supported values and the + restriction on combining this with other enabled subscriptions. - `apply_delay` is the number of seconds to delay replication; the default is `0` seconds. - `force_text_transfer` forces the provider to replicate all columns using @@ -192,6 +218,9 @@ Parameters: the subscription is started immediately; if set to `false` (the default), it will only be started at the end of the current transaction. +This fails if another subscription on this node already has forwarding +active; see [Origin Forwarding](#origin-forwarding) above. + ### spock.sub_alter_interface Use `spock.sub_alter_interface` to modify the subscription to use a different diff --git a/docs/spock_release_notes.md b/docs/spock_release_notes.md index f9b4ffc1..8a3b9e0d 100644 --- a/docs/spock_release_notes.md +++ b/docs/spock_release_notes.md @@ -168,6 +168,13 @@ included in ORIGIN messages when the protocol version is 5 or higher. This ensures that conflict evaluation on Node C has accurate origin information even when changes pass through intermediate Node B. +A subscription cannot activate `forward_origins` while another subscription +is already enabled on the same node, and a subscription cannot be enabled +while another has forwarding active; clear `forward_origins` +(`spock.sub_alter_options`) or disable the other subscription first. See +[Origin Forwarding](spock_functions/sub_mgmt.md#origin-forwarding) for +details. + ### Per-subscription conflict statistics On PostgreSQL 18+, Spock registers a custom pgstat kind From 36683ecb532640cb4b6609888b5cf47e6f044ab2 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Mon, 22 Jun 2026 14:06:38 +0500 Subject: [PATCH 04/13] =?UTF-8?q?Add=20spock=5Fcreate=5Fsubscriber=20?= =?UTF-8?q?=E2=80=94=20bootstrap=20a=20subscriber=20from=20a=20physical=20?= =?UTF-8?q?basebackup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a standalone frontend utility that: - Takes a 'pg_basebackup' from the provider and replays to a named restore point. - Creates per-database logical slots on the provider (optionally '--drop-slot-if-exists'). - Starts the new node to catch up with the provider to the restore point. - Installs the Spock extension, creates/advances the replication origin to the restore-point LSN, and creates the subscription to subscribe to the provider. --- .gitignore | 1 - Makefile | 1 + utils/spock_create_subscriber/Makefile | 15 + .../spock_create_subscriber.c | 1775 +++++++++++++++++ 4 files changed, 1791 insertions(+), 1 deletion(-) create mode 100644 utils/spock_create_subscriber/Makefile create mode 100644 utils/spock_create_subscriber/spock_create_subscriber.c diff --git a/.gitignore b/.gitignore index 71736e1d..0d9c3d3c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ results regression_output tmp_check -spock_create_subscriber .vimrc *.o *.so diff --git a/Makefile b/Makefile index 000138d7..523d5e3a 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ EXTENSION = spock PGFILEDESC = "spock - multi-master replication" MODULES = spock_output +SUBDIRS = utils/spock_create_subscriber # Lookup source directory vpath % src src/compat/$(PGVER) diff --git a/utils/spock_create_subscriber/Makefile b/utils/spock_create_subscriber/Makefile new file mode 100644 index 00000000..54e39446 --- /dev/null +++ b/utils/spock_create_subscriber/Makefile @@ -0,0 +1,15 @@ +# Makefile for spock_create_subscriber utility +PG_CONFIG ?= pg_config +PROGRAM = spock_create_subscriber + +PG_CPPFLAGS = -I../../include -I$(shell $(PG_CONFIG) --includedir) +PG_LDFLAGS = -lpq -L$(shell $(PG_CONFIG) --libdir) + +# create symlink to spock_fe.c here +spock_fe.c: ../../src/spock_fe.c + ln -sf $< $@ +OBJS = spock_create_subscriber.o spock_fe.o + +# PGXS +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c new file mode 100644 index 00000000..8147c68b --- /dev/null +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -0,0 +1,1775 @@ +/* ------------------------------------------------------------------------- + * + * spock_create_subscriber.c + * Initialize a new spock subscriber from a physical base backup + * + * Copyright (c) 2022-2024, pgEdge, Inc. + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, The Regents of the University of California + * + * ------------------------------------------------------------------------- + */ + +/* dirent.h on port/win32_msvc expects MAX_PATH to be defined */ +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Note the order is important for debian here. */ +#if !defined(pg_attribute_printf) + +/* GCC and XLC support format attributes */ +#if defined(__GNUC__) || defined(__IBMC__) +#define pg_attribute_format_arg(a) __attribute__((format_arg(a))) +#define pg_attribute_printf(f,a) __attribute__((format(PG_PRINTF_ATTRIBUTE, f, a))) +#else +#define pg_attribute_format_arg(a) +#define pg_attribute_printf(f,a) +#endif + +#endif + +#include "libpq-fe.h" +#include "postgres_fe.h" +#include "pqexpbuffer.h" + +#include "getopt_long.h" + +#include "miscadmin.h" + +#include "access/timeline.h" +#include "access/xlog_internal.h" +#include "catalog/pg_control.h" + +#include "spock_fe.h" + +#define MAX_APPLY_DELAY 86400 + +typedef struct RemoteInfo { + Oid nodeid; + char *node_name; + char *sysid; + char *dbname; + char *replication_sets; +} RemoteInfo; + +typedef enum { + VERBOSITY_NORMAL, + VERBOSITY_VERBOSE, + VERBOSITY_DEBUG +} VerbosityLevelEnum; + +static char *argv0 = NULL; +static const char *progname; +static char *data_dir = NULL; +static char pid_file[MAXPGPATH]; +static time_t start_time; +static VerbosityLevelEnum verbosity = VERBOSITY_NORMAL; + +/* defined as static so that die() can close them */ +static PGconn *subscriber_conn = NULL; +static PGconn *provider_conn = NULL; + +static void signal_handler(int sig); +static void usage(void); +static void die(const char *fmt,...) +pg_attribute_printf(1, 2); +static void print_msg(VerbosityLevelEnum level, const char *fmt,...) +pg_attribute_printf(2, 3); + +static int run_pg_ctl(const char *arg); +static void run_basebackup(const char *provider_connstr, const char *data_dir, + const char *extra_basebackup_args); +static void wait_postmaster_connection(const char *connstr); +static void wait_primary_connection(const char *connstr); +static void wait_postmaster_shutdown(void); + +static char *validate_replication_set_input(char *replication_sets); + +static void remove_unwanted_data(PGconn *conn); +static void initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn); +static char *create_restore_point(PGconn *conn, char *restore_point_name); +static char *initialize_replication_slot(PGconn *conn, char *dbname, + char *provider_node_name, char *subscription_name, + bool drop_slot_if_exists); +static void spock_subscribe(PGconn *conn, char *subscriber_name, + char *subscriber_dsn, + char *provider_connstr, + char *replication_sets, + int apply_delay, + bool force_text_transfer); + +static RemoteInfo *get_remote_info(PGconn* conn); + +static bool extension_exists(PGconn *conn, const char *extname); +static void install_extension(PGconn *conn, const char *extname); + +static void initialize_data_dir(char *data_dir, char *connstr, + char *postgresql_conf, char *pg_hba_conf, + char *extra_basebackup_args); +static bool check_data_dir(char *data_dir, RemoteInfo *remoteinfo); + +static char *read_sysid(const char *data_dir); + +static void WriteRecoveryConf(PQExpBuffer contents); +static void CopyConfFile(char *fromfile, char *tofile, bool append); + +static char *get_connstr_dbname(char *connstr); +static char *get_connstr(char *connstr, char *dbname); +static char *PQconninfoParamsToConnstr(const char *const * keywords, const char *const * values); +static void appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str); + +static bool file_exists(const char *path); +static bool is_pg_dir(const char *path); +static void copy_file(char *fromfile, char *tofile, bool append); +static char *find_other_exec_or_die(const char *argv0, const char *target); +static bool postmaster_is_alive(pid_t pid); +static long get_pgpid(void); +static char **get_database_list(char *databases, int *n_databases); +static char *generate_restore_point_name(void); + +static PGconn * +connectdb(const char *connstr) +{ + PGconn *conn; + + conn = PQconnectdb(connstr); + if (PQstatus(conn) != CONNECTION_OK) + die(_("Connection to database failed: %s, connection string was: %s\n"), PQerrorMessage(conn), connstr); + + return conn; +} + +void signal_handler(int sig) +{ + if (sig == SIGINT) + { + die(_("\nCanceling...\n")); + } +} + + +int +main(int argc, char **argv) +{ + int i; + int c; + PQExpBuffer recoveryconfcontents = createPQExpBuffer(); + RemoteInfo *remote_info; + char *remote_lsn; + bool stop = false; + bool drop_slot_if_exists = false; + int optindex; + char *subscriber_name = NULL; + char *base_sub_connstr = NULL; + char *base_prov_connstr = NULL; + char *replication_sets = NULL; + char *databases = NULL; + char *postgresql_conf = NULL, + *pg_hba_conf = NULL, + *recovery_conf = NULL; + int apply_delay = 0; + bool force_text_transfer = false; + char **slot_names; + char *sub_connstr; + char *prov_connstr; + char **database_list = { NULL }; + int n_databases = 1; + int dbnum; + bool use_existing_data_dir = false; + int pg_ctl_ret, + logfd; + char *restore_point_name = NULL; + char *extra_basebackup_args = NULL; + + static struct option long_options[] = { + {"subscriber-name", required_argument, NULL, 'n'}, + {"pgdata", required_argument, NULL, 'D'}, + {"provider-dsn", required_argument, NULL, 1}, + {"subscriber-dsn", required_argument, NULL, 2}, + {"replication-sets", required_argument, NULL, 3}, + {"postgresql-conf", required_argument, NULL, 4}, + {"hba-conf", required_argument, NULL, 5}, + {"recovery-conf", required_argument, NULL, 6}, + {"stop", no_argument, NULL, 's'}, + {"drop-slot-if-exists", no_argument, NULL, 7}, + {"apply-delay", required_argument, NULL, 8}, + {"databases", required_argument, NULL, 9}, + {"extra-basebackup-args", required_argument, NULL, 10}, + {"text-types", no_argument, NULL, 11}, + {NULL, 0, NULL, 0} + }; + + argv0 = argv[0]; + progname = get_progname(argv[0]); + start_time = time(NULL); + signal(SIGINT, signal_handler); + + /* check for --help */ + if (argc > 1) + { + for (i = 1; i < argc; i++) + { + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-?") == 0) + { + usage(); + exit(0); + } + } + } + + /* Option parsing and validation */ + while ((c = getopt_long(argc, argv, "D:n:sv", long_options, &optindex)) != -1) + { + switch (c) + { + case 'D': + data_dir = pg_strdup(optarg); + break; + case 'n': + subscriber_name = pg_strdup(optarg); + break; + case 1: + base_prov_connstr = pg_strdup(optarg); + break; + case 2: + base_sub_connstr = pg_strdup(optarg); + break; + case 3: + replication_sets = validate_replication_set_input(pg_strdup(optarg)); + break; + case 4: + { + postgresql_conf = pg_strdup(optarg); + if (postgresql_conf != NULL && !file_exists(postgresql_conf)) + die(_("The specified postgresql.conf file does not exist.")); + break; + } + case 5: + { + pg_hba_conf = pg_strdup(optarg); + if (pg_hba_conf != NULL && !file_exists(pg_hba_conf)) + die(_("The specified pg_hba.conf file does not exist.")); + break; + } + case 6: + { + recovery_conf = pg_strdup(optarg); + if (recovery_conf != NULL && !file_exists(recovery_conf)) + die(_("The specified recovery configuration file does not exist.")); + break; + } + case 'v': + verbosity++; + break; + case 's': + stop = true; + break; + case 7: + drop_slot_if_exists = true; + break; + case 8: + apply_delay = atoi(optarg); + break; + case 9: + databases = pg_strdup(optarg); + break; + case 10: + extra_basebackup_args = pg_strdup(optarg); + break; + case 11: + force_text_transfer = true; + break; + default: + fprintf(stderr, _("Unknown option\n")); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + } + + /* + * Sanity checks + */ + + if (data_dir == NULL) + { + fprintf(stderr, _("No data directory specified\n")); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + else if (subscriber_name == NULL) + { + fprintf(stderr, _("No subscriber name specified\n")); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + + if (!base_prov_connstr || !strlen(base_prov_connstr)) + die(_("Provider connection string must be specified.\n")); + if (!base_sub_connstr || !strlen(base_sub_connstr)) + die(_("Subscriber connection string must be specified.\n")); + + if (apply_delay < 0) + die(_("Apply delay cannot be negative.\n")); + + if (apply_delay > MAX_APPLY_DELAY) + die(_("Apply delay cannot be more than %d.\n"), MAX_APPLY_DELAY); + + if (!replication_sets || !strlen(replication_sets)) + replication_sets = "default,default_insert_only,ddl_sql"; + + /* Init random numbers used for slot suffixes, etc */ + srand(time(NULL)); + + /* Parse database list or connection string. */ + if (databases != NULL) + { + database_list = get_database_list(databases, &n_databases); + } + else + { + char *dbname = get_connstr_dbname(base_prov_connstr); + + if (!dbname) + die(_("Either provider connection string must contain database " + "name or --databases option must be specified.\n")); + + n_databases = 1; + database_list = palloc(n_databases * sizeof(char *)); + database_list[0] = dbname; + } + + slot_names = palloc(n_databases * sizeof(char *)); + + /* + * Check connection strings for validity before doing anything + * expensive. + */ + for (dbnum = 0; dbnum < n_databases; dbnum++) + { + char *db = database_list[dbnum]; + + prov_connstr = get_connstr(base_prov_connstr, db); + if (!prov_connstr || !strlen(prov_connstr)) + die(_("Provider connection string is not valid.\n")); + + sub_connstr = get_connstr(base_sub_connstr, db); + if (!sub_connstr || !strlen(sub_connstr)) + die(_("Subscriber connection string is not valid.\n")); + } + + /* + * Create log file where new postgres instance will log to while being + * initialized. + */ + logfd = open("spock_create_subscriber_postgres.log", O_CREAT | O_RDWR, + S_IRUSR | S_IWUSR); + if (logfd == -1) + { + die(_("Creating spock_create_subscriber_postgres.log failed: %s"), + strerror(errno)); + } + /* Safe to close() unchecked, we didn't write */ + (void) close(logfd); + + /* Let's start the real work... */ + print_msg(VERBOSITY_NORMAL, _("%s: starting ...\n"), progname); + + for (dbnum = 0; dbnum < n_databases; dbnum++) + { + char *db = database_list[dbnum]; + + prov_connstr = get_connstr(base_prov_connstr, db); + if (!prov_connstr || !strlen(prov_connstr)) + die(_("Provider connection string is not valid.\n")); + + /* Read the remote server indetification. */ + print_msg(VERBOSITY_NORMAL, + _("Getting information for database %s ...\n"), db); + provider_conn = connectdb(prov_connstr); + remote_info = get_remote_info(provider_conn); + + /* only need to do this piece once */ + + if (dbnum == 0) + { + use_existing_data_dir = check_data_dir(data_dir, remote_info); + + if (use_existing_data_dir && + strcmp(remote_info->sysid, read_sysid(data_dir)) != 0) + die(_("Subscriber data directory is not basebackup of remote node.\n")); + } + + /* + * Create replication slots on remote node. + */ + print_msg(VERBOSITY_NORMAL, + _("Creating replication slot in database %s ...\n"), db); + slot_names[dbnum] = initialize_replication_slot(provider_conn, + remote_info->dbname, + remote_info->node_name, + subscriber_name, + drop_slot_if_exists); + PQfinish(provider_conn); + provider_conn = NULL; + } + + /* + * Create basebackup or use existing one + */ + prov_connstr = get_connstr(base_prov_connstr, database_list[0]); + sub_connstr = get_connstr(base_sub_connstr, database_list[0]); + + initialize_data_dir(data_dir, + use_existing_data_dir ? NULL : prov_connstr, + postgresql_conf, pg_hba_conf, + extra_basebackup_args); + snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); + + restore_point_name = generate_restore_point_name(); + + print_msg(VERBOSITY_NORMAL, _("Creating restore point \"%s\" on remote node ...\n"), + restore_point_name); + provider_conn = connectdb(prov_connstr); + remote_lsn = create_restore_point(provider_conn, restore_point_name); + PQfinish(provider_conn); + provider_conn = NULL; + + /* + * Get subscriber db to consistent state (for lsn after slot creation). + */ + print_msg(VERBOSITY_NORMAL, + _("Bringing subscriber node to the restore point ...\n")); + if (recovery_conf) + { + CopyConfFile(recovery_conf, "postgresql.auto.conf", true); + } + else + { + appendPQExpBuffer(recoveryconfcontents, "primary_conninfo = '%s'\n", + escape_single_quotes_ascii(prov_connstr)); + } + appendPQExpBuffer(recoveryconfcontents, "recovery_target_name = '%s'\n", restore_point_name); + appendPQExpBuffer(recoveryconfcontents, "recovery_target_inclusive = true\n"); + appendPQExpBuffer(recoveryconfcontents, "recovery_target_action = promote\n"); + WriteRecoveryConf(recoveryconfcontents); + + free(restore_point_name); + restore_point_name = NULL; + + /* + * Start subscriber node with spock disabled, and wait until it starts + * accepting connections which means it has caught up to the restore point. + */ + pg_ctl_ret = run_pg_ctl("start -l \"spock_create_subscriber_postgres.log\" -o \"-c shared_preload_libraries=''\""); + if (pg_ctl_ret != 0) + die(_("Postgres startup for restore point catchup failed with %d. See spock_create_subscriber_postgres.log."), pg_ctl_ret); + + wait_primary_connection(sub_connstr); + + /* + * Clean any per-node data that were copied by pg_basebackup. + */ + print_msg(VERBOSITY_VERBOSE, + _("Removing old spock configuration ...\n")); + + for (dbnum = 0; dbnum < n_databases; dbnum++) + { + char *db = database_list[dbnum]; + + sub_connstr = get_connstr(base_sub_connstr, db); + + if (!sub_connstr || !strlen(sub_connstr)) + die(_("Subscriber connection string is not valid.\n")); + + subscriber_conn = connectdb(sub_connstr); + remove_unwanted_data(subscriber_conn); + PQfinish(subscriber_conn); + subscriber_conn = NULL; + } + + /* Stop Postgres so we can reset system id and start it with spock loaded. */ + pg_ctl_ret = run_pg_ctl("stop"); + if (pg_ctl_ret != 0) + die(_("Postgres stop after restore point catchup failed with %d. See spock_create_subscriber_postgres.log."), pg_ctl_ret); + wait_postmaster_shutdown(); + + /* + * Start the node again, now with spock active so that we can start the + * logical replication. This is final start, so don't log to to special log + * file anymore. + */ + print_msg(VERBOSITY_NORMAL, + _("Initializing spock on the subscriber node:\n")); + + pg_ctl_ret = run_pg_ctl("start"); + if (pg_ctl_ret != 0) + die(_("Postgres restart with spock enabled failed with %d."), pg_ctl_ret); + wait_postmaster_connection(base_sub_connstr); + + for (dbnum = 0; dbnum < n_databases; dbnum++) + { + char *db = database_list[dbnum]; + + sub_connstr = get_connstr(base_sub_connstr, db); + prov_connstr = get_connstr(base_prov_connstr, db); + + subscriber_conn = connectdb(sub_connstr); + + /* Create the extension. */ + print_msg(VERBOSITY_VERBOSE, + _("Creating spock extension for database %s...\n"), db); + if (PQserverVersion(subscriber_conn) < 90500) + install_extension(subscriber_conn, "spock_origin"); + install_extension(subscriber_conn, "spock"); + + /* + * Create the identifier which is setup with the position to which we + * already caught up using physical replication. + */ + print_msg(VERBOSITY_VERBOSE, + _("Creating replication origin for database %s...\n"), db); + initialize_replication_origin(subscriber_conn, slot_names[dbnum], remote_lsn); + + /* + * And finally add the node to the cluster. + */ + print_msg(VERBOSITY_NORMAL, _("Creating subscriber %s for database %s...\n"), + subscriber_name, db); + print_msg(VERBOSITY_VERBOSE, _("Replication sets: %s\n"), replication_sets); + + spock_subscribe(subscriber_conn, subscriber_name, sub_connstr, + prov_connstr, replication_sets, apply_delay, + force_text_transfer); + + PQfinish(subscriber_conn); + subscriber_conn = NULL; + } + + /* If user does not want the node to be running at the end, stop it. */ + if (stop) + { + print_msg(VERBOSITY_NORMAL, _("Stopping the subscriber node ...\n")); + pg_ctl_ret = run_pg_ctl("stop"); + if (pg_ctl_ret != 0) + die(_("Stopping postgres after successful subscribtion failed with %d."), pg_ctl_ret); + wait_postmaster_shutdown(); + } + + print_msg(VERBOSITY_NORMAL, _("All done\n")); + + return 0; +} + + +/* + * Print help. + */ +static void +usage(void) +{ + printf(_("%s create new spock subscriber from basebackup of provider.\n\n"), progname); + printf(_("Usage:\n")); + printf(_(" %s [OPTION]...\n"), progname); + printf(_("\nGeneral options:\n")); + printf(_(" -D, --pgdata=DIRECTORY data directory to be used for new node,\n")); + printf(_(" can be either empty/non-existing directory,\n")); + printf(_(" or directory populated using\n")); + printf(_(" pg_basebackup -X stream command\n")); + printf(_(" --databases optional list of databases to replicate\n")); + printf(_(" -n, --subscriber-name=NAME name of the newly created subscriber\n")); + printf(_(" --subscriber-dsn=CONNSTR connection string to the newly created subscriber\n")); + printf(_(" --provider-dsn=CONNSTR connection string to the provider\n")); + printf(_(" --replication-sets=SETS comma separated list of replication set names\n")); + printf(_(" --apply-delay=DELAY apply delay in seconds (by default 0)\n")); + printf(_(" --drop-slot-if-exists drop replication slot of conflicting name\n")); + printf(_(" -s, --stop stop the server once the initialization is done\n")); + printf(_(" -v increase logging verbosity\n")); + printf(_(" --extra-basebackup-args additional arguments to pass to pg_basebackup.\n")); + printf(_(" Safe options: -T, -c, --xlogdir/--waldir\n")); + printf(_("\nConfiguration files override:\n")); + printf(_(" --hba-conf path to the new pg_hba.conf\n")); + printf(_(" --postgresql-conf path to the new postgresql.conf\n")); + printf(_(" --recovery-conf path to the template recovery configuration\n")); +} + +/* + * Print error and exit. + */ +static void +die(const char *fmt,...) +{ + va_list argptr; + va_start(argptr, fmt); + vfprintf(stderr, fmt, argptr); + va_end(argptr); + + if (subscriber_conn) + PQfinish(subscriber_conn); + if (provider_conn) + PQfinish(provider_conn); + + if (get_pgpid()) + { + if (!run_pg_ctl("stop -s")) + { + fprintf(stderr, _("WARNING: postgres seems to be running, but could not be stopped\n")); + } + } + + exit(1); +} + +/* + * Print message to stdout and flush + */ +static void +print_msg(VerbosityLevelEnum level, const char *fmt,...) +{ + if (verbosity >= level) + { + va_list argptr; + va_start(argptr, fmt); + vfprintf(stdout, fmt, argptr); + va_end(argptr); + fflush(stdout); + } +} + + +/* + * Start pg_ctl with given argument(s) - used to start/stop postgres + * + * Returns the exit code reported by pg_ctl. If pg_ctl exits due to a + * signal this call will die and not return. + */ +static int +run_pg_ctl(const char *arg) +{ + int ret; + PQExpBuffer cmd = createPQExpBuffer(); + char *exec_path = find_other_exec_or_die(argv0, "pg_ctl"); + + appendPQExpBuffer(cmd, "%s %s -D \"%s\"", exec_path, arg, data_dir); + + /* Run pg_ctl in silent mode unless we run in debug mode. */ + if (verbosity < VERBOSITY_DEBUG) + appendPQExpBuffer(cmd, " -s"); + + print_msg(VERBOSITY_DEBUG, _("Running pg_ctl: %s.\n"), cmd->data); + ret = system(cmd->data); + + destroyPQExpBuffer(cmd); + + if (WIFEXITED(ret)) + return WEXITSTATUS(ret); + else if (WIFSIGNALED(ret)) + die(_("pg_ctl exited with signal %d"), WTERMSIG(ret)); + else + die(_("pg_ctl exited for an unknown reason (system() returned %d)"), ret); + + return -1; +} + + +/* + * Run pg_basebackup to create the copy of the origin node. + */ +static void +run_basebackup(const char *provider_connstr, const char *data_dir, + const char *extra_basebackup_args) +{ + int ret; + PQExpBuffer cmd = createPQExpBuffer(); + char *exec_path = find_other_exec_or_die(argv0, "pg_basebackup"); + + appendPQExpBuffer(cmd, "%s -D \"%s\" -d \"%s\" -X s -P", exec_path, data_dir, provider_connstr); + + /* Run pg_basebackup in verbose mode if we are running in verbose mode. */ + if (verbosity >= VERBOSITY_VERBOSE) + appendPQExpBuffer(cmd, " -v"); + + if (extra_basebackup_args != NULL) + appendPQExpBuffer(cmd, "%s", extra_basebackup_args); + + print_msg(VERBOSITY_DEBUG, _("Running pg_basebackup: %s.\n"), cmd->data); + ret = system(cmd->data); + + destroyPQExpBuffer(cmd); + + if (WIFEXITED(ret) && WEXITSTATUS(ret) == 0) + return; + if (WIFEXITED(ret)) + die(_("pg_basebackup failed with exit status %d, cannot continue.\n"), WEXITSTATUS(ret)); + else if (WIFSIGNALED(ret)) + die(_("pg_basebackup exited with signal %d, cannot continue"), WTERMSIG(ret)); + else + die(_("pg_basebackup exited for an unknown reason (system() returned %d)"), ret); +} + +/* + * Init the datadir + * + * This function can either ensure provided datadir is a postgres datadir, + * or create it using pg_basebackup. + * + * In any case, new postresql.conf and pg_hba.conf will be copied to the + * datadir if they are provided. + */ +static void +initialize_data_dir(char *data_dir, char *connstr, + char *postgresql_conf, char *pg_hba_conf, + char *extra_basebackup_args) +{ + if (connstr) + { + print_msg(VERBOSITY_NORMAL, + _("Creating base backup of the remote node...\n")); + run_basebackup(connstr, data_dir, extra_basebackup_args); + } + + if (postgresql_conf) + CopyConfFile(postgresql_conf, "postgresql.conf", false); + if (pg_hba_conf) + CopyConfFile(pg_hba_conf, "pg_hba.conf", false); +} + +/* + * This function checks if provided datadir is clone of the remote node + * described by the remote info, or if it's emtpy directory that can be used + * as new datadir. + */ +static bool +check_data_dir(char *data_dir, RemoteInfo *remoteinfo) +{ + /* Run basebackup as needed. */ + switch (pg_check_dir(data_dir)) + { + case 0: /* Does not exist */ + case 1: /* Exists, empty */ + return false; + case 2: + case 3: /* Exists, not empty */ + case 4: + { + if (!is_pg_dir(data_dir)) + die(_("Directory \"%s\" exists but is not valid postgres data directory.\n"), + data_dir); + return true; + } + case -1: /* Access problem */ + die(_("Could not access directory \"%s\": %s.\n"), + data_dir, strerror(errno)); + } + + /* Unreachable */ + die(_("Unexpected result from pg_check_dir() call")); + return false; +} + +/* + * Initialize replication slots + */ +static char * +initialize_replication_slot(PGconn *conn, char *dbname, + char *provider_node_name, char *subscription_name, + bool drop_slot_if_exists) +{ + PQExpBufferData query; + char *slot_name; + PGresult *res; + + /* Generate the slot name. */ + initPQExpBuffer(&query); + printfPQExpBuffer(&query, + "SELECT spock.spock_gen_slot_name(%s, %s, %s)", + PQescapeLiteral(conn, dbname, strlen(dbname)), + PQescapeLiteral(conn, provider_node_name, + strlen(provider_node_name)), + PQescapeLiteral(conn, subscription_name, + strlen(subscription_name))); + + res = PQexec(conn, query.data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("Could generate slot name: %s"), PQerrorMessage(conn)); + + slot_name = pstrdup(PQgetvalue(res, 0, 0)); + + PQclear(res); + resetPQExpBuffer(&query); + + /* Check if the current slot exists. */ + printfPQExpBuffer(&query, + "SELECT 1 FROM pg_catalog.pg_replication_slots WHERE slot_name = %s", + PQescapeLiteral(conn, slot_name, strlen(slot_name))); + + res = PQexec(conn, query.data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("Could not fetch existing slot information: %s"), PQerrorMessage(conn)); + + /* Drop the existing slot when asked for it or error if it already exists. */ + if (PQntuples(res) > 0) + { + PQclear(res); + resetPQExpBuffer(&query); + + if (!drop_slot_if_exists) + die(_("Slot %s already exists, drop it or use --drop-slot-if-exists to drop it automatically.\n"), + slot_name); + + print_msg(VERBOSITY_VERBOSE, + _("Droping existing slot %s ...\n"), slot_name); + + printfPQExpBuffer(&query, + "SELECT pg_catalog.pg_drop_replication_slot(%s)", + PQescapeLiteral(conn, slot_name, strlen(slot_name))); + + res = PQexec(conn, query.data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("Could not drop existing slot %s: %s"), slot_name, + PQerrorMessage(conn)); + } + + PQclear(res); + resetPQExpBuffer(&query); + + /* And finally, create the slot. */ + appendPQExpBuffer(&query, "SELECT pg_create_logical_replication_slot(%s, '%s');", + PQescapeLiteral(conn, slot_name, strlen(slot_name)), + "spock_output"); + + res = PQexec(conn, query.data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("Could not create replication slot, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + + PQclear(res); + termPQExpBuffer(&query); + + return slot_name; +} + +/* + * Read replication info about remote connection + * + * TODO: unify with spock_remote_node_info in spock_rpc + */ +static RemoteInfo * +get_remote_info(PGconn* conn) +{ + RemoteInfo *ri = (RemoteInfo *)pg_malloc0(sizeof(RemoteInfo)); + PGresult *res; + + if (!extension_exists(conn, "spock")) + die(_("The remote node is not configured as a spock provider.\n")); + + res = PQexec(conn, "SELECT node_id, node_name, sysid, dbname, replication_sets FROM spock.node_info()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not fetch remote node info: %s\n"), PQerrorMessage(conn)); + + /* No nodes found? */ + if (PQntuples(res) == 0) + die(_("The remote database is not configured as a spock node.\n")); + + if (PQntuples(res) > 1) + die(_("The remote database has multiple nodes configured. That is not supported with current version of spock.\n")); + +#define atooid(x) ((Oid) strtoul((x), NULL, 10)) + + ri->nodeid = atooid(PQgetvalue(res, 0, 0)); + ri->node_name = pstrdup(PQgetvalue(res, 0, 1)); + ri->sysid = pstrdup(PQgetvalue(res, 0, 2)); + ri->dbname = pstrdup(PQgetvalue(res, 0, 3)); + ri->replication_sets = pstrdup(PQgetvalue(res, 0, 4)); + + PQclear(res); + + return ri; +} + +/* + * Check if extension exists. + */ +static bool +extension_exists(PGconn *conn, const char *extname) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + bool ret; + + printfPQExpBuffer(query, "SELECT 1 FROM pg_catalog.pg_extension WHERE extname = %s;", + PQescapeLiteral(conn, extname, strlen(extname))); + res = PQexec(conn, query->data); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("Could not read extension info: %s\n"), PQerrorMessage(conn)); + } + + ret = PQntuples(res) == 1; + + PQclear(res); + destroyPQExpBuffer(query); + + return ret; +} + +/* + * Create extension. + */ +static void +install_extension(PGconn *conn, const char *extname) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + + printfPQExpBuffer(query, "CREATE EXTENSION IF NOT EXISTS %s;", + PQescapeIdentifier(conn, extname, strlen(extname))); + res = PQexec(conn, query->data); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + PQclear(res); + die(_("Could not install %s extension: %s\n"), extname, PQerrorMessage(conn)); + } + + PQclear(res); + destroyPQExpBuffer(query); +} + +/* + * Clean all the data that was copied from remote node but we don't + * want it here (currently shared security labels and replication identifiers). + */ +static void +remove_unwanted_data(PGconn *conn) +{ + PGresult *res; + + /* + * Remove replication identifiers (9.4 will get them removed by dropping + * the extension later as we emulate them there). + */ + if (PQserverVersion(conn) >= 90500) + { + res = PQexec(conn, "SELECT pg_replication_origin_drop(external_id) FROM pg_replication_origin_status;"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("Could not remove existing replication origins: %s\n"), PQerrorMessage(conn)); + } + PQclear(res); + } + + res = PQexec(conn, "DROP EXTENSION spock CASCADE;"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + die(_("Could not clean the spock extension, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); +} + +/* + * Initialize new remote identifier to specific position. + */ +static void +initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn) +{ + PGresult *res; + PQExpBuffer query = createPQExpBuffer(); + + if (PQserverVersion(conn) >= 90500) + { + printfPQExpBuffer(query, "SELECT pg_replication_origin_create(%s)", + PQescapeLiteral(conn, origin_name, strlen(origin_name))); + + res = PQexec(conn, query->data); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("Could not create replication origin \"%s\": status %s: %s\n"), + query->data, + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); + + if (remote_lsn) + { + printfPQExpBuffer(query, "SELECT pg_replication_origin_advance(%s, '%s')", + PQescapeLiteral(conn, origin_name, strlen(origin_name)), + remote_lsn); + + res = PQexec(conn, query->data); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("Could not advance replication origin \"%s\": status %s: %s\n"), + query->data, + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); + } + } + else + { + printfPQExpBuffer(query, "INSERT INTO spock_origin.replication_origin (roident, roname, roremote_lsn) SELECT COALESCE(MAX(roident::int), 0) + 1, %s, %s FROM spock_origin.replication_origin", + PQescapeLiteral(conn, origin_name, strlen(origin_name)), + remote_lsn ? PQescapeLiteral(conn, remote_lsn, strlen(remote_lsn)) : "0"); + + res = PQexec(conn, query->data); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + die(_("Could not create replication origin \"%s\": status %s: %s\n"), + query->data, + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); + } + + destroyPQExpBuffer(query); +} + + +/* + * Create remote restore point which will be used to get into synchronized + * state through physical replay. + */ +static char * +create_restore_point(PGconn *conn, char *restore_point_name) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + char *remote_lsn = NULL; + + printfPQExpBuffer(query, "SELECT pg_create_restore_point('%s')", restore_point_name); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("Could not create restore point, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + remote_lsn = pstrdup(PQgetvalue(res, 0, 0)); + + PQclear(res); + destroyPQExpBuffer(query); + + return remote_lsn; +} + +static void +spock_subscribe(PGconn *conn, char *subscriber_name, char *subscriber_dsn, + char *provider_dsn, char *replication_sets, + int apply_delay, bool force_text_transfer) +{ + PQExpBufferData query; + PQExpBufferData repsets; + PGresult *res; + + initPQExpBuffer(&query); + printfPQExpBuffer(&query, + "SELECT spock.node_create(node_name := %s, dsn := %s);", + PQescapeLiteral(conn, subscriber_name, strlen(subscriber_name)), + PQescapeLiteral(conn, subscriber_dsn, strlen(subscriber_dsn))); + + res = PQexec(conn, query.data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("Could not create local node, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); + + resetPQExpBuffer(&query); + initPQExpBuffer(&repsets); + + printfPQExpBuffer(&repsets, "{%s}", replication_sets); + printfPQExpBuffer(&query, + "SELECT spock.sub_create(" + "subscription_name := %s, provider_dsn := %s, " + "replication_sets := %s, " + "apply_delay := '%d seconds'::interval, " + "synchronize_structure := false, " + "synchronize_data := false, " + "force_text_transfer := '%s');", + PQescapeLiteral(conn, subscriber_name, strlen(subscriber_name)), + PQescapeLiteral(conn, provider_dsn, strlen(provider_dsn)), + PQescapeLiteral(conn, repsets.data, repsets.len), + apply_delay, (force_text_transfer ? "t" : "f")); + + res = PQexec(conn, query.data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("Could not create subscription, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); + + /* TODO */ + res = PQexec(conn, "UPDATE spock.local_sync_status SET sync_status = 'r'"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + die(_("Could not update subscription, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + + PQclear(res); + + termPQExpBuffer(&repsets); + termPQExpBuffer(&query); +} + + +/* + * Validates input of the replication sets and returns normalized data. + */ +static char * +validate_replication_set_input(char *replication_sets) +{ + char *name; + PQExpBuffer retbuf = createPQExpBuffer(); + char *ret; + bool first = true; + + if (!replication_sets) + return NULL; + + name = strtok(replication_sets, " ,"); + while (name != NULL) + { + const char *cp; + + if (strlen(name) == 0) + die(_("Replication set name \"%s\" is too short\n"), name); + + if (strlen(name) > NAMEDATALEN) + die(_("Replication set name \"%s\" is too long\n"), name); + + for (cp = name; *cp; cp++) + { + if (!((*cp >= 'a' && *cp <= 'z') + || (*cp >= '0' && *cp <= '9') + || (*cp == '_') + || (*cp == '-'))) + { + die(_("Replication set name \"%s\" contains invalid character\n"), + name); + } + } + + if (first) + first = false; + else + appendPQExpBufferStr(retbuf, ", "); + appendPQExpBufferStr(retbuf, name); + + name = strtok(NULL, " ,"); + } + + ret = pg_strdup(retbuf->data); + destroyPQExpBuffer(retbuf); + + return ret; +} + +static char * +get_connstr_dbname(char *connstr) +{ + PQconninfoOption *conn_opts = NULL; + PQconninfoOption *conn_opt; + char *err_msg = NULL; + char *ret = NULL; + + conn_opts = PQconninfoParse(connstr, &err_msg); + if (conn_opts == NULL) + { + die(_("Invalid connection string: %s\n"), err_msg); + } + + for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++) + { + if (strcmp(conn_opt->keyword, "dbname") == 0) + { + ret = pstrdup(conn_opt->val); + break; + } + } + + PQconninfoFree(conn_opts); + + return ret; +} + + +/* + * Build connection string from individual parameter. + * + * dbname can be specified in connstr parameter + */ +static char * +get_connstr(char *connstr, char *dbname) +{ + char *ret; + int argcount = 4; /* dbname, host, user, port */ + int i; + const char **keywords; + const char **values; + PQconninfoOption *conn_opts = NULL; + PQconninfoOption *conn_opt; + char *err_msg = NULL; + + /* + * Merge the connection info inputs given in form of connection string + * and options + */ + i = 0; + if (connstr && + (strncmp(connstr, "postgresql://", 13) == 0 || + strncmp(connstr, "postgres://", 11) == 0 || + strchr(connstr, '=') != NULL)) + { + conn_opts = PQconninfoParse(connstr, &err_msg); + if (conn_opts == NULL) + { + die(_("Invalid connection string: %s\n"), err_msg); + } + + for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++) + { + if (conn_opt->val != NULL && conn_opt->val[0] != '\0') + argcount++; + } + + keywords = pg_malloc0((argcount + 1) * sizeof(*keywords)); + values = pg_malloc0((argcount + 1) * sizeof(*values)); + + for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++) + { + /* If db* parameters were provided, we'll fill them later. */ + if (dbname && strcmp(conn_opt->keyword, "dbname") == 0) + continue; + + if (conn_opt->val != NULL && conn_opt->val[0] != '\0') + { + keywords[i] = conn_opt->keyword; + values[i] = conn_opt->val; + i++; + } + } + } + else + { + keywords = pg_malloc0((argcount + 1) * sizeof(*keywords)); + values = pg_malloc0((argcount + 1) * sizeof(*values)); + + /* + * If connstr was provided but it's not in connection string format and + * the dbname wasn't provided then connstr is actually dbname. + */ + if (connstr && !dbname) + dbname = connstr; + } + + if (dbname) + { + keywords[i] = "dbname"; + values[i] = dbname; + i++; + } + + ret = PQconninfoParamsToConnstr(keywords, values); + + /* Connection ok! */ + pg_free(values); + pg_free(keywords); + if (conn_opts) + PQconninfoFree(conn_opts); + + return ret; +} + + +/* + * Reads the pg_control file of the existing data dir. + */ +static char * +read_sysid(const char *data_dir) +{ + ControlFileData ControlFile; + int fd; + char ControlFilePath[MAXPGPATH]; + char *res = (char *) pg_malloc0(33); + + snprintf(ControlFilePath, MAXPGPATH, "%s/global/pg_control", data_dir); + + if ((fd = open(ControlFilePath, O_RDONLY | PG_BINARY, 0)) == -1) + die(_("%s: could not open file \"%s\" for reading: %s\n"), + progname, ControlFilePath, strerror(errno)); + + if (read(fd, &ControlFile, sizeof(ControlFileData)) != sizeof(ControlFileData)) + die(_("%s: could not read file \"%s\": %s\n"), + progname, ControlFilePath, strerror(errno)); + + close(fd); + + snprintf(res, 33, UINT64_FORMAT, ControlFile.system_identifier); + return res; +} + +/* + * Write contents of recovery.conf or postgresql.auto.conf + */ +static void +WriteRecoveryConf(PQExpBuffer contents) +{ + char filename[MAXPGPATH]; + FILE *cf; + + sprintf(filename, "%s/postgresql.auto.conf", data_dir); + + cf = fopen(filename, "a"); + if (cf == NULL) + { + die(_("%s: could not create file \"%s\": %s\n"), progname, filename, strerror(errno)); + } + + if (fwrite(contents->data, contents->len, 1, cf) != 1) + { + die(_("%s: could not write to file \"%s\": %s\n"), + progname, filename, strerror(errno)); + } + + fclose(cf); + + { + sprintf(filename, "%s/standby.signal", data_dir); + cf = fopen(filename, "w"); + if (cf == NULL) + { + die(_("%s: could not create file \"%s\": %s\n"), progname, filename, strerror(errno)); + } + + fclose(cf); + } +} + +/* + * Copy file to data + */ +static void +CopyConfFile(char *fromfile, char *tofile, bool append) +{ + char filename[MAXPGPATH]; + + sprintf(filename, "%s/%s", data_dir, tofile); + + print_msg(VERBOSITY_DEBUG, _("Copying \"%s\" to \"%s\".\n"), + fromfile, filename); + copy_file(fromfile, filename, append); +} + + +/* + * Convert PQconninfoOption array into conninfo string + */ +static char * +PQconninfoParamsToConnstr(const char *const * keywords, const char *const * values) +{ + PQExpBuffer retbuf = createPQExpBuffer(); + char *ret; + int i = 0; + + for (i = 0; keywords[i] != NULL; i++) + { + if (i > 0) + appendPQExpBufferChar(retbuf, ' '); + appendPQExpBuffer(retbuf, "%s=", keywords[i]); + appendPQExpBufferConnstrValue(retbuf, values[i]); + } + + ret = pg_strdup(retbuf->data); + destroyPQExpBuffer(retbuf); + + return ret; +} + +/* + * Escape connection info value + */ +static void +appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str) +{ + const char *s; + bool needquotes; + + /* + * If the string consists entirely of plain ASCII characters, no need to + * quote it. This is quite conservative, but better safe than sorry. + */ + needquotes = false; + for (s = str; *s; s++) + { + if (!((*s >= 'a' && *s <= 'z') || (*s >= 'A' && *s <= 'Z') || + (*s >= '0' && *s <= '9') || *s == '_' || *s == '.')) + { + needquotes = true; + break; + } + } + + if (needquotes) + { + appendPQExpBufferChar(buf, '\''); + while (*str) + { + /* ' and \ must be escaped by to \' and \\ */ + if (*str == '\'' || *str == '\\') + appendPQExpBufferChar(buf, '\\'); + + appendPQExpBufferChar(buf, *str); + str++; + } + appendPQExpBufferChar(buf, '\''); + } + else + appendPQExpBufferStr(buf, str); +} + + +/* + * Find the pgport and try a connection + */ +static void +wait_postmaster_connection(const char *connstr) +{ + PGPing res; + long pmpid = 0; + + print_msg(VERBOSITY_VERBOSE, "Waiting for PostgreSQL to accept connections ..."); + + /* First wait for Postmaster to come up. */ + for (;;) + { + if ((pmpid = get_pgpid()) != 0 && + postmaster_is_alive((pid_t) pmpid)) + break; + + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_VERBOSE, "."); + } + + /* Now wait for Postmaster to either accept connections or die. */ + for (;;) + { + res = PQping(connstr); + if (res == PQPING_OK) + break; + else if (res == PQPING_NO_ATTEMPT) + break; + + /* + * Check if the process is still alive. This covers cases where the + * postmaster successfully created the pidfile but then crashed without + * removing it. + */ + if (!postmaster_is_alive((pid_t) pmpid)) + break; + + /* No response; wait */ + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_VERBOSE, "."); + } + + print_msg(VERBOSITY_VERBOSE, "\n"); +} + + +/* + * Wait for PostgreSQL to leave recovery/standby mode + */ +static void +wait_primary_connection(const char *connstr) +{ + bool ispri = false; + PGconn *conn = NULL; + PGresult *res; + + wait_postmaster_connection(connstr); + + print_msg(VERBOSITY_VERBOSE, "Waiting for PostgreSQL to become primary..."); + + while (!ispri) + { + if (!conn || PQstatus(conn) != CONNECTION_OK) + { + if (conn) + PQfinish(conn); + wait_postmaster_connection(connstr); + conn = connectdb(connstr); + } + + res = PQexec(conn, "SELECT pg_is_in_recovery()"); + if (PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) == 1 && *PQgetvalue(res, 0, 0) == 'f') + ispri = true; + else + { + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_VERBOSE, "."); + } + + PQclear(res); + } + + PQfinish(conn); + print_msg(VERBOSITY_VERBOSE, "\n"); +} + +/* + * Wait for postmaster to die + */ +static void +wait_postmaster_shutdown(void) +{ + long pid; + + print_msg(VERBOSITY_VERBOSE, "Waiting for PostgreSQL to shutdown ..."); + + for (;;) + { + if ((pid = get_pgpid()) != 0) + { + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_NORMAL, "."); + } + else + break; + } + + print_msg(VERBOSITY_VERBOSE, "\n"); +} + +static bool +file_exists(const char *path) +{ + struct stat statbuf; + + if (stat(path, &statbuf) != 0) + return false; + + return true; +} + +static bool +is_pg_dir(const char *path) +{ + struct stat statbuf; + char version_file[MAXPGPATH]; + + if (stat(path, &statbuf) != 0) + return false; + + snprintf(version_file, MAXPGPATH, "%s/PG_VERSION", data_dir); + if (stat(version_file, &statbuf) != 0 && errno == ENOENT) + { + return false; + } + + return true; +} + +/* + * copy one file + */ +static void +copy_file(char *fromfile, char *tofile, bool append) +{ + char *buffer; + int srcfd; + int dstfd; + int nbytes; + +#define COPY_BUF_SIZE (8 * BLCKSZ) + + buffer = malloc(COPY_BUF_SIZE); + + /* + * Open the files + */ + srcfd = open(fromfile, O_RDONLY | PG_BINARY, 0); + if (srcfd < 0) + die(_("could not open file \"%s\""), fromfile); + + dstfd = open(tofile, O_RDWR | O_CREAT | (append ? O_APPEND : O_TRUNC) | PG_BINARY, + S_IRUSR | S_IWUSR); + if (dstfd < 0) + die(_("could not create file \"%s\""), tofile); + + /* + * Do the data copying. + */ + for (;;) + { + nbytes = read(srcfd, buffer, COPY_BUF_SIZE); + if (nbytes < 0) + die(_("could not read file \"%s\""), fromfile); + if (nbytes == 0) + break; + errno = 0; + if ((int) write(dstfd, buffer, nbytes) != nbytes) + { + /* if write didn't set errno, assume problem is no disk space */ + if (errno == 0) + errno = ENOSPC; + die(_("could not write to file \"%s\""), tofile); + } + } + + if (close(dstfd)) + die(_("could not close file \"%s\""), tofile); + + /* we don't care about errors here */ + close(srcfd); + + free(buffer); +} + + +static char * +find_other_exec_or_die(const char *argv0, const char *target) +{ + int ret; + char *found_path; + uint32 bin_version; + + found_path = pg_malloc(MAXPGPATH); + + ret = find_other_exec_version(argv0, target, &bin_version, found_path); + + if (ret < 0) + { + char full_path[MAXPGPATH]; + + if (find_my_exec(argv0, full_path) < 0) + strlcpy(full_path, progname, sizeof(full_path)); + + if (ret == -1) + die(_("The program \"%s\" is needed by %s " + "but was not found in the\n" + "same directory as \"%s\".\n" + "Check your installation.\n"), + target, progname, full_path); + else + die(_("The program \"%s\" was found by \"%s\"\n" + "but was not the same version as %s.\n" + "Check your installation.\n"), + target, full_path, progname); + } + else + { + char full_path[MAXPGPATH]; + + if (find_my_exec(argv0, full_path) < 0) + strlcpy(full_path, progname, sizeof(full_path)); + + if (bin_version / 100 != PG_VERSION_NUM / 100) + die(_("The program \"%s\" was found by \"%s\"\n" + "but was not the same version as %s.\n" + "Check your installation.\n"), + target, full_path, progname); + + } + + return found_path; +} + +static bool +postmaster_is_alive(pid_t pid) +{ + /* + * Test to see if the process is still there. Note that we do not + * consider an EPERM failure to mean that the process is still there; + * EPERM must mean that the given PID belongs to some other userid, and + * considering the permissions on $PGDATA, that means it's not the + * postmaster we are after. + * + * Don't believe that our own PID or parent shell's PID is the postmaster, + * either. (Windows hasn't got getppid(), though.) + */ + if (pid == getpid()) + return false; +#ifndef WIN32 + if (pid == getppid()) + return false; +#endif + if (kill(pid, 0) == 0) + return true; + return false; +} + +static long +get_pgpid(void) +{ + FILE *pidf; + long pid; + + pidf = fopen(pid_file, "r"); + if (pidf == NULL) + { + return 0; + } + if (fscanf(pidf, "%ld", &pid) != 1) + { + return 0; + } + fclose(pidf); + return pid; +} + +static char ** +get_database_list(char *databases, int *n_databases) +{ + char *c; + char **result; + int num = 1; + for (c = databases; *c; c++ ) + if (*c == ',') + num++; + *n_databases = num; + result = palloc(num * sizeof(char *)); + num = 0; + /* clone the argument so we don't destroy it with strtok*/ + databases = pstrdup(databases); + c = strtok(databases, ","); + while (c != NULL) + { + result[num] = pstrdup(c); + num++; + c = strtok(NULL,","); + } + pfree(databases); + return result; +} + +static char * +generate_restore_point_name(void) +{ + char *rpn = malloc(NAMEDATALEN); + snprintf(rpn, NAMEDATALEN-1, "spock_create_subscriber_%lx", random()); + return rpn; +} From 344923a93aa09b9b61f28ba2e19f0ae21e7f6284 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Mon, 22 Jun 2026 14:06:38 +0500 Subject: [PATCH 05/13] spock_create_subscriber: drop legacy paths and extension usage --- .../spock_create_subscriber.c | 64 ++++++------------- 1 file changed, 20 insertions(+), 44 deletions(-) diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index 8147c68b..fbf85fdf 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -531,8 +531,6 @@ main(int argc, char **argv) /* Create the extension. */ print_msg(VERBOSITY_VERBOSE, _("Creating spock extension for database %s...\n"), db); - if (PQserverVersion(subscriber_conn) < 90500) - install_extension(subscriber_conn, "spock_origin"); install_extension(subscriber_conn, "spock"); /* @@ -965,16 +963,13 @@ remove_unwanted_data(PGconn *conn) * Remove replication identifiers (9.4 will get them removed by dropping * the extension later as we emulate them there). */ - if (PQserverVersion(conn) >= 90500) + res = PQexec(conn, "SELECT pg_replication_origin_drop(external_id) FROM pg_replication_origin_status;"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) { - res = PQexec(conn, "SELECT pg_replication_origin_drop(external_id) FROM pg_replication_origin_status;"); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - PQclear(res); - die(_("Could not remove existing replication origins: %s\n"), PQerrorMessage(conn)); - } PQclear(res); + die(_("Could not remove existing replication origins: %s\n"), PQerrorMessage(conn)); } + PQclear(res); res = PQexec(conn, "DROP EXTENSION spock CASCADE;"); if (PQresultStatus(res) != PGRES_COMMAND_OK) @@ -994,49 +989,30 @@ initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn) PGresult *res; PQExpBuffer query = createPQExpBuffer(); - if (PQserverVersion(conn) >= 90500) - { - printfPQExpBuffer(query, "SELECT pg_replication_origin_create(%s)", - PQescapeLiteral(conn, origin_name, strlen(origin_name))); - - res = PQexec(conn, query->data); + printfPQExpBuffer(query, "SELECT pg_replication_origin_create(%s)", + PQescapeLiteral(conn, origin_name, strlen(origin_name))); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - die(_("Could not create replication origin \"%s\": status %s: %s\n"), - query->data, - PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); - } - PQclear(res); - - if (remote_lsn) - { - printfPQExpBuffer(query, "SELECT pg_replication_origin_advance(%s, '%s')", - PQescapeLiteral(conn, origin_name, strlen(origin_name)), - remote_lsn); - - res = PQexec(conn, query->data); + res = PQexec(conn, query->data); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - die(_("Could not advance replication origin \"%s\": status %s: %s\n"), - query->data, - PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); - } - PQclear(res); - } + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("Could not create replication origin \"%s\": status %s: %s\n"), + query->data, + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); } - else + PQclear(res); + + if (remote_lsn) { - printfPQExpBuffer(query, "INSERT INTO spock_origin.replication_origin (roident, roname, roremote_lsn) SELECT COALESCE(MAX(roident::int), 0) + 1, %s, %s FROM spock_origin.replication_origin", - PQescapeLiteral(conn, origin_name, strlen(origin_name)), - remote_lsn ? PQescapeLiteral(conn, remote_lsn, strlen(remote_lsn)) : "0"); + printfPQExpBuffer(query, "SELECT pg_replication_origin_advance(%s, '%s')", + PQescapeLiteral(conn, origin_name, strlen(origin_name)), + remote_lsn); res = PQexec(conn, query->data); - if (PQresultStatus(res) != PGRES_COMMAND_OK) + if (PQresultStatus(res) != PGRES_TUPLES_OK) { - die(_("Could not create replication origin \"%s\": status %s: %s\n"), + die(_("Could not advance replication origin \"%s\": status %s: %s\n"), query->data, PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); } From d2ab54b8f7da86aa3781dfdbfbdd8c04798c9583 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Mon, 22 Jun 2026 14:06:38 +0500 Subject: [PATCH 06/13] Added usage document for spock_create_subscriber --- docs/creating_subscriber_nodes.md | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 docs/creating_subscriber_nodes.md diff --git a/docs/creating_subscriber_nodes.md b/docs/creating_subscriber_nodes.md new file mode 100644 index 00000000..bc462239 --- /dev/null +++ b/docs/creating_subscriber_nodes.md @@ -0,0 +1,37 @@ +## Creating a Subscriber Node with pg_basebackup + +Spock supports creating a subscriber node by cloning the provider with [`pg_basebackup`](https://www.postgresql.org/docs/current/app-pgbasebackup.html) and starting it as a Spock subscriber. Use the `spock_create_subscriber` utility (located in the `bin` directory of your pgEdge platform installation) to register the node. + +#### Synopsis: + + `spock_create_subscriber [OPTION]...` + +**Options** + +Specify the following options as needed. + +| Option | Description +|----------|------------- +| `-D`, `--pgdata=DIRECTORY` | The `data` directory to be used for new node. This can be either an empty/non-existing directory, or a directory populated using the `pg_basebackup -X stream` command. +| `--databases` | An optional list of databases to replicate. +| `-n`, `--subscriber-name=NAME` | The name of the newly created subscriber. +| `--subscriber-dsn=CONNSTR` | A connection string to the newly created subscriber. +| `--provider-dsn=CONNSTR` | A connection string to the provider. +| `--replication-sets=SETS` | A comma separated list of replication set names. +| `--apply-delay=DELAY` | The apply delay in seconds (by default 0). +| `--drop-slot-if-exists` | Drop replication slot of conflicting name. +| `-s`, `--stop` | Stop the server once the initialization is done. +| `-v` | Increase logging verbosity. +| `--extra-basebackup-args` | Additional arguments to pass to `pg_basebackup`. Safe options are: `-T`, `-c`, `--xlogdir`/`--waldir` + +**Configuration files overrides** + +You can use the following options to override the location of the configuration files. + +| Option | Description +|----------|------------- +|`--hba-conf` | path to the new `pg_hba.conf` +| `--postgresql-conf` | path to the new `postgresql.conf` +| `--recovery-conf` | path to the template recovery configuration + +Unlike `spock.sub_create`'s other data sync options, this method of cloning ignores replication sets and copies all tables on all databases. However, it's often much faster, especially over high-bandwidth connections. From e08dbddf49c204775c13ae7af03bde7c428d57d4 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Wed, 22 Jul 2026 10:35:51 +0500 Subject: [PATCH 07/13] spock_create_subscriber address review feedback Harden option parsing and cleanup unrelated to bidirectional join: - Reject --apply-delay values that aren't a clean integer instead of silently taking atoi()'s partial parse. - Validate --extra-basebackup-args against shell metacharacters before it is appended to a system() command string; the args are otherwise a command-injection vector. - Free the read_sysid() allocation on the existing-data-dir sysid check. - Fix a missing separator when appending --extra-basebackup-args to the pg_basebackup command line. - Replace sprintf with snprintf when writing postgresql.auto.conf. - Scope the post-sub_create sync_status fixup with a WHERE clause instead of unconditionally rewriting every row. - Document --text-types and a docs formatting nit. --- docs/creating_subscriber_nodes.md | 5 +- .../spock_create_subscriber.c | 63 +++++++++++++++---- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/docs/creating_subscriber_nodes.md b/docs/creating_subscriber_nodes.md index bc462239..64528071 100644 --- a/docs/creating_subscriber_nodes.md +++ b/docs/creating_subscriber_nodes.md @@ -2,7 +2,7 @@ Spock supports creating a subscriber node by cloning the provider with [`pg_basebackup`](https://www.postgresql.org/docs/current/app-pgbasebackup.html) and starting it as a Spock subscriber. Use the `spock_create_subscriber` utility (located in the `bin` directory of your pgEdge platform installation) to register the node. -#### Synopsis: +### Synopsis: `spock_create_subscriber [OPTION]...` @@ -17,12 +17,13 @@ Specify the following options as needed. | `-n`, `--subscriber-name=NAME` | The name of the newly created subscriber. | `--subscriber-dsn=CONNSTR` | A connection string to the newly created subscriber. | `--provider-dsn=CONNSTR` | A connection string to the provider. -| `--replication-sets=SETS` | A comma separated list of replication set names. +| `--replication-sets=SETS` | A comma-separated list of replication set names. | `--apply-delay=DELAY` | The apply delay in seconds (by default 0). | `--drop-slot-if-exists` | Drop replication slot of conflicting name. | `-s`, `--stop` | Stop the server once the initialization is done. | `-v` | Increase logging verbosity. | `--extra-basebackup-args` | Additional arguments to pass to `pg_basebackup`. Safe options are: `-T`, `-c`, `--xlogdir`/`--waldir` +| `--text-types` | Transfer all column values as text rather than binary during initial sync. Use this when provider and subscriber differ in endianness or type representation. **Configuration files overrides** diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index fbf85fdf..7f7b76d5 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -90,6 +90,7 @@ static void print_msg(VerbosityLevelEnum level, const char *fmt,...) pg_attribute_printf(2, 3); static int run_pg_ctl(const char *arg); +static void validate_extra_basebackup_args(const char *args); static void run_basebackup(const char *provider_connstr, const char *data_dir, const char *extra_basebackup_args); static void wait_postmaster_connection(const char *connstr); @@ -281,13 +282,19 @@ main(int argc, char **argv) drop_slot_if_exists = true; break; case 8: - apply_delay = atoi(optarg); + { + char *endptr; + apply_delay = (int) strtol(optarg, &endptr, 10); + if (*endptr != '\0' || endptr == optarg) + die(_("--apply-delay requires an integer value\n")); + } break; case 9: databases = pg_strdup(optarg); break; case 10: extra_basebackup_args = pg_strdup(optarg); + validate_extra_basebackup_args(extra_basebackup_args); break; case 11: force_text_transfer = true; @@ -407,9 +414,14 @@ main(int argc, char **argv) { use_existing_data_dir = check_data_dir(data_dir, remote_info); - if (use_existing_data_dir && - strcmp(remote_info->sysid, read_sysid(data_dir)) != 0) - die(_("Subscriber data directory is not basebackup of remote node.\n")); + if (use_existing_data_dir) + { + char *local_sysid = read_sysid(data_dir); + bool mismatch = strcmp(remote_info->sysid, local_sysid) != 0; + free(local_sysid); + if (mismatch) + die(_("Subscriber data directory is not basebackup of remote node.\n")); + } } /* @@ -597,6 +609,9 @@ usage(void) printf(_(" -v increase logging verbosity\n")); printf(_(" --extra-basebackup-args additional arguments to pass to pg_basebackup.\n")); printf(_(" Safe options: -T, -c, --xlogdir/--waldir\n")); + printf(_(" --text-types transfer column values as text rather than binary\n")); + printf(_(" (use when provider and subscriber differ in type\n")); + printf(_(" representation or endianness)\n")); printf(_("\nConfiguration files override:\n")); printf(_(" --hba-conf path to the new pg_hba.conf\n")); printf(_(" --postgresql-conf path to the new postgresql.conf\n")); @@ -682,6 +697,27 @@ run_pg_ctl(const char *arg) } +/* + * Reject --extra-basebackup-args values containing shell control + * characters. The args are appended to a system() command string, so + * semicolons, pipes, backticks, and similar metacharacters would allow + * arbitrary command injection. + */ +static void +validate_extra_basebackup_args(const char *args) +{ + const char *p; + + for (p = args; *p; p++) + { + if (*p == ';' || *p == '|' || *p == '&' || *p == '`' || + *p == '$' || *p == '(' || *p == ')' || + *p == '<' || *p == '>' || *p == '{' || *p == '}' || + *p == '\n' || *p == '\r') + die(_("--extra-basebackup-args contains unsafe shell characters\n")); + } +} + /* * Run pg_basebackup to create the copy of the origin node. */ @@ -700,7 +736,7 @@ run_basebackup(const char *provider_connstr, const char *data_dir, appendPQExpBuffer(cmd, " -v"); if (extra_basebackup_args != NULL) - appendPQExpBuffer(cmd, "%s", extra_basebackup_args); + appendPQExpBuffer(cmd, " %s", extra_basebackup_args); print_msg(VERBOSITY_DEBUG, _("Running pg_basebackup: %s.\n"), cmd->data); ret = system(cmd->data); @@ -1097,8 +1133,8 @@ spock_subscribe(PGconn *conn, char *subscriber_name, char *subscriber_dsn, } PQclear(res); - /* TODO */ - res = PQexec(conn, "UPDATE spock.local_sync_status SET sync_status = 'r'"); + res = PQexec(conn, "UPDATE spock.local_sync_status SET sync_status = 'r'" + " WHERE sync_status != 'r'"); if (PQresultStatus(res) != PGRES_COMMAND_OK) { die(_("Could not update subscription, status %s: %s\n"), @@ -1317,7 +1353,7 @@ WriteRecoveryConf(PQExpBuffer contents) char filename[MAXPGPATH]; FILE *cf; - sprintf(filename, "%s/postgresql.auto.conf", data_dir); + snprintf(filename, sizeof(filename), "%s/postgresql.auto.conf", data_dir); cf = fopen(filename, "a"); if (cf == NULL) @@ -1334,7 +1370,7 @@ WriteRecoveryConf(PQExpBuffer contents) fclose(cf); { - sprintf(filename, "%s/standby.signal", data_dir); + snprintf(filename, sizeof(filename), "%s/standby.signal", data_dir); cf = fopen(filename, "w"); if (cf == NULL) { @@ -1353,7 +1389,7 @@ CopyConfFile(char *fromfile, char *tofile, bool append) { char filename[MAXPGPATH]; - sprintf(filename, "%s/%s", data_dir, tofile); + snprintf(filename, sizeof(filename), "%s/%s", data_dir, tofile); print_msg(VERBOSITY_DEBUG, _("Copying \"%s\" to \"%s\".\n"), fromfile, filename); @@ -1560,7 +1596,7 @@ is_pg_dir(const char *path) if (stat(path, &statbuf) != 0) return false; - snprintf(version_file, MAXPGPATH, "%s/PG_VERSION", data_dir); + snprintf(version_file, MAXPGPATH, "%s/PG_VERSION", path); if (stat(version_file, &statbuf) != 0 && errno == ENOENT) { return false; @@ -1711,6 +1747,7 @@ get_pgpid(void) } if (fscanf(pidf, "%ld", &pid) != 1) { + fclose(pidf); return 0; } fclose(pidf); @@ -1746,6 +1783,8 @@ static char * generate_restore_point_name(void) { char *rpn = malloc(NAMEDATALEN); - snprintf(rpn, NAMEDATALEN-1, "spock_create_subscriber_%lx", random()); + if (rpn == NULL) + die(_("out of memory\n")); + snprintf(rpn, NAMEDATALEN, "spock_create_subscriber_%lx", random()); return rpn; } From 428eb88c038452a858fa7541a7c40c761e29dd30 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Thu, 9 Jul 2026 15:43:36 +0500 Subject: [PATCH 08/13] spock_create_subscriber: add bidirectional join plumbing Add --bidirectional, --stall-timeout, --max-wait, and --cleanup options to support the bidirectional node-join procedure defined in the SPOC-601 design. In --bidirectional mode the tool connects to the source cluster, discovers all peer nodes via spock.subscription/node/node_interface, verifies preconditions (Spock >= 6.0.0, track_commit_timestamp on on all nodes, no pending DDL, full-mesh topology, per-peer connectivity), then writes a JSON manifest to /spock_bidirectional_manifest.json and exits. The manifest records peer names, DSNs, slot names, and sub names so that later phases can resume idempotently. In --cleanup mode the tool reads the manifest and idempotently removes any partial state left by a prior attempt: drops replication slots on the source and each peer, drops reverse subscriptions, and removes the manifest file. Connectivity failures during cleanup are logged as warnings rather than being fatal. No replication behavior is changed in this commit; the subscriber DSN is not required in --bidirectional mode since the subscriber database does not exist yet at this stage. --- .../spock_create_subscriber.c | 779 +++++++++++++++++- 1 file changed, 773 insertions(+), 6 deletions(-) diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index 7f7b76d5..bff53736 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -52,6 +52,8 @@ #include "access/timeline.h" #include "access/xlog_internal.h" #include "catalog/pg_control.h" +#include "common/jsonapi.h" +#include "mb/pg_wchar.h" #include "spock_fe.h" @@ -65,6 +67,30 @@ typedef struct RemoteInfo { char *replication_sets; } RemoteInfo; +typedef struct PeerNodeInfo +{ + char *node_name; + char *dsn; + char *slot_name; /* from spock.spock_gen_slot_name() */ + char *sub_name; /* "sub__" */ + bool disabled_sub_created; + bool slot_created; + bool reverse_sub_created; +} PeerNodeInfo; + +typedef struct BidirectionalState +{ + bool enabled; + int num_peers; + PeerNodeInfo *peers; + int stall_timeout; /* default 600s */ + int max_wait; /* default 0 = unbounded */ + char *source_slot_name; + char *source_origin_name; + bool cleanup_mode; + char *manifest_path; +} BidirectionalState; + typedef enum { VERBOSITY_NORMAL, VERBOSITY_VERBOSE, @@ -141,6 +167,20 @@ static long get_pgpid(void); static char **get_database_list(char *databases, int *n_databases); static char *generate_restore_point_name(void); +static int discover_peer_nodes(PGconn *source_conn, const char *source_node_name, + const char *subscriber_name, const char *dbname, + PeerNodeInfo **peers_out); +static void check_preconditions(PGconn *source_conn, PeerNodeInfo *peers, int num_peers); +static void write_manifest(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn); +static bool read_manifest(const char *manifest_path, BidirectionalState *state, + char **subscriber_name_out, char **dbname_out, + char **source_dsn_out); +static void cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn, + bool force_rm_datadir); +static void append_json_string(PQExpBuffer buf, const char *str); + static PGconn * connectdb(const char *connstr) { @@ -161,6 +201,654 @@ void signal_handler(int sig) } } +/* + * Append str to buf with JSON string escaping applied, without the + * surrounding quotes (the caller supplies those). Control characters + * below 0x20 are emitted as \uXXXX. jsonapi.h provides a JSON parser but + * no encoder, so this is a small local encoder in the same style as + * src/bin/pg_combinebackup/write_manifest.c. + */ +static void +append_json_string(PQExpBuffer buf, const char *str) +{ + const char *p; + + for (p = str; *p; p++) + { + switch (*p) + { + case '\b': appendPQExpBufferStr(buf, "\\b"); break; + case '\f': appendPQExpBufferStr(buf, "\\f"); break; + case '\n': appendPQExpBufferStr(buf, "\\n"); break; + case '\r': appendPQExpBufferStr(buf, "\\r"); break; + case '\t': appendPQExpBufferStr(buf, "\\t"); break; + case '"': appendPQExpBufferStr(buf, "\\\""); break; + case '\\': appendPQExpBufferStr(buf, "\\\\"); break; + default: + if ((unsigned char) *p < 0x20) + appendPQExpBuffer(buf, "\\u%04x", (unsigned char) *p); + else + appendPQExpBufferChar(buf, *p); + break; + } + } +} + +/* + * Query the source for all peer nodes in the multi-master cluster. + * Returns the peer count; *peers_out is set to a pg_malloc0'd array. For + * each peer, sub_name is derived as "sub__" + * and slot_name is obtained via spock.spock_gen_slot_name() on the source. + */ +static int +discover_peer_nodes(PGconn *source_conn, const char *source_node_name, + const char *subscriber_name, const char *dbname, + PeerNodeInfo **peers_out) +{ + static const char *discover_sql = + "SELECT DISTINCT n.node_name, ni.if_dsn" + " FROM spock.subscription s" + " JOIN spock.node n ON s.sub_origin = n.node_id" + " JOIN spock.node_interface ni ON n.node_id = ni.if_nodeid" + " WHERE n.node_name != $1" + " ORDER BY n.node_name"; + const char *paramValues[3]; + PGresult *res; + PGresult *slot_res; + int npeers; + PeerNodeInfo *peers; + int i; + + paramValues[0] = source_node_name; + res = PQexecParams(source_conn, discover_sql, + 1, NULL, paramValues, NULL, NULL, 0); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not discover peer nodes: %s"), + PQerrorMessage(source_conn)); + + npeers = PQntuples(res); + if (npeers == 0) + { + PQclear(res); + die(_("no peer nodes found; source does not appear to be part of a " + "multi-master cluster")); + } + + peers = pg_malloc0(npeers * sizeof(PeerNodeInfo)); + + for (i = 0; i < npeers; i++) + { + PQExpBuffer sub_name_buf = createPQExpBuffer(); + + peers[i].node_name = pg_strdup(PQgetvalue(res, i, 0)); + peers[i].dsn = pg_strdup(PQgetvalue(res, i, 1)); + + appendPQExpBuffer(sub_name_buf, "sub_%s_%s", + subscriber_name, peers[i].node_name); + peers[i].sub_name = pg_strdup(sub_name_buf->data); + destroyPQExpBuffer(sub_name_buf); + + paramValues[0] = dbname; + paramValues[1] = peers[i].node_name; + paramValues[2] = peers[i].sub_name; + slot_res = PQexecParams(source_conn, + "SELECT spock.spock_gen_slot_name" + "($1::name, $2::name, $3::name)", + 3, NULL, paramValues, NULL, NULL, 0); + if (PQresultStatus(slot_res) != PGRES_TUPLES_OK) + die(_("could not generate slot name for peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(source_conn)); + + peers[i].slot_name = pg_strdup(PQgetvalue(slot_res, 0, 0)); + PQclear(slot_res); + + print_msg(VERBOSITY_VERBOSE, + _(" discovered peer: %s (slot: %s)\n"), + peers[i].node_name, peers[i].slot_name); + } + + PQclear(res); + *peers_out = peers; + return npeers; +} + +/* + * Verify that the source cluster and all peers meet the requirements for + * a bidirectional join: Spock >= 6.0.0, track_commit_timestamp on, no + * pending DDL, full-mesh topology, and peer connectivity. + */ +static void +check_preconditions(PGconn *source_conn, PeerNodeInfo *peers, int num_peers) +{ + PGresult *res; + int i; + + /* Spock version gate: require >= 6.0.0 */ + res = PQexec(source_conn, + "SELECT extversion FROM pg_extension WHERE extname = 'spock'"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not query Spock extension version: %s"), + PQerrorMessage(source_conn)); + if (PQntuples(res) == 0) + die(_("Spock extension is not installed on the source node")); + { + const char *ver = PQgetvalue(res, 0, 0); + int major = 0; + + if (sscanf(ver, "%d.", &major) < 1) + die(_("could not parse Spock version \"%s\""), ver); + if (major < 6) + die(_("Spock version %s on source is too old for bidirectional " + "join; require >= 6.0.0"), ver); + } + PQclear(res); + + /* track_commit_timestamp must be on at the source */ + res = PQexec(source_conn, "SHOW track_commit_timestamp"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not check track_commit_timestamp: %s"), + PQerrorMessage(source_conn)); + if (strcmp(PQgetvalue(res, 0, 0), "on") != 0) + die(_("track_commit_timestamp must be on for bidirectional join (source)")); + PQclear(res); + + /* No pending DDL in spock.queue */ + res = PQexec(source_conn, "SELECT COUNT(*) FROM spock.queue"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not check spock.queue: %s"), + PQerrorMessage(source_conn)); + if (strcmp(PQgetvalue(res, 0, 0), "0") != 0) + die(_("pending DDL in spock.queue; wait for replication to drain " + "before joining")); + PQclear(res); + + /* Full-mesh assertion: subscriptions on source == num_peers */ + res = PQexec(source_conn, "SELECT COUNT(*) FROM spock.subscription"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not count subscriptions: %s"), + PQerrorMessage(source_conn)); + { + int sub_count = atoi(PQgetvalue(res, 0, 0)); + + if (sub_count != num_peers) + die(_("source node has %d active subscription(s) but %d peer(s) " + "discovered; partial-mesh topologies are not supported"), + sub_count, num_peers); + } + PQclear(res); + + /* + * Per-peer: connectivity and track_commit_timestamp. + * + * Spock version is not checked on peers here; peer version checking is + * deferred to the subscription-setup phase. + */ + for (i = 0; i < num_peers; i++) + { + PGconn *peer_conn; + + print_msg(VERBOSITY_VERBOSE, + _(" checking peer %s ...\n"), peers[i].node_name); + + peer_conn = PQconnectdb(peers[i].dsn); + if (PQstatus(peer_conn) != CONNECTION_OK) + die(_("cannot connect to peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(peer_conn)); + + res = PQexec(peer_conn, "SHOW track_commit_timestamp"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + PQfinish(peer_conn); + die(_("could not check track_commit_timestamp on peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(peer_conn)); + } + if (strcmp(PQgetvalue(res, 0, 0), "on") != 0) + { + PQclear(res); + PQfinish(peer_conn); + die(_("track_commit_timestamp must be on for bidirectional join " + "(peer \"%s\")"), peers[i].node_name); + } + PQclear(res); + PQfinish(peer_conn); + } + + print_msg(VERBOSITY_NORMAL, _("Preconditions verified.\n")); +} + +/* + * Write the bidirectional state manifest to state->manifest_path + * atomically (write to .tmp, then rename). The manifest is a simple + * hand-formatted JSON file, with string values escaped by + * append_json_string(). + */ +static void +write_manifest(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn) +{ + PQExpBuffer buf = createPQExpBuffer(); + char tmp_path[MAXPGPATH]; + FILE *f; + int i; + + snprintf(tmp_path, MAXPGPATH, "%s.tmp", state->manifest_path); + + appendPQExpBufferStr(buf, "{\n"); + appendPQExpBufferStr(buf, " \"version\": 1,\n"); + + appendPQExpBufferStr(buf, " \"subscriber_name\": \""); + append_json_string(buf, subscriber_name); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"dbname\": \""); + append_json_string(buf, dbname); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"source_dsn\": \""); + append_json_string(buf, source_dsn); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"source_slot_name\": \""); + if (state->source_slot_name) + append_json_string(buf, state->source_slot_name); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"source_origin_name\": \""); + if (state->source_origin_name) + append_json_string(buf, state->source_origin_name); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"peers\": [\n"); + for (i = 0; i < state->num_peers; i++) + { + PeerNodeInfo *p = &state->peers[i]; + bool last = (i == state->num_peers - 1); + + appendPQExpBufferStr(buf, " {\n"); + + appendPQExpBufferStr(buf, " \"node_name\": \""); + append_json_string(buf, p->node_name); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"peer_dsn\": \""); + append_json_string(buf, p->dsn); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"sub_name_on_n3\": \""); + append_json_string(buf, p->sub_name); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"peer_slot_name\": \""); + append_json_string(buf, p->slot_name); + appendPQExpBufferStr(buf, "\"\n"); + + appendPQExpBufferStr(buf, last ? " }\n" : " },\n"); + } + appendPQExpBufferStr(buf, " ]\n"); + appendPQExpBufferStr(buf, "}\n"); + + f = fopen(tmp_path, "w"); + if (f == NULL) + die(_("could not create manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + + if (fwrite(buf->data, 1, buf->len, f) != buf->len) + { + fclose(f); + unlink(tmp_path); + die(_("could not write manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + if (fclose(f) != 0) + { + unlink(tmp_path); + die(_("could not close manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + if (rename(tmp_path, state->manifest_path) != 0) + die(_("could not rename manifest to \"%s\": %s"), + state->manifest_path, strerror(errno)); + + destroyPQExpBuffer(buf); +} + +/* + * Semantic-action state for read_manifest(). Passed as void *semstate to all + * pg_parse_json callbacks; tracks nesting depth and accumulates field values. + */ +typedef struct ManifestParseState +{ + /* outputs written by scalar callback */ + char **subscriber_name_out; + char **dbname_out; + char **source_dsn_out; + BidirectionalState *bidir; + + /* parser context */ + int depth; /* object/array nesting depth */ + bool in_peers; /* inside the top-level "peers" array */ + bool in_peer_obj; /* inside one peer object */ + char *cur_field; /* current object field name (owned by us) */ + + /* per-peer accumulator, flushed on each object_end inside peers */ + char *peer_node_name; + char *peer_dsn; + char *peer_sub_name; + char *peer_slot_name; + int peer_capacity; +} ManifestParseState; + +static JsonParseErrorType +manifest_object_start(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + s->depth++; + if (s->in_peers && s->depth == 3) + s->in_peer_obj = true; + return JSON_SUCCESS; +} + +static JsonParseErrorType +manifest_object_end(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + if (s->in_peer_obj && s->depth == 3) + { + int i = s->bidir->num_peers; + + if (i >= s->peer_capacity) + { + s->peer_capacity = (s->peer_capacity > 0) ? s->peer_capacity * 2 : 4; + s->bidir->peers = pg_realloc(s->bidir->peers, + s->peer_capacity * sizeof(PeerNodeInfo)); + } + s->bidir->peers[i].node_name = s->peer_node_name; + s->bidir->peers[i].dsn = s->peer_dsn; + s->bidir->peers[i].sub_name = s->peer_sub_name; + s->bidir->peers[i].slot_name = s->peer_slot_name; + s->bidir->num_peers++; + s->peer_node_name = s->peer_dsn = s->peer_sub_name = s->peer_slot_name = NULL; + s->in_peer_obj = false; + } + s->depth--; + return JSON_SUCCESS; +} + +static JsonParseErrorType +manifest_array_start(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + s->depth++; + if (s->depth == 2 && s->cur_field != NULL && + strcmp(s->cur_field, "peers") == 0) + s->in_peers = true; + return JSON_SUCCESS; +} + +static JsonParseErrorType +manifest_array_end(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + if (s->in_peers && s->depth == 2) + s->in_peers = false; + s->depth--; + return JSON_SUCCESS; +} + +static JsonParseErrorType +manifest_ofield_start(void *st, char *fname, bool isnull) +{ + ManifestParseState *s = (ManifestParseState *) st; + + (void) isnull; + pg_free(s->cur_field); + s->cur_field = pg_strdup(fname); + pg_free(fname); /* callback owns the token */ + return JSON_SUCCESS; +} + +static JsonParseErrorType +manifest_scalar(void *st, char *token, JsonTokenType tokentype) +{ + ManifestParseState *s = (ManifestParseState *) st; + + if (s->cur_field == NULL || tokentype != JSON_TOKEN_STRING) + { + pg_free(token); + return JSON_SUCCESS; + } + + if (!s->in_peer_obj) + { + /* top-level scalar fields */ + if (strcmp(s->cur_field, "subscriber_name") == 0) + *s->subscriber_name_out = token; + else if (strcmp(s->cur_field, "dbname") == 0) + *s->dbname_out = token; + else if (strcmp(s->cur_field, "source_dsn") == 0) + *s->source_dsn_out = token; + else if (strcmp(s->cur_field, "source_slot_name") == 0) + s->bidir->source_slot_name = token; + else if (strcmp(s->cur_field, "source_origin_name") == 0) + s->bidir->source_origin_name = token; + else + pg_free(token); + } + else + { + /* per-peer scalar fields */ + if (strcmp(s->cur_field, "node_name") == 0) + s->peer_node_name = token; + else if (strcmp(s->cur_field, "peer_dsn") == 0) + s->peer_dsn = token; + else if (strcmp(s->cur_field, "sub_name_on_n3") == 0) + s->peer_sub_name = token; + else if (strcmp(s->cur_field, "peer_slot_name") == 0) + s->peer_slot_name = token; + else + pg_free(token); + } + return JSON_SUCCESS; +} + +/* + * Read the bidirectional manifest from manifest_path. Returns false if + * the file does not exist (nothing to clean up); dies if it exists but + * cannot be read or is malformed. On success, sets *subscriber_name_out, + * *dbname_out, *source_dsn_out, and populates state->peers[]. + * + * Uses pg_parse_json (common/jsonapi.h) for JSON lexing, so string + * quoting, escape sequences, and nesting are handled correctly. + */ +static bool +read_manifest(const char *manifest_path, BidirectionalState *state, + char **subscriber_name_out, char **dbname_out, + char **source_dsn_out) +{ + struct stat st; + char *content; + FILE *f; + JsonLexContext *lex; + JsonSemAction sem; + ManifestParseState pstate; + JsonParseErrorType result; + + if (stat(manifest_path, &st) != 0) + return false; + + content = pg_malloc(st.st_size + 1); + f = fopen(manifest_path, "r"); + if (f == NULL) + die(_("could not open manifest file \"%s\": %s"), + manifest_path, strerror(errno)); + + if ((size_t) fread(content, 1, st.st_size, f) != (size_t) st.st_size) + { + fclose(f); + die(_("could not read manifest file \"%s\": %s"), + manifest_path, strerror(errno)); + } + content[st.st_size] = '\0'; + fclose(f); + + memset(&pstate, 0, sizeof(pstate)); + pstate.subscriber_name_out = subscriber_name_out; + pstate.dbname_out = dbname_out; + pstate.source_dsn_out = source_dsn_out; + pstate.bidir = state; + + memset(&sem, 0, sizeof(sem)); + sem.semstate = &pstate; + sem.object_start = manifest_object_start; + sem.object_end = manifest_object_end; + sem.array_start = manifest_array_start; + sem.array_end = manifest_array_end; + sem.object_field_start = manifest_ofield_start; + sem.scalar = manifest_scalar; + + lex = makeJsonLexContextCstringLen(NULL, content, st.st_size, + PG_UTF8, true); + result = pg_parse_json(lex, &sem); + pg_free(content); + pg_free(pstate.cur_field); + + if (result != JSON_SUCCESS) + { + char *detail = json_errdetail(result, lex); + + freeJsonLexContext(lex); + die(_("manifest file \"%s\" is malformed: %s"), manifest_path, detail); + } + freeJsonLexContext(lex); + + if (!*subscriber_name_out || !*dbname_out || !*source_dsn_out) + die(_("manifest file \"%s\" is malformed or missing required fields"), + manifest_path); + + return true; +} + +/* + * Idempotently remove bidirectional join state from all reachable nodes. + * Connects to the source and each peer, drops replication slots and + * reverse subscriptions created during a previous join attempt. All + * operations are best-effort: connectivity failures are logged as + * warnings rather than being fatal. + */ +static void +cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn, + bool force_rm_datadir) +{ + PGconn *source_conn; + PGresult *res; + PQExpBuffer query = createPQExpBuffer(); + int i; + + print_msg(VERBOSITY_NORMAL, + _("Cleaning up partial bidirectional join state ...\n")); + + source_conn = PQconnectdb(source_dsn); + if (PQstatus(source_conn) != CONNECTION_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: cannot connect to source node; skipping " + "source-side cleanup: %s\n"), + PQerrorMessage(source_conn)); + PQfinish(source_conn); + source_conn = NULL; + } + + /* Drop source replication slot if it was created */ + if (source_conn && state->source_slot_name && state->source_slot_name[0]) + { + printfPQExpBuffer(query, + "SELECT pg_drop_replication_slot(slot_name)" + " FROM pg_replication_slots" + " WHERE slot_name = '%s'", + state->source_slot_name); + res = PQexec(source_conn, query->data); + if (PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) > 0) + print_msg(VERBOSITY_NORMAL, + _(" dropped source slot %s\n"), + state->source_slot_name); + PQclear(res); + } + + /* Per-peer: drop slot and any reverse subscription */ + for (i = 0; i < state->num_peers; i++) + { + PeerNodeInfo *peer = &state->peers[i]; + PGconn *peer_conn; + char reverse_sub[NAMEDATALEN]; + + if (!peer->dsn || !peer->dsn[0]) + continue; + + peer_conn = PQconnectdb(peer->dsn); + if (PQstatus(peer_conn) != CONNECTION_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: cannot connect to peer \"%s\"; skipping " + "peer-side cleanup: %s\n"), + peer->node_name, PQerrorMessage(peer_conn)); + PQfinish(peer_conn); + continue; + } + + if (peer->slot_name && peer->slot_name[0]) + { + printfPQExpBuffer(query, + "SELECT pg_drop_replication_slot(slot_name)" + " FROM pg_replication_slots" + " WHERE slot_name = '%s'", + peer->slot_name); + res = PQexec(peer_conn, query->data); + if (PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) > 0) + print_msg(VERBOSITY_NORMAL, + _(" dropped peer slot %s on %s\n"), + peer->slot_name, peer->node_name); + PQclear(res); + } + + /* + * Drop the reverse subscription (peer -> new subscriber) if it was + * created during a previous attempt. The sub_drop second argument + * is ifexists=true. + */ + snprintf(reverse_sub, sizeof(reverse_sub), "sub_%s_%s", + peer->node_name, subscriber_name); + printfPQExpBuffer(query, + "SELECT spock.sub_drop('%s', true)", + reverse_sub); + res = PQexec(peer_conn, query->data); + PQclear(res); + + PQfinish(peer_conn); + print_msg(VERBOSITY_NORMAL, + _(" cleaned up peer %s\n"), peer->node_name); + } + + if (source_conn) + PQfinish(source_conn); + + destroyPQExpBuffer(query); + + if (state->manifest_path && state->manifest_path[0]) + { + unlink(state->manifest_path); + print_msg(VERBOSITY_NORMAL, + _(" removed manifest %s\n"), state->manifest_path); + } + + print_msg(VERBOSITY_NORMAL, _("Cleanup complete.\n")); +} + int main(int argc, char **argv) @@ -194,6 +882,8 @@ main(int argc, char **argv) logfd; char *restore_point_name = NULL; char *extra_basebackup_args = NULL; + BidirectionalState bidir = {0}; + char bidir_manifest_path[MAXPGPATH] = {0}; static struct option long_options[] = { {"subscriber-name", required_argument, NULL, 'n'}, @@ -210,6 +900,10 @@ main(int argc, char **argv) {"databases", required_argument, NULL, 9}, {"extra-basebackup-args", required_argument, NULL, 10}, {"text-types", no_argument, NULL, 11}, + {"bidirectional", no_argument, NULL, 12}, + {"stall-timeout", required_argument, NULL, 13}, + {"max-wait", required_argument, NULL, 14}, + {"cleanup", no_argument, NULL, 15}, {NULL, 0, NULL, 0} }; @@ -299,6 +993,22 @@ main(int argc, char **argv) case 11: force_text_transfer = true; break; + case 12: + bidir.enabled = true; + break; + case 13: + bidir.stall_timeout = atoi(optarg); + if (bidir.stall_timeout <= 0) + die(_("--stall-timeout must be a positive integer")); + break; + case 14: + bidir.max_wait = atoi(optarg); + if (bidir.max_wait < 0) + die(_("--max-wait must be a non-negative integer")); + break; + case 15: + bidir.cleanup_mode = true; + break; default: fprintf(stderr, _("Unknown option\n")); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); @@ -316,16 +1026,20 @@ main(int argc, char **argv) fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } - else if (subscriber_name == NULL) + else if (subscriber_name == NULL && !bidir.cleanup_mode) { fprintf(stderr, _("No subscriber name specified\n")); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } - if (!base_prov_connstr || !strlen(base_prov_connstr)) + if (bidir.cleanup_mode && !bidir.enabled) + die(_("--cleanup requires --bidirectional.\n")); + + if (!bidir.cleanup_mode && (!base_prov_connstr || !strlen(base_prov_connstr))) die(_("Provider connection string must be specified.\n")); - if (!base_sub_connstr || !strlen(base_sub_connstr)) + if (!bidir.enabled && !bidir.cleanup_mode && + (!base_sub_connstr || !strlen(base_sub_connstr))) die(_("Subscriber connection string must be specified.\n")); if (apply_delay < 0) @@ -337,6 +1051,33 @@ main(int argc, char **argv) if (!replication_sets || !strlen(replication_sets)) replication_sets = "default,default_insert_only,ddl_sql"; + /* Build the manifest path from --pgdata */ + if (bidir.enabled || bidir.cleanup_mode) + { + snprintf(bidir_manifest_path, MAXPGPATH, + "%s/spock_bidirectional_manifest.json", data_dir); + bidir.manifest_path = bidir_manifest_path; + if (bidir.stall_timeout == 0) + bidir.stall_timeout = 600; + } + + /* --cleanup: read manifest, remove partial state, exit */ + if (bidir.cleanup_mode) + { + char *sub_name = NULL; + char *db = NULL; + char *src_dsn = NULL; + + if (!read_manifest(bidir.manifest_path, &bidir, &sub_name, &db, &src_dsn)) + { + fprintf(stderr, _("No manifest found at %s; nothing to clean up.\n"), + bidir.manifest_path); + exit(0); + } + cleanup_partial_state(&bidir, sub_name, db, src_dsn, false); + exit(0); + } + /* Init random numbers used for slot suffixes, etc */ srand(time(NULL)); @@ -372,9 +1113,12 @@ main(int argc, char **argv) if (!prov_connstr || !strlen(prov_connstr)) die(_("Provider connection string is not valid.\n")); - sub_connstr = get_connstr(base_sub_connstr, db); - if (!sub_connstr || !strlen(sub_connstr)) - die(_("Subscriber connection string is not valid.\n")); + if (!bidir.enabled) + { + sub_connstr = get_connstr(base_sub_connstr, db); + if (!sub_connstr || !strlen(sub_connstr)) + die(_("Subscriber connection string is not valid.\n")); + } } /* @@ -408,6 +1152,29 @@ main(int argc, char **argv) provider_conn = connectdb(prov_connstr); remote_info = get_remote_info(provider_conn); + /* + * --bidirectional: discover peers, verify preconditions, write the + * manifest, then exit. The rest of the join resumes from this + * manifest once the physical backup has been taken and the + * subscriber is running. + */ + if (bidir.enabled) + { + bidir.num_peers = discover_peer_nodes(provider_conn, + remote_info->node_name, + subscriber_name, db, + &bidir.peers); + check_preconditions(provider_conn, bidir.peers, bidir.num_peers); + write_manifest(&bidir, subscriber_name, db, base_prov_connstr); + print_msg(VERBOSITY_NORMAL, + _("Bidirectional plumbing complete: %d peer(s) discovered, " + "preconditions OK, manifest written to %s.\n"), + bidir.num_peers, bidir.manifest_path); + PQfinish(provider_conn); + provider_conn = NULL; + exit(0); + } + /* only need to do this piece once */ if (dbnum == 0) From 0eb427498664b8f1234b6d4ced3ecd2c81b5fe86 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Mon, 13 Jul 2026 19:00:47 +0500 Subject: [PATCH 09/13] spock_create_subscriber: add TAP test for --bidirectional plumbing --- tests/tap/schedule | 2 + tests/tap/t/047_bidir_plumbing.pl | 165 ++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 tests/tap/t/047_bidir_plumbing.pl diff --git a/tests/tap/schedule b/tests/tap/schedule index 48cf5ae4..f195915c 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -55,6 +55,7 @@ test: 032_lolor_largeobject_repset test: 033_zodan_lolor_add_node test: 034_reserved_object_ddl test: 044_apply_change_logging +test: 047_bidir_plumbing # Upgrade schema match test (builds from source, slow): #test: 018_upgrade_schema_match # @@ -62,3 +63,4 @@ test: 044_apply_change_logging # Regression tests test: 103_manager_worker_dboid_race test: 105_sub_disable_retransmit_after_disconnect + diff --git a/tests/tap/t/047_bidir_plumbing.pl b/tests/tap/t/047_bidir_plumbing.pl new file mode 100644 index 00000000..64d19f12 --- /dev/null +++ b/tests/tap/t/047_bidir_plumbing.pl @@ -0,0 +1,165 @@ +#!/usr/bin/perl +# ============================================================================= +# Test: 047_bidir_plumbing.pl - spock_create_subscriber --bidirectional +# ============================================================================= +# Validates the plumbing phase of the SPOC-601 bidirectional node-join +# procedure. The test does NOT start a third PostgreSQL instance; it only +# exercises the utility's plumbing phase against an existing 2-node cluster: +# +# --bidirectional discover peers, check preconditions, write manifest +# --cleanup idempotently remove partial state / manifest +# +# Topology: +# n1 <-> n2 (full bidirectional Spock subscriptions, track_commit_timestamp=on) +# +# The utility is run with --pgdata pointing at a plain temp directory (no PG +# cluster) that exists solely to hold the manifest file. +# +# Test count breakdown: +# 1 binary found +# 1 temp pgdata created +# 5 create_cluster(2) [2 pg_isready + 2 spock checks + 1 pass] +# 1 cross_wire n1<->n2 +# 1 --bidirectional exits 0 +# 1 manifest file written +# 1 manifest: version 1 +# 1 manifest: subscriber_name n3 +# 1 manifest: dbname regression +# 1 manifest: source_dsn present +# 1 manifest: peer n2 listed +# 1 manifest: peer_slot_name present +# 1 --cleanup exits 0 +# 1 manifest removed +# 1 --cleanup with no manifest exits 0 (idempotent) +# 1 destroy_cluster +# --- +# 20 total +# ============================================================================= + +use strict; +use warnings; +use Test::More tests => 20; +use File::Path qw(remove_tree make_path); +use lib '.'; +use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail + command_ok system_maybe get_test_config scalar_query psql_or_bail); + +# ============================================================================= +# Locate spock_create_subscriber binary +# ============================================================================= +my $SCS_BIN; +for my $dir (split(':', $ENV{PATH} // '')) { + my $c = "$dir/spock_create_subscriber"; + if (-x $c) { $SCS_BIN = $c; last; } +} +unless (defined $SCS_BIN) { + # Fall back to the build tree (CWD is tests/tap/ during make check_prove) + my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; + $SCS_BIN = $bt if -x $bt; +} +BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") + unless defined $SCS_BIN; +pass("spock_create_subscriber binary found"); + +# ============================================================================= +# Scratch directory that stands in for n3's future PGDATA. +# It just needs to exist so the manifest can be written there. +# ============================================================================= +my $N3_PGDATA = '/tmp/spock_bidir_test_n3_pgdata'; +my $MANIFEST = "$N3_PGDATA/spock_bidirectional_manifest.json"; + +remove_tree($N3_PGDATA) if -d $N3_PGDATA; +make_path($N3_PGDATA) + or BAIL_OUT("could not create temp pgdata dir: $N3_PGDATA"); +pass("temp pgdata dir for n3 created"); + +# ============================================================================= +# SETUP: 2-node cluster, cross-wired bidirectionally +# create_cluster counts as 5 tests (pg_isready + spock check per node + pass) +# ============================================================================= +create_cluster(2, 'Create bidirectional 2-node cluster'); + +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $dbname = $config->{db_name}; +my $host = $config->{host}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; + +my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" + . " user=$db_user password=$db_password"; + +# Create bidirectional subscriptions n1->n2 and n2->n1 (1 test) +cross_wire(2, ['n1', 'n2'], 'Cross-wire n1 <-> n2 bidirectionally'); + +# ============================================================================= +# TEST: --bidirectional mode +# Discovers peer n2 from n1, checks preconditions, writes manifest. +# ============================================================================= + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--pgdata', $N3_PGDATA, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + ], + '--bidirectional plumbing exits 0' +); + +ok(-f $MANIFEST, + 'manifest written to /spock_bidirectional_manifest.json'); + +# Read and inspect manifest content +my $manifest_content = ''; +if (-f $MANIFEST) { + open my $fh, '<', $MANIFEST or die "Cannot read manifest: $!"; + local $/; + $manifest_content = <$fh>; + close $fh; +} + +like($manifest_content, qr/"version":\s*1/, + 'manifest: version is 1'); +like($manifest_content, qr/"subscriber_name":\s*"n3"/, + 'manifest: subscriber_name is n3'); +like($manifest_content, qr/"dbname":\s*"$dbname"/, + 'manifest: dbname matches provider dbname'); +ok(index($manifest_content, '"source_dsn":') >= 0, + 'manifest: source_dsn field present'); +like($manifest_content, qr/"node_name":\s*"n2"/, + 'manifest: peer n2 is listed in peers array'); +ok(index($manifest_content, '"peer_slot_name":') >= 0, + 'manifest: peer_slot_name field present'); + +# ============================================================================= +# TEST: --cleanup mode — removes manifest, exits 0 +# ============================================================================= + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--pgdata', $N3_PGDATA, + ], + '--cleanup with manifest exits 0' +); + +ok(!-f $MANIFEST, + 'manifest file removed by --cleanup'); + +# Second cleanup with no manifest must also exit 0 (idempotent) +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--pgdata', $N3_PGDATA, + ], + '--cleanup with no manifest exits 0 (idempotent)' +); + +# ============================================================================= +# CLEANUP +# ============================================================================= +remove_tree($N3_PGDATA); +destroy_cluster('Cleanup'); From e31491a6ee92c31acdd7ec7bf0a2335f61c04055 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Tue, 28 Jul 2026 14:01:41 +0500 Subject: [PATCH 10/13] spock_create_subscriber: physical backup, catalog strip, repset restore Continue --bidirectional past peer discovery into the physical-backup pipeline (source slot creation, pg_basebackup, restore point, recovery), stopping just before the catchup subscription: - Capture replication-set definitions, table memberships, and sequence state from the local catalog before DROP EXTENSION removes it. - Drop all replication origins, then guard DROP EXTENSION ... CASCADE with a pg_depend inventory of non-spock dependents, failing loudly instead of silently destroying user objects. - Create the local node, immediately set spock.readonly = 'local', and restore the captured replication-set state. The DSN registered for the node is derived from --subscriber-dsn, not a separate option. - Give the new node its own system identifier right after promotion and before any catalog mutation, so a plain system_identifier comparison can verify --subscriber-dsn actually reaches it. - New --force option: also remove the data directory on --cleanup. Preconditions are hardened beyond a simple subscription count: full-mesh validation now checks actual subscription health and peer identity, not just sub_enabled, and rejects duplicate edges from the same origin; replication-set and schema fingerprints are compared between the source and every peer, covering column type, typmod, collation, nullability, generated/identity columns, relation kind, and replica identity. --- .gitignore | 2 + src/spock_fe.c | 9 +- tests/tap/schedule | 2 +- tests/tap/t/047_bidir_plumbing.pl | 165 -- tests/tap/t/048_bidir_pr3.pl | 585 ++++ .../spock_create_subscriber.c | 2474 +++++++++++++++-- 6 files changed, 2826 insertions(+), 411 deletions(-) delete mode 100644 tests/tap/t/047_bidir_plumbing.pl create mode 100644 tests/tap/t/048_bidir_pr3.pl diff --git a/.gitignore b/.gitignore index 0d9c3d3c..20f6c237 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ results regression_output tmp_check +/utils/spock_create_subscriber/spock_create_subscriber +/utils/spock_create_subscriber/.deps/ .vimrc *.o *.so diff --git a/src/spock_fe.c b/src/spock_fe.c index ebe62a82..0a6d66e0 100644 --- a/src/spock_fe.c +++ b/src/spock_fe.c @@ -226,10 +226,12 @@ appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str) bool needquotes; /* - * If the string consists entirely of plain ASCII characters, no need to - * quote it. This is quite conservative, but better safe than sorry. + * If the string is one or more plain ASCII characters, no need to quote + * it. An empty string must default to needing quotes -- an unquoted + * empty value doesn't parse as empty, it swallows the entire next + * "keyword=value" token. */ - needquotes = false; + needquotes = true; for (s = str; *s; s++) { if (!((*s >= 'a' && *s <= 'z') || (*s >= 'A' && *s <= 'Z') || @@ -238,6 +240,7 @@ appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str) needquotes = true; break; } + needquotes = false; } if (needquotes) diff --git a/tests/tap/schedule b/tests/tap/schedule index f195915c..9b2c208f 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -55,7 +55,7 @@ test: 032_lolor_largeobject_repset test: 033_zodan_lolor_add_node test: 034_reserved_object_ddl test: 044_apply_change_logging -test: 047_bidir_plumbing +test: 048_bidir_pr3 # Upgrade schema match test (builds from source, slow): #test: 018_upgrade_schema_match # diff --git a/tests/tap/t/047_bidir_plumbing.pl b/tests/tap/t/047_bidir_plumbing.pl deleted file mode 100644 index 64d19f12..00000000 --- a/tests/tap/t/047_bidir_plumbing.pl +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/perl -# ============================================================================= -# Test: 047_bidir_plumbing.pl - spock_create_subscriber --bidirectional -# ============================================================================= -# Validates the plumbing phase of the SPOC-601 bidirectional node-join -# procedure. The test does NOT start a third PostgreSQL instance; it only -# exercises the utility's plumbing phase against an existing 2-node cluster: -# -# --bidirectional discover peers, check preconditions, write manifest -# --cleanup idempotently remove partial state / manifest -# -# Topology: -# n1 <-> n2 (full bidirectional Spock subscriptions, track_commit_timestamp=on) -# -# The utility is run with --pgdata pointing at a plain temp directory (no PG -# cluster) that exists solely to hold the manifest file. -# -# Test count breakdown: -# 1 binary found -# 1 temp pgdata created -# 5 create_cluster(2) [2 pg_isready + 2 spock checks + 1 pass] -# 1 cross_wire n1<->n2 -# 1 --bidirectional exits 0 -# 1 manifest file written -# 1 manifest: version 1 -# 1 manifest: subscriber_name n3 -# 1 manifest: dbname regression -# 1 manifest: source_dsn present -# 1 manifest: peer n2 listed -# 1 manifest: peer_slot_name present -# 1 --cleanup exits 0 -# 1 manifest removed -# 1 --cleanup with no manifest exits 0 (idempotent) -# 1 destroy_cluster -# --- -# 20 total -# ============================================================================= - -use strict; -use warnings; -use Test::More tests => 20; -use File::Path qw(remove_tree make_path); -use lib '.'; -use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail - command_ok system_maybe get_test_config scalar_query psql_or_bail); - -# ============================================================================= -# Locate spock_create_subscriber binary -# ============================================================================= -my $SCS_BIN; -for my $dir (split(':', $ENV{PATH} // '')) { - my $c = "$dir/spock_create_subscriber"; - if (-x $c) { $SCS_BIN = $c; last; } -} -unless (defined $SCS_BIN) { - # Fall back to the build tree (CWD is tests/tap/ during make check_prove) - my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; - $SCS_BIN = $bt if -x $bt; -} -BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") - unless defined $SCS_BIN; -pass("spock_create_subscriber binary found"); - -# ============================================================================= -# Scratch directory that stands in for n3's future PGDATA. -# It just needs to exist so the manifest can be written there. -# ============================================================================= -my $N3_PGDATA = '/tmp/spock_bidir_test_n3_pgdata'; -my $MANIFEST = "$N3_PGDATA/spock_bidirectional_manifest.json"; - -remove_tree($N3_PGDATA) if -d $N3_PGDATA; -make_path($N3_PGDATA) - or BAIL_OUT("could not create temp pgdata dir: $N3_PGDATA"); -pass("temp pgdata dir for n3 created"); - -# ============================================================================= -# SETUP: 2-node cluster, cross-wired bidirectionally -# create_cluster counts as 5 tests (pg_isready + spock check per node + pass) -# ============================================================================= -create_cluster(2, 'Create bidirectional 2-node cluster'); - -my $config = get_test_config(); -my $node_ports = $config->{node_ports}; -my $dbname = $config->{db_name}; -my $host = $config->{host}; -my $db_user = $config->{db_user}; -my $db_password = $config->{db_password}; - -my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" - . " user=$db_user password=$db_password"; - -# Create bidirectional subscriptions n1->n2 and n2->n1 (1 test) -cross_wire(2, ['n1', 'n2'], 'Cross-wire n1 <-> n2 bidirectionally'); - -# ============================================================================= -# TEST: --bidirectional mode -# Discovers peer n2 from n1, checks preconditions, writes manifest. -# ============================================================================= - -command_ok( - [ $SCS_BIN, - '--bidirectional', - '--pgdata', $N3_PGDATA, - '--subscriber-name', 'n3', - '--provider-dsn', $n1_dsn, - ], - '--bidirectional plumbing exits 0' -); - -ok(-f $MANIFEST, - 'manifest written to /spock_bidirectional_manifest.json'); - -# Read and inspect manifest content -my $manifest_content = ''; -if (-f $MANIFEST) { - open my $fh, '<', $MANIFEST or die "Cannot read manifest: $!"; - local $/; - $manifest_content = <$fh>; - close $fh; -} - -like($manifest_content, qr/"version":\s*1/, - 'manifest: version is 1'); -like($manifest_content, qr/"subscriber_name":\s*"n3"/, - 'manifest: subscriber_name is n3'); -like($manifest_content, qr/"dbname":\s*"$dbname"/, - 'manifest: dbname matches provider dbname'); -ok(index($manifest_content, '"source_dsn":') >= 0, - 'manifest: source_dsn field present'); -like($manifest_content, qr/"node_name":\s*"n2"/, - 'manifest: peer n2 is listed in peers array'); -ok(index($manifest_content, '"peer_slot_name":') >= 0, - 'manifest: peer_slot_name field present'); - -# ============================================================================= -# TEST: --cleanup mode — removes manifest, exits 0 -# ============================================================================= - -command_ok( - [ $SCS_BIN, - '--bidirectional', - '--cleanup', - '--pgdata', $N3_PGDATA, - ], - '--cleanup with manifest exits 0' -); - -ok(!-f $MANIFEST, - 'manifest file removed by --cleanup'); - -# Second cleanup with no manifest must also exit 0 (idempotent) -command_ok( - [ $SCS_BIN, - '--bidirectional', - '--cleanup', - '--pgdata', $N3_PGDATA, - ], - '--cleanup with no manifest exits 0 (idempotent)' -); - -# ============================================================================= -# CLEANUP -# ============================================================================= -remove_tree($N3_PGDATA); -destroy_cluster('Cleanup'); diff --git a/tests/tap/t/048_bidir_pr3.pl b/tests/tap/t/048_bidir_pr3.pl new file mode 100644 index 00000000..a317e451 --- /dev/null +++ b/tests/tap/t/048_bidir_pr3.pl @@ -0,0 +1,585 @@ +#!/usr/bin/perl +# ============================================================================= +# Test: 048_bidir_pr3.pl - spock_create_subscriber --bidirectional +# ============================================================================= +# Validates the bidirectional node-join procedure: physical backup, recovery +# to a restore point, catalog strip (capture + origin drop + guarded DROP +# EXTENSION), and replication-set/table/sequence restore -- stopping short +# of the catchup subscription (a later step). +# +# Topology: +# n1 <-> n2 (full bidirectional Spock subscriptions, existing 2-node +# cluster from create_cluster/cross_wire) +# n3 a real third PostgreSQL instance built via +# `spock_create_subscriber --bidirectional`, physically backed +# up from n1. +# +# Test count breakdown: +# 1 binary found +# 5 create_cluster(2) +# 1 cross_wire n1<->n2 +# 1 custom replication set created on n1 +# 1 table with row_filter added to custom set on n1 +# 1 table with explicit column list added to custom set on n1 +# 1 sequence added to custom set on n1 +# 1 sequence advanced past its initial value on n1 (setval fidelity check) +# 1 partitioned table (parent + 2 children) added to custom set on n1 +# 1 sequence with apostrophe in name added to custom set on n1 +# 1 --bidirectional exits 0 +# 1 n3 postgres is running +# 1 spock extension installed cleanly on n3 (exactly one row) +# 1 n3 has no leftover replication origins from the basebackup +# 1 n3 was given its own system identifier (pg_resetwal), distinct from n1 +# 1 spock.readonly is 'local' on n3 +# 1 custom replication set restored on n3 with correct flags +# 1 table membership restored with correct row_filter +# 1 table membership restored with correct explicit column list +# 1 sequence value restored exactly (last_value) +# 1 sequence is_called restored exactly +# 1 sequence pr3_test_seq is a member of pr3_test_repset on n3 +# 1 partitioned table parent + 2 children all present in repset on n3 +# 1 apostrophe-named sequence value restored on n3 +# 1 apostrophe-named sequence is_called restored on n3 +# 1 apostrophe-named sequence is a member of pr3_test_repset on n3 +# 1 manifest: source_slot_name populated +# 1 manifest: source_restore_lsn populated +# 1 manifest: node_dsn populated +# 1 source slot exists on n1 +# 1 --cleanup --force exits 0 +# 1 source slot removed from n1 after cleanup +# 1 n3 data directory removed after cleanup --force +# 1 manifest removed after cleanup +# 1 --bidirectional rejects a multi-database request +# 1 --bidirectional aborts when another database on the source has spock configured +# 1 --bidirectional rejects a broken full-mesh topology (disabled subscription) +# 1 --bidirectional rejects mismatched replication-set flags between source and peer +# 1 a broken --extra-basebackup-args makes the base backup fail +# 1 pending-cleanup sidecar written before the failed backup +# 1 pending-cleanup sidecar is mode 0600 +# 1 source slot still exists on n1 after the failed backup (orphaned) +# 1 --cleanup --force recovers via the pending sidecar +# 1 source slot removed from n1 via sidecar-based cleanup +# 1 pending-cleanup sidecar removed after cleanup +# 1 a broken backup orphans a slot for the retry-cleanup test +# 1 pending sidecar written for the retry-cleanup test +# 1 --cleanup exits non-zero when the source is unreachable +# 1 pending sidecar retained after an incomplete cleanup +# 1 n1 postgres is running again +# 1 --cleanup --force succeeds once the source is reachable again +# 1 pending sidecar removed once cleanup actually completed +# 1 destroy_cluster +# --- +# 57 total +# ============================================================================= + +use strict; +use warnings; +use Test::More tests => 57; +use File::Path qw(remove_tree); +use lib '.'; +use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail + command_ok system_maybe get_test_config scalar_query + psql_or_bail wait_for_pg_ready); + +# ============================================================================= +# Locate spock_create_subscriber binary +# ============================================================================= +my $SCS_BIN; +for my $dir (split(':', $ENV{PATH} // '')) { + my $c = "$dir/spock_create_subscriber"; + if (-x $c) { $SCS_BIN = $c; last; } +} +unless (defined $SCS_BIN) { + my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; + $SCS_BIN = $bt if -x $bt; +} +BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") + unless defined $SCS_BIN; +pass("spock_create_subscriber binary found"); + +# ============================================================================= +# SETUP: 2-node cluster, cross-wired bidirectionally +# ============================================================================= +create_cluster(2, 'Create bidirectional 2-node cluster'); + +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $dbname = $config->{db_name}; +my $host = $config->{host}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; + +my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" + . " user=$db_user password=$db_password"; + +my $n1_sysid = scalar_query(1, "SELECT system_identifier FROM pg_control_system()"); + +cross_wire(2, ['n1', 'n2'], 'Cross-wire n1 <-> n2 bidirectionally'); + +# ============================================================================= +# Seed n1 with a custom replication set, a table with a row_filter, and a +# sequence, to exercise the catalog capture/restore with non-default state +# rather than just the three built-in sets. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_create('pr3_test_repset', true, true, true, false)"; +pass('custom replication set created on n1'); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_tbl (id serial primary key, region text, value integer)"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_table(set_name := 'pr3_test_repset', " . + "relation := 'pr3_test_tbl', synchronize_data := false, " . + "row_filter := 'region = ''east''')"; +pass('table with row_filter added to custom set on n1'); + +# Table with an explicit, non-default column list, to exercise the +# columns := restore path (captured/restored as a bare array- +# literal string relying on implicit text[] coercion) -- previously +# untested, so a round-trip regression here could pass silently. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_cols (id serial primary key, region text, " . + "value integer, secret text)"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_table(set_name := 'pr3_test_repset', " . + "relation := 'pr3_test_cols', synchronize_data := false, " . + "columns := ARRAY['id', 'region', 'value'])"; +pass('table with explicit column list added to custom set on n1'); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE SEQUENCE pr3_test_seq"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_seq('pr3_test_repset', 'pr3_test_seq')"; +pass('sequence added to custom set on n1'); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT setval('pr3_test_seq', 42, true)"; +my $seq_before = scalar_query(1, "SELECT last_value FROM pr3_test_seq"); +is($seq_before, '42', 'sequence advanced past its initial value on n1'); + +# Partitioned table: parent + 2 children get separate captured membership +# rows (that's how include_partitions => true populated them here); restore +# must not try to re-add children a second time via the parent's own +# include_partitions => true call. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_part (id int, region text, PRIMARY KEY (id, region)) PARTITION BY LIST (region)"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_part_east PARTITION OF pr3_test_part FOR VALUES IN ('east')"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_part_west PARTITION OF pr3_test_part FOR VALUES IN ('west')"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_table(set_name := 'pr3_test_repset', " . + "relation := 'pr3_test_part', synchronize_data := false, " . + "include_partitions := true)"; +pass('partitioned table (parent + 2 children) added to custom set on n1'); + +# Sequence with an apostrophe in its name, to exercise setval() quoting. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + q(CREATE SEQUENCE "weird's_seq"); +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + q(SELECT spock.repset_add_seq('pr3_test_repset', '"weird''s_seq"')); +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + q(SELECT setval('"weird''s_seq"', 7, true)); +pass('sequence with apostrophe in name added to custom set on n1'); + +# check_preconditions() requires all of n1's outbound replication to have +# caught up (no unreplicated DDL/data still in flight to n2); wait for the +# setup above to drain. +for (1 .. 15) { + my $lag = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots" . + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" . + " AND (confirmed_flush_lsn IS NULL OR confirmed_flush_lsn < pg_current_wal_lsn())"); + last if defined $lag && $lag eq '0'; + sleep(1); +} + +# ============================================================================= +# TEST: --bidirectional continues through physical backup / catalog strip / +# repset restore, stopping before the catchup subscription. +# ============================================================================= +my $n3_port = $node_ports->[1] + 1; +my $n3_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3'; +my $manifest = "$n3_datadir/spock_bidirectional_manifest.json"; +my $n3_dsn = "host=$host port=$n3_port dbname=$dbname" + . " user=$db_user password=$db_password"; + +remove_tree($n3_datadir) if -d $n3_datadir; + +# n3's postgresql.conf is copied verbatim from n1 by the basebackup, port and +# all -- since all nodes run on the same host in this test, n3 must be given +# an override with its own port (a real cross-host join wouldn't need this). +my $n3_conf = '/tmp/tmp_spock_node_2_postgresql.conf.override'; +open my $conf_fh, '>', $n3_conf or die "Cannot write $n3_conf: $!"; +print $conf_fh "shared_buffers=1GB\n"; +print $conf_fh "shared_preload_libraries='spock'\n"; +print $conf_fh "wal_level=logical\n"; +print $conf_fh "spock.enable_ddl_replication=on\n"; +print $conf_fh "spock.include_ddl_repset=on\n"; +print $conf_fh "spock.allow_ddl_from_functions=on\n"; +print $conf_fh "spock.exception_behaviour=sub_disable\n"; +print $conf_fh "spock.conflict_resolution=last_update_wins\n"; +print $conf_fh "track_commit_timestamp=on\n"; +print $conf_fh "spock.exception_replay_queue_size='1MB'\n"; +print $conf_fh "spock.enable_spill=on\n"; +print $conf_fh "port=$n3_port\n"; +print $conf_fh "listen_addresses='*'\n"; +print $conf_fh "logging_collector=on\n"; +print $conf_fh "log_directory='" . $config->{log_dir} . "'\n"; +print $conf_fh "log_filename='00${n3_port}.log'\n"; +close $conf_fh; + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--pgdata', $n3_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--postgresql-conf', $n3_conf, + ], + '--bidirectional exits 0' +); + +ok(wait_for_pg_ready($host, $n3_port, $pg_bin, 30), 'n3 postgres is running'); + +my $ext_count = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM pg_extension WHERE extname = 'spock'"`; +$ext_count =~ s/\s+//g; +is($ext_count, '1', 'spock extension installed cleanly on n3 (exactly one row)'); + +my $origin_count = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM pg_replication_origin"`; +$origin_count =~ s/\s+//g; +is($origin_count, '0', 'n3 has no leftover replication origins from the basebackup'); + +my $n3_sysid = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT system_identifier FROM pg_control_system()"`; +$n3_sysid =~ s/\s+//g; +isnt($n3_sysid, $n1_sysid, + 'n3 was given its own system identifier (pg_resetwal), distinct from n1'); + +my $readonly = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SHOW spock.readonly"`; +$readonly =~ s/\s+//g; +is($readonly, 'local', "spock.readonly is 'local' on n3"); + +my $repset_flags = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT replicate_insert, replicate_update, replicate_delete, replicate_truncate FROM spock.replication_set WHERE set_name = 'pr3_test_repset'"`; +$repset_flags =~ s/\s+//g; +is($repset_flags, 't|t|t|f', 'custom replication set restored on n3 with correct flags'); + +my $row_filter = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT pg_get_expr(rts.set_row_filter, rts.set_reloid) FROM spock.replication_set_table rts JOIN spock.replication_set rs ON rts.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset'"`; +$row_filter =~ s/^\s+|\s+$//g; +like($row_filter, qr/region\s*=\s*'east'/, 'table membership restored with correct row_filter'); + +my $columns = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT rts.set_att_list FROM spock.replication_set_table rts JOIN spock.replication_set rs ON rts.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rts.set_reloid::regclass::text = 'pr3_test_cols'"`; +$columns =~ s/^\s+|\s+$//g; +is($columns, '{id,region,value}', + 'table membership restored with correct explicit column list'); + +my $seq_last_value = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT last_value FROM pr3_test_seq"`; +$seq_last_value =~ s/\s+//g; +is($seq_last_value, '42', 'sequence value restored exactly (last_value)'); + +my $seq_is_called = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT is_called FROM pr3_test_seq"`; +$seq_is_called =~ s/\s+//g; +is($seq_is_called, 't', 'sequence is_called restored exactly'); + +# pr3_test_seq must be an actual member of pr3_test_repset on n3, not just +# have its value restored (a regression here is the sequence-membership bug: +# setval() alone leaves the sequence unpublished). +my $seq_member = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT COUNT(*) FROM spock.replication_set_seq rss JOIN spock.replication_set rs ON rss.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rss.set_seqoid::regclass::text = 'pr3_test_seq'"`; +$seq_member =~ s/\s+//g; +is($seq_member, '1', 'sequence pr3_test_seq is a member of pr3_test_repset on n3'); + +# Partitioned table: parent + 2 children must all be present as distinct +# memberships (a regression here is include_partitions => true re-adding +# already-captured children and violating the (set_id, set_reloid) PK, +# which would have aborted the join above rather than just miscounting). +my $part_member_count = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT COUNT(*) FROM spock.replication_set_table rts JOIN spock.replication_set rs ON rts.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rts.set_reloid::regclass::text LIKE 'pr3_test_part%'"`; +$part_member_count =~ s/\s+//g; +is($part_member_count, '3', 'partitioned table parent + 2 children all present in repset on n3'); + +# Sequence with an apostrophe in its name: value/is_called restored and +# membership present, without a SQL syntax error breaking the whole run. +sub psql_capture { + my (@args) = @_; + open(my $fh, '-|', "$pg_bin/psql", @args) or die "cannot run psql: $!"; + local $/; + my $out = <$fh>; + close $fh; + $out =~ s/^\s+|\s+$//g if defined $out; + return $out; +} + +my $weird_seq_value = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', 'SELECT last_value FROM "weird\'s_seq"'); +is($weird_seq_value, '7', "apostrophe-named sequence value restored on n3"); + +my $weird_seq_called = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', 'SELECT is_called FROM "weird\'s_seq"'); +is($weird_seq_called, 't', "apostrophe-named sequence is_called restored on n3"); + +my $weird_seq_member = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT COUNT(*) FROM spock.replication_set_seq rss JOIN spock.replication_set rs ON rss.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rss.set_seqoid::regclass::text = '\"weird''s_seq\"'"); +is($weird_seq_member, '1', "apostrophe-named sequence is a member of pr3_test_repset on n3"); + +# ============================================================================= +# Manifest content checks +# ============================================================================= +my $manifest_content = ''; +if (-f $manifest) { + open my $fh, '<', $manifest or die "Cannot read manifest: $!"; + local $/; + $manifest_content = <$fh>; + close $fh; +} + +ok($manifest_content =~ /"source_slot_name":\s*"[^"]+"/, + 'manifest: source_slot_name populated'); +ok($manifest_content =~ /"source_restore_lsn":\s*"[0-9A-Fa-f]+\/[0-9A-Fa-f]+"/, + 'manifest: source_restore_lsn populated'); +ok($manifest_content =~ /"node_dsn":\s*"[^"]+"/, + 'manifest: node_dsn populated'); + +my $source_slot_exists = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +ok($source_slot_exists >= 1, 'source slot exists on n1'); + +# ============================================================================= +# TEST: --cleanup --force removes source slot, data directory, and manifest +# ============================================================================= +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $n3_datadir, + ], + '--cleanup --force exits 0' +); + +my $source_slot_after = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +is($source_slot_after, '0', 'source slot removed from n1 after cleanup'); + +ok(!-d $n3_datadir, 'n3 data directory removed after cleanup --force'); +ok(!-f $manifest, 'manifest removed after cleanup'); + +# ============================================================================= +# TEST: --bidirectional hard-rejects a multi-database request outright, +# rather than silently joining only the first-named database -- all join +# state is per-database. +# ============================================================================= +my $multidb_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_multidb'; +remove_tree($multidb_datadir) if -d $multidb_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $multidb_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--databases', "$dbname,postgres"), + '--bidirectional rejects a multi-database request'); +remove_tree($multidb_datadir) if -d $multidb_datadir; + +# ============================================================================= +# TEST: --bidirectional aborts if the source instance has spock configured +# on another database too, even though that database was never named via +# --databases (check_single_spock_database() must fail closed). +# ============================================================================= +system_or_bail "$pg_bin/createdb", '-p', $node_ports->[0], 'pr3_other_db'; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', 'pr3_other_db', '-c', + "CREATE EXTENSION spock"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', 'pr3_other_db', '-c', + "SELECT spock.node_create('pr3_other_node', 'dbname=pr3_other_db')"; + +my $otherdb_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_otherdb'; +remove_tree($otherdb_datadir) if -d $otherdb_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $otherdb_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn), + '--bidirectional aborts when another database on the source has spock configured'); +remove_tree($otherdb_datadir) if -d $otherdb_datadir; + +system_maybe "$pg_bin/psql", '-p', $node_ports->[0], '-d', 'pr3_other_db', '-c', + "SELECT spock.node_drop('pr3_other_node')"; +system_maybe "$pg_bin/dropdb", '-p', $node_ports->[0], 'pr3_other_db'; + +# ============================================================================= +# TEST: --bidirectional rejects a broken full-mesh topology -- a disabled +# subscription is not a valid mesh edge, even though it still exists. A +# plain subscription COUNT would not catch this. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SELECT spock.sub_disable('sub_n2_n1', true)"; +for (1 .. 15) { + my $enabled = scalar_query(2, + "SELECT sub_enabled FROM spock.subscription WHERE sub_name = 'sub_n2_n1'"); + last if defined $enabled && $enabled eq 'f'; + sleep(1); +} + +my $mesh_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_mesh'; +remove_tree($mesh_datadir) if -d $mesh_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $mesh_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn), + '--bidirectional rejects a broken full-mesh topology (disabled subscription)'); +remove_tree($mesh_datadir) if -d $mesh_datadir; + +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SELECT spock.sub_enable('sub_n2_n1', true)"; +for (1 .. 15) { + my $enabled = scalar_query(2, + "SELECT sub_enabled FROM spock.subscription WHERE sub_name = 'sub_n2_n1'"); + last if defined $enabled && $enabled eq 't'; + sleep(1); +} + +# ============================================================================= +# TEST: --bidirectional rejects mismatched replication-set definitions for a +# selected (subscription-referenced) set between source and peer -- a +# repset the forwarding path and a future direct-peer path disagree on can +# permanently drop changes on cutover. DDL replication is disabled for the +# ALTER itself so the mismatch is real and local to n2. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; " . + "SELECT spock.repset_alter('default', replicate_truncate := false)"; + +my $repset_mismatch_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_repset_mismatch'; +remove_tree($repset_mismatch_datadir) if -d $repset_mismatch_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $repset_mismatch_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn), + '--bidirectional rejects mismatched replication-set flags between source and peer'); +remove_tree($repset_mismatch_datadir) if -d $repset_mismatch_datadir; + +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; " . + "SELECT spock.repset_alter('default', replicate_truncate := true)"; + +# ============================================================================= +# TEST: a failed base backup leaves the source slot recoverable via +# --cleanup, even though the real manifest was never written -- a +# pending-cleanup sidecar is persisted right after source slot creation, +# before the backup even starts, since data_dir must stay empty until +# pg_basebackup runs. +# ============================================================================= +my $failed_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_failed'; +my $pending_sidecar = "${failed_datadir}.spock_bidir_pending.json"; +remove_tree($failed_datadir) if -d $failed_datadir; +unlink($pending_sidecar) if -f $pending_sidecar; + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $failed_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--extra-basebackup-args', '--waldir=/nonexistent_pr3_test_waldir_xyz'), + 'a broken --extra-basebackup-args makes the base backup fail'); + +ok(-f $pending_sidecar, 'pending-cleanup sidecar written before the failed backup'); + +my $sidecar_mode = (stat($pending_sidecar))[2] & 07777; +is(sprintf('%04o', $sidecar_mode), '0600', + 'pending-cleanup sidecar is mode 0600 (may carry a DSN password)'); + +my $slot_after_failed_backup = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +ok($slot_after_failed_backup >= 1, + 'source slot still exists on n1 after the failed backup (orphaned)'); + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $failed_datadir, + ], + '--cleanup --force recovers via the pending sidecar (no real manifest exists)' +); + +my $slot_after_sidecar_cleanup = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +is($slot_after_sidecar_cleanup, '0', + 'source slot removed from n1 via sidecar-based cleanup'); + +ok(!-f $pending_sidecar, 'pending-cleanup sidecar removed after cleanup'); +remove_tree($failed_datadir) if -d $failed_datadir; + +# ============================================================================= +# TEST: an incomplete cleanup (source unreachable) exits non-zero and keeps +# the pending sidecar so it can be retried, instead of unconditionally +# deleting the only retry record. +# ============================================================================= +my $retry_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_retry'; +my $retry_sidecar = "${retry_datadir}.spock_bidir_pending.json"; +remove_tree($retry_datadir) if -d $retry_datadir; +unlink($retry_sidecar) if -f $retry_sidecar; + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $retry_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--extra-basebackup-args', '--waldir=/nonexistent_pr3_test_waldir_retry'), + 'a broken backup orphans a slot for the retry-cleanup test'); +ok(-f $retry_sidecar, 'pending sidecar written for the retry-cleanup test'); + +my $n1_datadir = $config->{node_datadirs}->[0]; +system_or_bail "$pg_bin/pg_ctl", 'stop', '-D', $n1_datadir, '-m', 'fast'; + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $retry_datadir), + '--cleanup exits non-zero when the source is unreachable'); +ok(-f $retry_sidecar, + 'pending sidecar retained after an incomplete cleanup (retryable)'); + +system_or_bail "$pg_bin/pg_ctl", 'start', '-D', $n1_datadir, + '-l', "$config->{log_dir}/n1_retry_restart.log"; +ok(wait_for_pg_ready($host, $node_ports->[0], $pg_bin, 30), + 'n1 postgres is running again'); + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $retry_datadir, + ], + '--cleanup --force succeeds once the source is reachable again' +); +ok(!-f $retry_sidecar, + 'pending sidecar removed once cleanup actually completed'); +remove_tree($retry_datadir) if -d $retry_datadir; + +# ============================================================================= +# CLEANUP +# ============================================================================= +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP TABLE IF EXISTS pr3_test_tbl"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP TABLE IF EXISTS pr3_test_part"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP TABLE IF EXISTS pr3_test_cols"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP SEQUENCE IF EXISTS pr3_test_seq"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + q(DROP SEQUENCE IF EXISTS "weird's_seq"); +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_drop('pr3_test_repset')"; +unlink($n3_conf) if -f $n3_conf; +destroy_cluster('Cleanup'); diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index bff53736..b8349a15 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -52,8 +53,12 @@ #include "access/timeline.h" #include "access/xlog_internal.h" #include "catalog/pg_control.h" +#include "common/controldata_utils.h" +#include "common/file_utils.h" #include "common/jsonapi.h" +#include "common/logging.h" #include "mb/pg_wchar.h" +#include "port.h" #include "spock_fe.h" @@ -87,10 +92,57 @@ typedef struct BidirectionalState int max_wait; /* default 0 = unbounded */ char *source_slot_name; char *source_origin_name; + char *source_restore_lsn; /* recovery target LSN; consumed by the + * disabled-first catchup sub_create */ + char *node_dsn; /* DSN registered via spock.node_create(); + * the address peers use to connect back to + * this node. Derived from --subscriber-dsn. */ bool cleanup_mode; + bool force_cleanup; /* --force: also remove the data directory + * on --cleanup, not just remote state */ char *manifest_path; } BidirectionalState; +/* + * Replication-set / table-membership / sequence state captured from the + * source's catalog before DROP EXTENSION spock removes it. Utility-side + * memory only; never written to the manifest. + */ +typedef struct RepsetCapture +{ + char *set_name; + bool replicate_insert; + bool replicate_update; + bool replicate_delete; + bool replicate_truncate; +} RepsetCapture; + +typedef struct RepsetTableCapture +{ + char *set_name; + char *qualified_table; /* rts.set_reloid::regclass */ + char *columns; /* rts.set_att_list, NULL if all columns */ + char *row_filter; /* pg_get_expr(...), NULL if none */ +} RepsetTableCapture; + +typedef struct SequenceCapture +{ + char *set_name; + char *qualified_seq; + int64 last_value; + bool is_called; +} SequenceCapture; + +typedef struct CatalogCapture +{ + RepsetCapture *repsets; + int num_repsets; + RepsetTableCapture *tables; + int num_tables; + SequenceCapture *sequences; + int num_sequences; +} CatalogCapture; + typedef enum { VERBOSITY_NORMAL, VERBOSITY_VERBOSE, @@ -119,8 +171,10 @@ static int run_pg_ctl(const char *arg); static void validate_extra_basebackup_args(const char *args); static void run_basebackup(const char *provider_connstr, const char *data_dir, const char *extra_basebackup_args); +static char *reset_subscriber_sysid(const char *data_dir); +static void run_pg_resetwal(const char *data_dir); static void wait_postmaster_connection(const char *connstr); -static void wait_primary_connection(const char *connstr); +static void wait_primary_connection(const char *connstr, int stall_timeout, int max_wait); static void wait_postmaster_shutdown(void); static char *validate_replication_set_input(char *replication_sets); @@ -170,17 +224,36 @@ static char *generate_restore_point_name(void); static int discover_peer_nodes(PGconn *source_conn, const char *source_node_name, const char *subscriber_name, const char *dbname, PeerNodeInfo **peers_out); -static void check_preconditions(PGconn *source_conn, PeerNodeInfo *peers, int num_peers); +static void check_preconditions(PGconn *source_conn, const char *source_node_name, + PeerNodeInfo *peers, int num_peers); +static void check_spock_version_at_least_6(PGconn *conn, const char *node_label); +static void check_mesh_edges(PGconn *conn, const char *this_node_name, + char **all_names, int total_nodes); +static void check_peer_identity(PGconn *peer_conn, const char *expected_name); +static void check_replication_set_equivalence(PGconn *source_conn, + const char *source_node_name, + PeerNodeInfo *peers, int num_peers); static void write_manifest(BidirectionalState *state, const char *subscriber_name, const char *dbname, const char *source_dsn); static bool read_manifest(const char *manifest_path, BidirectionalState *state, char **subscriber_name_out, char **dbname_out, char **source_dsn_out); -static void cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, +static bool cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, const char *dbname, const char *source_dsn, bool force_rm_datadir); static void append_json_string(PQExpBuffer buf, const char *str); +static void check_single_spock_database(PGconn *conn, const char *base_prov_connstr, + const char *current_dbname); +static void check_no_native_subscriptions(PGconn *conn); +static void capture_catalog_state(PGconn *conn, Oid source_nodeid, + CatalogCapture *capture); +static void remove_unwanted_data_bidir(PGconn *conn, CatalogCapture *capture); +static void restore_replication_sets(PGconn *conn, CatalogCapture *capture); +static void verify_replication_sets_restored(PGconn *conn, CatalogCapture *capture); +static void set_readonly_local(PGconn *conn); +static Oid get_local_node_id(PGconn *conn); + static PGconn * connectdb(const char *connstr) { @@ -195,7 +268,7 @@ connectdb(const char *connstr) void signal_handler(int sig) { - if (sig == SIGINT) + if (sig == SIGINT || sig == SIGTERM) { die(_("\nCanceling...\n")); } @@ -313,35 +386,523 @@ discover_peer_nodes(PGconn *source_conn, const char *source_node_name, } /* - * Verify that the source cluster and all peers meet the requirements for - * a bidirectional join: Spock >= 6.0.0, track_commit_timestamp on, no - * pending DDL, full-mesh topology, and peer connectivity. + * Verify Spock version on conn (the source or a peer): an old apply + * worker would advance the wrong-named origin, so this must be checked + * everywhere up front, not just on the source. */ static void -check_preconditions(PGconn *source_conn, PeerNodeInfo *peers, int num_peers) +check_spock_version_at_least_6(PGconn *conn, const char *node_label) { PGresult *res; - int i; - /* Spock version gate: require >= 6.0.0 */ - res = PQexec(source_conn, - "SELECT extversion FROM pg_extension WHERE extname = 'spock'"); + res = PQexec(conn, "SELECT extversion FROM pg_extension WHERE extname = 'spock'"); if (PQresultStatus(res) != PGRES_TUPLES_OK) - die(_("could not query Spock extension version: %s"), - PQerrorMessage(source_conn)); + { + PQclear(res); + die(_("could not query Spock extension version on \"%s\": %s"), + node_label, PQerrorMessage(conn)); + } if (PQntuples(res) == 0) - die(_("Spock extension is not installed on the source node")); + { + PQclear(res); + die(_("Spock extension is not installed on \"%s\""), node_label); + } { const char *ver = PQgetvalue(res, 0, 0); int major = 0; + /* + * die() exits immediately -- ver points inside res, so it must + * not be PQclear()'d first (that would be a use-after-free when + * die()'s own formatting reads ver). + */ if (sscanf(ver, "%d.", &major) < 1) - die(_("could not parse Spock version \"%s\""), ver); + die(_("could not parse Spock version \"%s\" on \"%s\""), ver, node_label); if (major < 6) - die(_("Spock version %s on source is too old for bidirectional " - "join; require >= 6.0.0"), ver); + die(_("Spock version %s on \"%s\" is too old for bidirectional " + "join; require >= 6.0.0"), ver, node_label); + } + PQclear(res); +} + +/* + * Validate the actual directed subscription graph from one node's own + * catalog, not just a count: exactly one healthy (status = 'replicating') + * subscription from every other node in the set, no self-reference, no + * edge from outside the set, and no duplicate edge from the same origin + * regardless of status. + */ +static void +check_mesh_edges(PGconn *conn, const char *this_node_name, + char **all_names, int total_nodes) +{ + PGresult *res; + bool *healthy; + int *edge_count; + int i; + + /* + * spock.sub_show_status() (not raw sub_enabled) so "enabled" also + * means "actually replicating" -- a worker that's down or still + * initializing must not satisfy the mesh. + */ + res = PQexec(conn, "SELECT provider_node, status FROM spock.sub_show_status()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check subscription topology on \"%s\": %s"), + this_node_name, PQerrorMessage(conn)); + } + + healthy = pg_malloc0(total_nodes * sizeof(bool)); + edge_count = pg_malloc0(total_nodes * sizeof(int)); + + for (i = 0; i < PQntuples(res); i++) + { + const char *origin_name = PQgetvalue(res, i, 0); + const char *status = PQgetvalue(res, i, 1); + int idx = -1; + int k; + + if (strcmp(origin_name, this_node_name) == 0) + die(_("node \"%s\" has a subscription whose origin is itself; " + "corrupt or misconfigured topology"), this_node_name); + + for (k = 0; k < total_nodes; k++) + { + if (strcmp(all_names[k], origin_name) == 0) + { + idx = k; + break; + } + } + if (idx == -1) + die(_("node \"%s\" has a subscription from \"%s\", which is not " + "part of the discovered node set; partial-mesh or " + "unknown-node topologies are not supported"), + this_node_name, origin_name); + + /* + * Count regardless of status: an extra disabled duplicate from + * the same origin is still a duplicate edge. + */ + edge_count[idx]++; + if (edge_count[idx] > 1) + die(_("node \"%s\" has more than one subscription from \"%s\" " + "(status \"%s\"); duplicate edges are not supported"), + this_node_name, origin_name, status); + + if (strcmp(status, "replicating") == 0) + healthy[idx] = true; + } + PQclear(res); + + for (i = 0; i < total_nodes; i++) + { + if (strcmp(all_names[i], this_node_name) == 0) + continue; /* skip self */ + if (!healthy[i]) + { + pg_free(healthy); + pg_free(edge_count); + die(_("node \"%s\" has no healthy (status = 'replicating') " + "subscription from \"%s\"; full-mesh topology of live " + "replication is required for bidirectional join"), + this_node_name, all_names[i]); + } + } + pg_free(healthy); + pg_free(edge_count); +} + +/* + * Confirm the peer identifies itself as the name it was discovered + * under, so a node-name collision or wrong DSN can't silently validate + * the mesh against the wrong node. + */ +static void +check_peer_identity(PGconn *peer_conn, const char *expected_name) +{ + PGresult *res; + + res = PQexec(peer_conn, "SELECT node_name FROM spock.node_info()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("could not verify identity of peer \"%s\": %s"), + expected_name, PQerrorMessage(peer_conn)); + } + if (strcmp(PQgetvalue(res, 0, 0), expected_name) != 0) + { + char *actual_name = pg_strdup(PQgetvalue(res, 0, 0)); + + PQclear(res); + die(_("peer discovered as \"%s\" identifies itself as \"%s\" once " + "connected; node-name/identity mismatch, refusing to trust " + "this topology\n"), expected_name, actual_name); + } + PQclear(res); +} + +/* + * Build a canonical, comparable fingerprint of one replication set as + * defined by its owning node: operation flags, each member table (sorted, + * with column list, row filter, and schema), then each member sequence + * (sorted). Scoped by node_id since spock.replication_set is keyed + * UNIQUE(set_nodeid, set_name) -- a set replicated via DDL becomes the + * replaying node's own row, not an echo. selected_filter restricts this + * to sets actually referenced by a subscription's sub_replication_sets, + * since unused/scratch repsets can legitimately differ between nodes. + */ +typedef struct RepsetFingerprintEntry +{ + char *set_name; + char *fingerprint; +} RepsetFingerprintEntry; + +static void +compute_repset_fingerprints(PGconn *conn, Oid node_id, const char *selected_filter, + RepsetFingerprintEntry **out, int *nout) +{ + PGresult *res; + RepsetFingerprintEntry *entries; + int n; + int i; + PQExpBuffer query = createPQExpBuffer(); + + printfPQExpBuffer(query, + "SELECT set_name, replicate_insert, replicate_update," + " replicate_delete, replicate_truncate" + " FROM spock.replication_set WHERE set_nodeid = %u" + " AND (%s)" + " ORDER BY set_name", node_id, selected_filter); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + destroyPQExpBuffer(query); + die(_("could not fingerprint replication sets: %s\n"), PQerrorMessage(conn)); + } + + n = PQntuples(res); + entries = pg_malloc0(n * sizeof(RepsetFingerprintEntry)); + + for (i = 0; i < n; i++) + { + PQExpBuffer fp = createPQExpBuffer(); + PGresult *tres; + PGresult *sres; + int j; + + entries[i].set_name = pg_strdup(PQgetvalue(res, i, 0)); + appendPQExpBuffer(fp, "flags=%s%s%s%s;", + PQgetvalue(res, i, 1)[0] == 't' ? "i" : "", + PQgetvalue(res, i, 2)[0] == 't' ? "u" : "", + PQgetvalue(res, i, 3)[0] == 't' ? "d" : "", + PQgetvalue(res, i, 4)[0] == 't' ? "t" : ""); + + printfPQExpBuffer(query, + "SELECT rts.set_reloid::regclass::text, rts.set_att_list," + " pg_get_expr(rts.set_row_filter, rts.set_reloid)" + " FROM spock.replication_set_table rts" + " JOIN spock.replication_set rs ON rts.set_id = rs.set_id" + " WHERE rs.set_nodeid = %u AND rs.set_name = %s" + " ORDER BY rts.set_reloid::regclass::text", + node_id, + PQescapeLiteral(conn, entries[i].set_name, strlen(entries[i].set_name))); + tres = PQexec(conn, query->data); + if (PQresultStatus(tres) != PGRES_TUPLES_OK) + { + PQclear(tres); + PQclear(res); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint table memberships for set \"%s\": %s\n"), + entries[i].set_name, PQerrorMessage(conn)); + } + + for (j = 0; j < PQntuples(tres); j++) + { + const char *qualified_table = PQgetvalue(tres, j, 0); + PGresult *cres; + PQExpBuffer schema_query = createPQExpBuffer(); + int k; + + appendPQExpBuffer(fp, "tbl=%s|cols=%s|filter=%s|schema=(", + qualified_table, + PQgetisnull(tres, j, 1) ? "*" : PQgetvalue(tres, j, 1), + PQgetisnull(tres, j, 2) ? "-" : PQgetvalue(tres, j, 2)); + + /* + * Schema fingerprint: relation kind and replica identity, + * then per-column name, type, typmod (varchar(10) vs + * varchar(100) is otherwise invisible), collation, + * nullability, and generated/identity status -- so a + * divergent column or relation definition is caught even if + * repset membership itself matches. + */ + printfPQExpBuffer(schema_query, + "SELECT relkind::text, relreplident::text" + " FROM pg_class WHERE oid = %s::regclass", + PQescapeLiteral(conn, qualified_table, strlen(qualified_table))); + cres = PQexec(conn, schema_query->data); + if (PQresultStatus(cres) != PGRES_TUPLES_OK || PQntuples(cres) != 1) + { + PQclear(cres); + PQclear(tres); + PQclear(res); + destroyPQExpBuffer(schema_query); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint relation kind of \"%s\": %s\n"), + qualified_table, PQerrorMessage(conn)); + } + appendPQExpBuffer(fp, "relkind=%s|replident=%s|", + PQgetvalue(cres, 0, 0), PQgetvalue(cres, 0, 1)); + PQclear(cres); + + printfPQExpBuffer(schema_query, + "SELECT a.attname, a.atttypid::regtype::text, a.atttypmod," + " a.attnotnull, a.attidentity, a.attgenerated," + " COALESCE(co.collname, '')" + " FROM pg_attribute a" + " LEFT JOIN pg_collation co ON co.oid = a.attcollation" + " WHERE a.attrelid = %s::regclass AND a.attnum > 0" + " AND NOT a.attisdropped ORDER BY a.attnum", + PQescapeLiteral(conn, qualified_table, strlen(qualified_table))); + cres = PQexec(conn, schema_query->data); + destroyPQExpBuffer(schema_query); + if (PQresultStatus(cres) != PGRES_TUPLES_OK) + { + PQclear(cres); + PQclear(tres); + PQclear(res); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint schema of \"%s\": %s\n"), + qualified_table, PQerrorMessage(conn)); + } + for (k = 0; k < PQntuples(cres); k++) + appendPQExpBuffer(fp, "%s%s:%s:%s:notnull=%s:ident=%s:gen=%s:coll=%s", + k > 0 ? "," : "", + PQgetvalue(cres, k, 0), + PQgetvalue(cres, k, 1), + PQgetvalue(cres, k, 2), + PQgetvalue(cres, k, 3), + PQgetvalue(cres, k, 4), + PQgetvalue(cres, k, 5), + PQgetvalue(cres, k, 6)); + appendPQExpBufferStr(fp, ");"); + PQclear(cres); + } + PQclear(tres); + + printfPQExpBuffer(query, + "SELECT rss.set_seqoid::regclass::text" + " FROM spock.replication_set_seq rss" + " JOIN spock.replication_set rs ON rss.set_id = rs.set_id" + " WHERE rs.set_nodeid = %u AND rs.set_name = %s" + " ORDER BY rss.set_seqoid::regclass::text", + node_id, + PQescapeLiteral(conn, entries[i].set_name, strlen(entries[i].set_name))); + sres = PQexec(conn, query->data); + if (PQresultStatus(sres) != PGRES_TUPLES_OK) + { + PQclear(sres); + PQclear(res); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint sequence memberships for set \"%s\": %s\n"), + entries[i].set_name, PQerrorMessage(conn)); + } + for (j = 0; j < PQntuples(sres); j++) + appendPQExpBuffer(fp, "seq=%s;", PQgetvalue(sres, j, 0)); + PQclear(sres); + + entries[i].fingerprint = pg_strdup(fp->data); + destroyPQExpBuffer(fp); } PQclear(res); + destroyPQExpBuffer(query); + + *out = entries; + *nout = n; +} + +static void +free_repset_fingerprints(RepsetFingerprintEntry *entries, int n) +{ + int i; + + for (i = 0; i < n; i++) + { + pg_free(entries[i].set_name); + pg_free(entries[i].fingerprint); + } + pg_free(entries); +} + +/* + * Build a SQL boolean expression ("set_name IN (...)") over the union of + * every replication set actually referenced by conn's own subscriptions + * (sub_replication_sets), rather than every set that happens to exist + * locally. Caller frees the result. + */ +static char * +build_selected_set_name_filter(PGconn *conn) +{ + PGresult *res; + PQExpBuffer filter; + char *result; + int i; + + res = PQexec(conn, + "SELECT DISTINCT s FROM spock.subscription," + " unnest(sub_replication_sets) AS s ORDER BY 1"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not determine selected replication sets: %s\n"), PQerrorMessage(conn)); + } + if (PQntuples(res) == 0) + { + PQclear(res); + die(_("no subscription references any replication set; cannot " + "verify replication-set equivalence\n")); + } + + filter = createPQExpBuffer(); + appendPQExpBufferStr(filter, "set_name IN ("); + for (i = 0; i < PQntuples(res); i++) + { + char *name = PQgetvalue(res, i, 0); + + appendPQExpBuffer(filter, "%s%s", i > 0 ? ", " : "", + PQescapeLiteral(conn, name, strlen(name))); + } + appendPQExpBufferStr(filter, ")"); + PQclear(res); + + result = pg_strdup(filter->data); + destroyPQExpBuffer(filter); + return result; +} + +/* + * The forwarding path (peer -> source -> n3) and the future direct path + * (peer -> n3) must select exactly the same changes, or a change omitted + * on one path is lost once the direct subscription takes over. Compare + * every selected replication set's fingerprint between the source and + * each peer; reject any mismatch or missing/extra set on either side. + */ +static void +check_replication_set_equivalence(PGconn *source_conn, const char *source_node_name, + PeerNodeInfo *peers, int num_peers) +{ + Oid source_nodeid = get_local_node_id(source_conn); + char *selected_filter = build_selected_set_name_filter(source_conn); + RepsetFingerprintEntry *source_fps; + int num_source_fps; + int i; + + compute_repset_fingerprints(source_conn, source_nodeid, selected_filter, + &source_fps, &num_source_fps); + + for (i = 0; i < num_peers; i++) + { + PGconn *peer_conn; + Oid peer_nodeid; + RepsetFingerprintEntry *peer_fps; + int num_peer_fps; + int j; + + peer_conn = PQconnectdb(peers[i].dsn); + if (PQstatus(peer_conn) != CONNECTION_OK) + die(_("cannot connect to peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(peer_conn)); + + peer_nodeid = get_local_node_id(peer_conn); + compute_repset_fingerprints(peer_conn, peer_nodeid, selected_filter, + &peer_fps, &num_peer_fps); + + /* + * die() exits immediately, so none of the branches below free + * source_fps/peer_fps before calling it -- freeing first and then + * still reading source_fps[j]/peer_fps[j] in the same die() call's + * arguments would be a use-after-free (the process is about to + * exit anyway; nothing else in this file frees before die() either). + */ + for (j = 0; j < num_source_fps; j++) + { + int k; + bool found = false; + + for (k = 0; k < num_peer_fps; k++) + { + if (strcmp(source_fps[j].set_name, peer_fps[k].set_name) != 0) + continue; + found = true; + if (strcmp(source_fps[j].fingerprint, peer_fps[k].fingerprint) != 0) + die(_("replication set \"%s\" differs between the source " + "and peer \"%s\" (membership, flags, columns, row " + "filter, or schema) -- the forwarding path and a " + "future direct peer subscription would not select " + "the same changes, risking permanently lost data " + "on cutover. Reconcile the definitions before " + "retrying.\n"), + source_fps[j].set_name, peers[i].node_name); + break; + } + if (!found) + die(_("replication set \"%s\" exists on the source but not " + "on peer \"%s\"\n"), source_fps[j].set_name, peers[i].node_name); + } + for (j = 0; j < num_peer_fps; j++) + { + int k; + bool found = false; + + for (k = 0; k < num_source_fps; k++) + if (strcmp(peer_fps[j].set_name, source_fps[k].set_name) == 0) + { + found = true; + break; + } + if (!found) + die(_("replication set \"%s\" exists on peer \"%s\" but not " + "on the source\n"), peer_fps[j].set_name, peers[i].node_name); + } + + free_repset_fingerprints(peer_fps, num_peer_fps); + PQfinish(peer_conn); + } + + free_repset_fingerprints(source_fps, num_source_fps); + pg_free(selected_filter); + (void) source_node_name; +} + +/* + * Verify that the source cluster and all peers meet the requirements for + * a bidirectional join: Spock >= 6.0.0 on every node, track_commit_timestamp + * on, no pending DDL, an actual full-mesh subscription graph (not just a + * count), replication-set/schema equivalence across the source and every + * peer, and peer connectivity. + */ +static void +check_preconditions(PGconn *source_conn, const char *source_node_name, + PeerNodeInfo *peers, int num_peers) +{ + PGresult *res; + int i; + int total_nodes = num_peers + 1; + char **all_names = pg_malloc(total_nodes * sizeof(char *)); + + all_names[0] = pg_strdup(source_node_name); + for (i = 0; i < num_peers; i++) + all_names[i + 1] = pg_strdup(peers[i].node_name); + + check_spock_version_at_least_6(source_conn, "source"); /* track_commit_timestamp must be on at the source */ res = PQexec(source_conn, "SHOW track_commit_timestamp"); @@ -352,36 +913,34 @@ check_preconditions(PGconn *source_conn, PeerNodeInfo *peers, int num_peers) die(_("track_commit_timestamp must be on for bidirectional join (source)")); PQclear(res); - /* No pending DDL in spock.queue */ - res = PQexec(source_conn, "SELECT COUNT(*) FROM spock.queue"); + /* + * All outbound replication caught up to the source's current WAL + * position -- i.e. nothing (DDL or data) still in flight to an + * existing peer. spock.queue's row count is not a usable signal here: + * queue_message() (spock_queue.c) only ever inserts into it, so its + * count is monotonically non-decreasing and is never zero on any node + * that has replicated so much as a single DDL statement. + */ + res = PQexec(source_conn, + "SELECT COUNT(*) FROM pg_replication_slots" + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" + " AND (confirmed_flush_lsn IS NULL" + " OR confirmed_flush_lsn < pg_current_wal_lsn())"); if (PQresultStatus(res) != PGRES_TUPLES_OK) - die(_("could not check spock.queue: %s"), + die(_("could not check replication slot lag: %s"), PQerrorMessage(source_conn)); if (strcmp(PQgetvalue(res, 0, 0), "0") != 0) - die(_("pending DDL in spock.queue; wait for replication to drain " - "before joining")); + die(_("source has unreplicated changes pending to an existing peer; " + "wait for replication to drain before joining")); PQclear(res); - /* Full-mesh assertion: subscriptions on source == num_peers */ - res = PQexec(source_conn, "SELECT COUNT(*) FROM spock.subscription"); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - die(_("could not count subscriptions: %s"), - PQerrorMessage(source_conn)); - { - int sub_count = atoi(PQgetvalue(res, 0, 0)); - - if (sub_count != num_peers) - die(_("source node has %d active subscription(s) but %d peer(s) " - "discovered; partial-mesh topologies are not supported"), - sub_count, num_peers); - } - PQclear(res); + /* Full-mesh directed-graph check, from the source's own perspective. */ + check_mesh_edges(source_conn, source_node_name, all_names, total_nodes); /* - * Per-peer: connectivity and track_commit_timestamp. - * - * Spock version is not checked on peers here; peer version checking is - * deferred to the subscription-setup phase. + * Per-peer: connectivity, Spock version, track_commit_timestamp, and + * the full-mesh directed-graph check from each peer's own perspective + * (a mesh that's only complete as seen from the source is not a mesh). */ for (i = 0; i < num_peers; i++) { @@ -395,11 +954,18 @@ check_preconditions(PGconn *source_conn, PeerNodeInfo *peers, int num_peers) die(_("cannot connect to peer \"%s\": %s"), peers[i].node_name, PQerrorMessage(peer_conn)); + check_peer_identity(peer_conn, peers[i].node_name); + check_spock_version_at_least_6(peer_conn, peers[i].node_name); + res = PQexec(peer_conn, "SHOW track_commit_timestamp"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { + /* + * die() exits immediately -- PQerrorMessage() needs peer_conn + * still open, so PQfinish() must not run first (that would be + * a use-after-free when die()'s own formatting reads it). + */ PQclear(res); - PQfinish(peer_conn); die(_("could not check track_commit_timestamp on peer \"%s\": %s"), peers[i].node_name, PQerrorMessage(peer_conn)); } @@ -411,12 +977,165 @@ check_preconditions(PGconn *source_conn, PeerNodeInfo *peers, int num_peers) "(peer \"%s\")"), peers[i].node_name); } PQclear(res); + + check_mesh_edges(peer_conn, peers[i].node_name, all_names, total_nodes); + PQfinish(peer_conn); } + /* Replication-set & schema equivalence: run once the mesh is sound. */ + check_replication_set_equivalence(source_conn, source_node_name, peers, num_peers); + + for (i = 0; i < total_nodes; i++) + pg_free(all_names[i]); + pg_free(all_names); + print_msg(VERBOSITY_NORMAL, _("Preconditions verified.\n")); } +/* + * The physical-backup path runs once per data directory, so it requires + * exactly one spock-configured database on the source instance. Checked + * against actual spock configuration, not --databases/--provider-dsn, + * since the instance can host other unrelated databases. Fails closed: + * any database we cannot inspect aborts the run rather than being + * treated as spock-free. datallowconn is not used to skip databases -- + * a database with connections disabled can still hold spock catalog + * state -- only true templates are excluded. + */ +static void +check_single_spock_database(PGconn *conn, const char *base_prov_connstr, + const char *current_dbname) +{ + PGresult *res; + int i; + PQExpBuffer others = createPQExpBuffer(); + int other_count = 0; + + res = PQexec(conn, "SELECT datname FROM pg_database WHERE NOT datistemplate"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not list databases on source: %s\n"), PQerrorMessage(conn)); + } + + for (i = 0; i < PQntuples(res); i++) + { + char *dbname = PQgetvalue(res, i, 0); + char *db_connstr; + PGconn *db_conn; + PGresult *ext_res; + PGresult *node_res; + + if (strcmp(dbname, current_dbname) == 0) + continue; + + db_connstr = get_connstr((char *) base_prov_connstr, dbname); + db_conn = PQconnectdb(db_connstr); + if (PQstatus(db_conn) != CONNECTION_OK) + { + char *errmsg = pg_strdup(PQerrorMessage(db_conn)); + + PQfinish(db_conn); + PQclear(res); + die(_("--bidirectional requires proving no other database on the " + "source has spock configured, but could not connect to " + "\"%s\" to check: %s\n"), dbname, errmsg); + } + + ext_res = PQexec(db_conn, "SELECT 1 FROM pg_extension WHERE extname = 'spock'"); + if (PQresultStatus(ext_res) != PGRES_TUPLES_OK) + { + char *errmsg = pg_strdup(PQerrorMessage(db_conn)); + + PQclear(ext_res); + PQfinish(db_conn); + PQclear(res); + die(_("--bidirectional requires proving no other database on the " + "source has spock configured, but could not query \"%s\": " + "%s\n"), dbname, errmsg); + } + + if (PQntuples(ext_res) > 0) + { + node_res = PQexec(db_conn, "SELECT 1 FROM spock.local_node"); + if (PQresultStatus(node_res) != PGRES_TUPLES_OK) + { + char *errmsg = pg_strdup(PQerrorMessage(db_conn)); + + PQclear(node_res); + PQclear(ext_res); + PQfinish(db_conn); + PQclear(res); + die(_("--bidirectional requires proving no other database on " + "the source has spock configured, but could not query " + "spock.local_node in \"%s\": %s\n"), dbname, errmsg); + } + + if (PQntuples(node_res) > 0) + { + appendPQExpBuffer(others, "%s%s", other_count ? ", " : "", dbname); + other_count++; + } + PQclear(node_res); + } + PQclear(ext_res); + PQfinish(db_conn); + } + PQclear(res); + + if (other_count > 0) + die(_("--bidirectional requires exactly one spock-configured database " + "on the source instance; also found spock configured on: %s\n"), + others->data); + + destroyPQExpBuffer(others); +} + +/* + * A physical base backup copies native (non-spock) logical subscriptions + * too, which DROP EXTENSION spock doesn't touch. Once n3 is promoted and + * restarted, an enabled native subscription would start consuming from + * its provider as a second, unintended consumer. pg_subscription is a + * shared catalog, so one query sees every database's rows. + */ +static void +check_no_native_subscriptions(PGconn *conn) +{ + PGresult *res; + + res = PQexec(conn, + "SELECT s.subname, d.datname" + " FROM pg_subscription s" + " JOIN pg_database d ON d.oid = s.subdbid" + " WHERE s.subenabled"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check for native logical subscriptions on the " + "source instance: %s\n"), PQerrorMessage(conn)); + } + + if (PQntuples(res) > 0) + { + PQExpBuffer list = createPQExpBuffer(); + int i; + + for (i = 0; i < PQntuples(res); i++) + appendPQExpBuffer(list, "\n - %s (database %s)", + PQgetvalue(res, i, 0), PQgetvalue(res, i, 1)); + + PQclear(res); + die(_("--bidirectional requires no enabled native (non-spock) logical " + "subscriptions anywhere on the source instance -- a physical " + "backup would copy them, and they would start consuming on " + "the new node as an unintended second consumer once " + "promoted: %s\nDisable or drop these subscriptions before " + "retrying.\n"), list->data); + } + PQclear(res); +} + /* * Write the bidirectional state manifest to state->manifest_path * atomically (write to .tmp, then rename). The manifest is a simple @@ -429,7 +1148,6 @@ write_manifest(BidirectionalState *state, const char *subscriber_name, { PQExpBuffer buf = createPQExpBuffer(); char tmp_path[MAXPGPATH]; - FILE *f; int i; snprintf(tmp_path, MAXPGPATH, "%s.tmp", state->manifest_path); @@ -459,6 +1177,16 @@ write_manifest(BidirectionalState *state, const char *subscriber_name, append_json_string(buf, state->source_origin_name); appendPQExpBufferStr(buf, "\",\n"); + appendPQExpBufferStr(buf, " \"source_restore_lsn\": \""); + if (state->source_restore_lsn) + append_json_string(buf, state->source_restore_lsn); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"node_dsn\": \""); + if (state->node_dsn) + append_json_string(buf, state->node_dsn); + appendPQExpBufferStr(buf, "\",\n"); + appendPQExpBufferStr(buf, " \"peers\": [\n"); for (i = 0; i < state->num_peers; i++) { @@ -481,34 +1209,83 @@ write_manifest(BidirectionalState *state, const char *subscriber_name, appendPQExpBufferStr(buf, " \"peer_slot_name\": \""); append_json_string(buf, p->slot_name); - appendPQExpBufferStr(buf, "\"\n"); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBuffer(buf, " \"disabled_sub_created\": %s,\n", + p->disabled_sub_created ? "true" : "false"); + appendPQExpBuffer(buf, " \"slot_created\": %s,\n", + p->slot_created ? "true" : "false"); + appendPQExpBuffer(buf, " \"reverse_sub_created\": %s\n", + p->reverse_sub_created ? "true" : "false"); appendPQExpBufferStr(buf, last ? " }\n" : " },\n"); } appendPQExpBufferStr(buf, " ]\n"); appendPQExpBufferStr(buf, "}\n"); - f = fopen(tmp_path, "w"); - if (f == NULL) - die(_("could not create manifest file \"%s\": %s"), - tmp_path, strerror(errno)); - - if (fwrite(buf->data, 1, buf->len, f) != buf->len) - { - fclose(f); - unlink(tmp_path); - die(_("could not write manifest file \"%s\": %s"), - tmp_path, strerror(errno)); - } - if (fclose(f) != 0) + /* + * The manifest can embed a password (source_dsn, node_dsn), so create + * with mode 0600 up front, not a post-hoc chmod. O_EXCL|O_NOFOLLOW + * refuses to write through a pre-existing file or planted symlink, + * except a leftover .tmp from a previous crashed run. + */ { - unlink(tmp_path); - die(_("could not close manifest file \"%s\": %s"), - tmp_path, strerror(errno)); + int fd; + ssize_t written; + + fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0600); + if (fd < 0 && errno == EEXIST) + { + if (unlink(tmp_path) != 0) + die(_("could not remove stale manifest temp file \"%s\": %s"), + tmp_path, strerror(errno)); + fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0600); + } + if (fd < 0) + die(_("could not create manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + + written = write(fd, buf->data, buf->len); + if (written < 0 || (size_t) written != buf->len) + { + close(fd); + unlink(tmp_path); + die(_("could not write manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + + /* + * fsync, rename, then fsync the directory -- a crash right after + * this returns must not lose the only cleanup record for the + * source slot created just before it. + */ + if (fsync(fd) != 0) + { + close(fd); + unlink(tmp_path); + die(_("could not fsync manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + if (close(fd) != 0) + { + unlink(tmp_path); + die(_("could not close manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + if (rename(tmp_path, state->manifest_path) != 0) + die(_("could not rename manifest to \"%s\": %s"), + state->manifest_path, strerror(errno)); + + /* + * fsync_parent_path() already treats "filesystem doesn't support + * directory fsync" as success internally, so a nonzero return + * here is a genuine failure that can orphan the source slot + * after a crash. Fatal, like the durability steps above. + */ + if (fsync_parent_path(state->manifest_path) != 0) + die(_("could not fsync directory containing \"%s\": %s\n"), + state->manifest_path, strerror(errno)); } - if (rename(tmp_path, state->manifest_path) != 0) - die(_("could not rename manifest to \"%s\": %s"), - state->manifest_path, strerror(errno)); destroyPQExpBuffer(buf); } @@ -536,6 +1313,9 @@ typedef struct ManifestParseState char *peer_dsn; char *peer_sub_name; char *peer_slot_name; + bool peer_disabled_sub_created; + bool peer_slot_created; + bool peer_reverse_sub_created; int peer_capacity; } ManifestParseState; @@ -569,8 +1349,12 @@ manifest_object_end(void *st) s->bidir->peers[i].dsn = s->peer_dsn; s->bidir->peers[i].sub_name = s->peer_sub_name; s->bidir->peers[i].slot_name = s->peer_slot_name; + s->bidir->peers[i].disabled_sub_created = s->peer_disabled_sub_created; + s->bidir->peers[i].slot_created = s->peer_slot_created; + s->bidir->peers[i].reverse_sub_created = s->peer_reverse_sub_created; s->bidir->num_peers++; s->peer_node_name = s->peer_dsn = s->peer_sub_name = s->peer_slot_name = NULL; + s->peer_disabled_sub_created = s->peer_slot_created = s->peer_reverse_sub_created = false; s->in_peer_obj = false; } s->depth--; @@ -617,7 +1401,32 @@ manifest_scalar(void *st, char *token, JsonTokenType tokentype) { ManifestParseState *s = (ManifestParseState *) st; - if (s->cur_field == NULL || tokentype != JSON_TOKEN_STRING) + if (s->cur_field == NULL) + { + pg_free(token); + return JSON_SUCCESS; + } + + /* + * Per-peer creation-state flags are JSON booleans, not strings -- + * handle them before the string-only fields below (which free and + * ignore anything that isn't JSON_TOKEN_STRING). + */ + if (s->in_peer_obj && tokentype != JSON_TOKEN_STRING) + { + bool value = (tokentype == JSON_TOKEN_TRUE); + + if (strcmp(s->cur_field, "disabled_sub_created") == 0) + s->peer_disabled_sub_created = value; + else if (strcmp(s->cur_field, "slot_created") == 0) + s->peer_slot_created = value; + else if (strcmp(s->cur_field, "reverse_sub_created") == 0) + s->peer_reverse_sub_created = value; + pg_free(token); + return JSON_SUCCESS; + } + + if (tokentype != JSON_TOKEN_STRING) { pg_free(token); return JSON_SUCCESS; @@ -636,6 +1445,10 @@ manifest_scalar(void *st, char *token, JsonTokenType tokentype) s->bidir->source_slot_name = token; else if (strcmp(s->cur_field, "source_origin_name") == 0) s->bidir->source_origin_name = token; + else if (strcmp(s->cur_field, "source_restore_lsn") == 0) + s->bidir->source_restore_lsn = token; + else if (strcmp(s->cur_field, "node_dsn") == 0) + s->bidir->node_dsn = token; else pg_free(token); } @@ -735,12 +1548,16 @@ read_manifest(const char *manifest_path, BidirectionalState *state, /* * Idempotently remove bidirectional join state from all reachable nodes. - * Connects to the source and each peer, drops replication slots and - * reverse subscriptions created during a previous join attempt. All - * operations are best-effort: connectivity failures are logged as - * warnings rather than being fatal. + * Connects to the source and each peer; drops replication slots and + * reverse subscriptions created during a previous join attempt. + * Connectivity and drop failures are logged as warnings, not fatal, so + * cleanup attempts every remaining resource -- but each failure is + * tracked, and the function returns true only if every recorded resource + * was confirmed gone. The manifest/sidecar record (the only way to + * retry) is removed only on a true return; an incomplete cleanup keeps + * it and the caller exits non-zero. */ -static void +static bool cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, const char *dbname, const char *source_dsn, bool force_rm_datadir) @@ -749,6 +1566,7 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, PGresult *res; PQExpBuffer query = createPQExpBuffer(); int i; + bool fully_cleaned = true; print_msg(VERBOSITY_NORMAL, _("Cleaning up partial bidirectional join state ...\n")); @@ -756,10 +1574,14 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, source_conn = PQconnectdb(source_dsn); if (PQstatus(source_conn) != CONNECTION_OK) { - print_msg(VERBOSITY_NORMAL, - _("warning: cannot connect to source node; skipping " - "source-side cleanup: %s\n"), - PQerrorMessage(source_conn)); + if (state->source_slot_name && state->source_slot_name[0]) + { + print_msg(VERBOSITY_NORMAL, + _("warning: cannot connect to source node; slot %s " + "may still exist: %s\n"), + state->source_slot_name, PQerrorMessage(source_conn)); + fully_cleaned = false; + } PQfinish(source_conn); source_conn = NULL; } @@ -773,10 +1595,20 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, " WHERE slot_name = '%s'", state->source_slot_name); res = PQexec(source_conn, query->data); - if (PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) > 0) + if (PQresultStatus(res) == PGRES_TUPLES_OK) + { + if (PQntuples(res) > 0) + print_msg(VERBOSITY_NORMAL, + _(" dropped source slot %s\n"), + state->source_slot_name); + } + else + { print_msg(VERBOSITY_NORMAL, - _(" dropped source slot %s\n"), - state->source_slot_name); + _("warning: could not drop source slot %s: %s\n"), + state->source_slot_name, PQerrorMessage(source_conn)); + fully_cleaned = false; + } PQclear(res); } @@ -790,18 +1622,29 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, if (!peer->dsn || !peer->dsn[0]) continue; + /* + * Only attempt to drop -- and only require connectivity for -- + * resources this run actually recorded as created. Slot/sub names + * are deterministic, not per-run unique, so --cleanup must not + * touch a same-named resource from an unrelated join, nor report + * "incomplete" over a peer that was never touched. + */ + if (!peer->slot_created && !peer->reverse_sub_created) + continue; + peer_conn = PQconnectdb(peer->dsn); if (PQstatus(peer_conn) != CONNECTION_OK) { print_msg(VERBOSITY_NORMAL, - _("warning: cannot connect to peer \"%s\"; skipping " - "peer-side cleanup: %s\n"), + _("warning: cannot connect to peer \"%s\"; its slot/" + "subscription may still exist: %s\n"), peer->node_name, PQerrorMessage(peer_conn)); + fully_cleaned = false; PQfinish(peer_conn); continue; } - if (peer->slot_name && peer->slot_name[0]) + if (peer->slot_created && peer->slot_name && peer->slot_name[0]) { printfPQExpBuffer(query, "SELECT pg_drop_replication_slot(slot_name)" @@ -809,25 +1652,49 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, " WHERE slot_name = '%s'", peer->slot_name); res = PQexec(peer_conn, query->data); - if (PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) > 0) + if (PQresultStatus(res) == PGRES_TUPLES_OK) + { + if (PQntuples(res) > 0) + print_msg(VERBOSITY_NORMAL, + _(" dropped peer slot %s on %s\n"), + peer->slot_name, peer->node_name); + } + else + { print_msg(VERBOSITY_NORMAL, - _(" dropped peer slot %s on %s\n"), - peer->slot_name, peer->node_name); + _("warning: could not drop peer slot %s on %s: %s\n"), + peer->slot_name, peer->node_name, + PQerrorMessage(peer_conn)); + fully_cleaned = false; + } PQclear(res); } /* - * Drop the reverse subscription (peer -> new subscriber) if it was - * created during a previous attempt. The sub_drop second argument - * is ifexists=true. + * Drop the reverse subscription (peer -> new subscriber) only if + * this run recorded having created it. The sub_drop second + * argument is ifexists=true, so an absent subscription is not an + * error -- only an actual query failure counts against + * fully_cleaned. */ - snprintf(reverse_sub, sizeof(reverse_sub), "sub_%s_%s", - peer->node_name, subscriber_name); - printfPQExpBuffer(query, - "SELECT spock.sub_drop('%s', true)", - reverse_sub); - res = PQexec(peer_conn, query->data); - PQclear(res); + if (peer->reverse_sub_created) + { + snprintf(reverse_sub, sizeof(reverse_sub), "sub_%s_%s", + peer->node_name, subscriber_name); + printfPQExpBuffer(query, + "SELECT spock.sub_drop(%s, true)", + PQescapeLiteral(peer_conn, reverse_sub, strlen(reverse_sub))); + res = PQexec(peer_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not drop reverse subscription %s on " + "%s: %s\n"), + reverse_sub, peer->node_name, PQerrorMessage(peer_conn)); + fully_cleaned = false; + } + PQclear(res); + } PQfinish(peer_conn); print_msg(VERBOSITY_NORMAL, @@ -839,14 +1706,112 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, destroyPQExpBuffer(query); + /* + * The data directory a partial run may have created via basebackup. + * Never touch it without --force. + */ + if (data_dir != NULL && data_dir[0] && file_exists(data_dir)) + { + if (force_rm_datadir) + { + struct stat st; + + snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); + if (stat(pid_file, &st) == 0) + { + print_msg(VERBOSITY_NORMAL, + _(" stopping postgres in %s before removing it ...\n"), + data_dir); + run_pg_ctl("stop -m fast"); + wait_postmaster_shutdown(); + } + + print_msg(VERBOSITY_NORMAL, + _(" removing data directory %s ...\n"), data_dir); + if (!rmtree(data_dir, true)) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not fully remove data directory " + "%s; remove it manually\n"), data_dir); + fully_cleaned = false; + } + } + else + { + print_msg(VERBOSITY_NORMAL, + _(" data directory %s was left in place; pass --force " + "to remove it, or clean it up manually.\n"), data_dir); + } + } + + if (!fully_cleaned) + { + print_msg(VERBOSITY_NORMAL, + _("Cleanup incomplete: some resource(s) above could not be " + "confirmed removed. Keeping the manifest/sidecar record " + "so --cleanup can be retried.\n")); + return false; + } + + /* + * Every remote/local resource above was confirmed gone; now remove the + * retry record(s) themselves. An unexpected removal failure here + * (anything but ENOENT, i.e. already gone) must also flip + * fully_cleaned -- otherwise the caller reports success and exits 0 + * while a stale record that still references now-removed resources + * lingers on disk, which a later --cleanup could misread as current. + */ if (state->manifest_path && state->manifest_path[0]) { - unlink(state->manifest_path); + if (unlink(state->manifest_path) == 0) + print_msg(VERBOSITY_NORMAL, + _(" removed manifest %s\n"), state->manifest_path); + else if (errno != ENOENT) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not remove manifest %s: %s\n"), + state->manifest_path, strerror(errno)); + fully_cleaned = false; + } + } + + /* + * Also remove any pending-cleanup sidecar, even if it wasn't the file + * that drove this cleanup: a stale one left behind by an earlier run + * whose own sidecar-unlink failed could otherwise be misread as + * current by a later --cleanup once the manifest above is gone, + * reporting resources as still-pending that were, in fact, already + * confirmed removed here. + */ + if (data_dir != NULL && data_dir[0]) + { + char sidecar_path[MAXPGPATH]; + + snprintf(sidecar_path, MAXPGPATH, "%s.spock_bidir_pending.json", data_dir); + if (unlink(sidecar_path) == 0) + print_msg(VERBOSITY_NORMAL, + _(" removed pending sidecar %s\n"), sidecar_path); + else if (errno != ENOENT) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not remove pending sidecar %s: %s\n"), + sidecar_path, strerror(errno)); + fully_cleaned = false; + } + } + + if (!fully_cleaned) + { print_msg(VERBOSITY_NORMAL, - _(" removed manifest %s\n"), state->manifest_path); + _("Cleanup incomplete: the manifest or sidecar record could " + "not be removed even though every resource it tracked " + "was confirmed gone. Retry --cleanup to remove the " + "stale record.\n")); + return false; } print_msg(VERBOSITY_NORMAL, _("Cleanup complete.\n")); + return true; } @@ -884,6 +1849,8 @@ main(int argc, char **argv) char *extra_basebackup_args = NULL; BidirectionalState bidir = {0}; char bidir_manifest_path[MAXPGPATH] = {0}; + char bidir_pending_path[MAXPGPATH] = {0}; + CatalogCapture capture = {0}; static struct option long_options[] = { {"subscriber-name", required_argument, NULL, 'n'}, @@ -904,13 +1871,16 @@ main(int argc, char **argv) {"stall-timeout", required_argument, NULL, 13}, {"max-wait", required_argument, NULL, 14}, {"cleanup", no_argument, NULL, 15}, + {"force", no_argument, NULL, 16}, {NULL, 0, NULL, 0} }; argv0 = argv[0]; progname = get_progname(argv[0]); + pg_logging_init(argv[0]); start_time = time(NULL); signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); /* check for --help */ if (argc > 1) @@ -1009,6 +1979,9 @@ main(int argc, char **argv) case 15: bidir.cleanup_mode = true; break; + case 16: + bidir.force_cleanup = true; + break; default: fprintf(stderr, _("Unknown option\n")); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); @@ -1036,11 +2009,18 @@ main(int argc, char **argv) if (bidir.cleanup_mode && !bidir.enabled) die(_("--cleanup requires --bidirectional.\n")); + if (bidir.force_cleanup && !bidir.cleanup_mode) + die(_("--force requires --cleanup.\n")); + if (!bidir.cleanup_mode && (!base_prov_connstr || !strlen(base_prov_connstr))) die(_("Provider connection string must be specified.\n")); - if (!bidir.enabled && !bidir.cleanup_mode && + if (!bidir.cleanup_mode && (!base_sub_connstr || !strlen(base_sub_connstr))) - die(_("Subscriber connection string must be specified.\n")); + die(_("Subscriber connection string must be specified: --subscriber-dsn " + "is used both for the tool's own connection to the newly " + "created node and, with --bidirectional, as the externally-" + "reachable address registered via spock.node_create() for " + "peers to connect back to it.\n")); if (apply_delay < 0) die(_("Apply delay cannot be negative.\n")); @@ -1057,6 +2037,14 @@ main(int argc, char **argv) snprintf(bidir_manifest_path, MAXPGPATH, "%s/spock_bidirectional_manifest.json", data_dir); bidir.manifest_path = bidir_manifest_path; + /* + * Sidecar path for the source slot orphan-protection record (see + * the write near source-slot creation below) -- lives next to, not + * inside, data_dir, since data_dir must still be empty when this is + * first written (pg_basebackup requires an empty target directory). + */ + snprintf(bidir_pending_path, MAXPGPATH, + "%s.spock_bidir_pending.json", data_dir); if (bidir.stall_timeout == 0) bidir.stall_timeout = 600; } @@ -1068,13 +2056,23 @@ main(int argc, char **argv) char *db = NULL; char *src_dsn = NULL; - if (!read_manifest(bidir.manifest_path, &bidir, &sub_name, &db, &src_dsn)) - { - fprintf(stderr, _("No manifest found at %s; nothing to clean up.\n"), - bidir.manifest_path); - exit(0); - } - cleanup_partial_state(&bidir, sub_name, db, src_dsn, false); + if (read_manifest(bidir.manifest_path, &bidir, &sub_name, &db, &src_dsn)) + exit(cleanup_partial_state(&bidir, sub_name, db, src_dsn, + bidir.force_cleanup) ? 0 : 1); + + /* + * No full manifest -- basebackup may never have completed. Fall + * back to the pending-cleanup sidecar written right after source + * slot creation, so a slot orphaned by a failed/interrupted backup + * is still reachable by --cleanup. + */ + if (read_manifest(bidir_pending_path, &bidir, &sub_name, &db, &src_dsn)) + /* cleanup_partial_state() removes the sidecar itself on success. */ + exit(cleanup_partial_state(&bidir, sub_name, db, src_dsn, + bidir.force_cleanup) ? 0 : 1); + + fprintf(stderr, _("No manifest found at %s or %s; nothing to clean up.\n"), + bidir.manifest_path, bidir_pending_path); exit(0); } @@ -1099,6 +2097,18 @@ main(int argc, char **argv) database_list[0] = dbname; } + /* + * Single database only: all join state is per-database, and the + * physical-backup/recovery path operates on one data directory. + * Reject a multi-database request rather than silently joining only + * database_list[0]. Separate from check_single_spock_database() + * below, which checks the instance for spock on other databases. + */ + if (bidir.enabled && n_databases > 1) + die(_("--bidirectional supports a single database only; " + "%d were named via --databases/--provider-dsn.\n"), + n_databases); + slot_names = palloc(n_databases * sizeof(char *)); /* @@ -1153,26 +2163,66 @@ main(int argc, char **argv) remote_info = get_remote_info(provider_conn); /* - * --bidirectional: discover peers, verify preconditions, write the - * manifest, then exit. The rest of the join resumes from this - * manifest once the physical backup has been taken and the - * subscriber is running. + * --bidirectional: discover peers, verify preconditions, then + * continue into the physical-backup pipeline below using the + * "sub__" slot naming convention. Manifest + * write is deferred until after the basebackup; see the comment + * there. */ if (bidir.enabled) { + PQExpBuffer sub_name_buf = createPQExpBuffer(); + char *source_sub_name; + bidir.num_peers = discover_peer_nodes(provider_conn, remote_info->node_name, subscriber_name, db, &bidir.peers); - check_preconditions(provider_conn, bidir.peers, bidir.num_peers); - write_manifest(&bidir, subscriber_name, db, base_prov_connstr); + check_preconditions(provider_conn, remote_info->node_name, + bidir.peers, bidir.num_peers); + check_single_spock_database(provider_conn, base_prov_connstr, db); + check_no_native_subscriptions(provider_conn); + use_existing_data_dir = check_data_dir(data_dir, remote_info); + if (use_existing_data_dir) + { + char *local_sysid = read_sysid(data_dir); + bool mismatch = strcmp(remote_info->sysid, local_sysid) != 0; + free(local_sysid); + if (mismatch) + die(_("Subscriber data directory is not basebackup of remote node.\n")); + } + + appendPQExpBuffer(sub_name_buf, "sub_%s_%s", + subscriber_name, remote_info->node_name); + source_sub_name = pg_strdup(sub_name_buf->data); + destroyPQExpBuffer(sub_name_buf); + print_msg(VERBOSITY_NORMAL, - _("Bidirectional plumbing complete: %d peer(s) discovered, " - "preconditions OK, manifest written to %s.\n"), - bidir.num_peers, bidir.manifest_path); + _("Creating source replication slot in database %s ...\n"), db); + bidir.source_slot_name = initialize_replication_slot(provider_conn, + remote_info->dbname, + remote_info->node_name, + source_sub_name, + drop_slot_if_exists); + bidir.source_origin_name = pg_strdup(bidir.source_slot_name); + pg_free(source_sub_name); + + /* + * Persist a pending-cleanup record now, before the base backup + * even starts: the source slot above already exists on the + * remote node, and a failed/interrupted backup would otherwise + * orphan it with nothing for --cleanup to find (the real + * manifest can't be written yet -- data_dir must stay empty for + * pg_basebackup). Superseded and removed once the real + * manifest is written below. + */ + bidir.manifest_path = bidir_pending_path; + write_manifest(&bidir, subscriber_name, db, base_prov_connstr); + bidir.manifest_path = bidir_manifest_path; + PQfinish(provider_conn); provider_conn = NULL; - exit(0); + break; /* single-database only, enforced above */ } /* only need to do this piece once */ @@ -1217,6 +2267,26 @@ main(int argc, char **argv) extra_basebackup_args); snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); + /* + * Manifest write is deferred until here: pg_basebackup requires an + * empty target directory, and a manifest file in data_dir earlier + * would make it look non-empty. The pending-cleanup sidecar written + * right after source slot creation covers the gap between then and + * now; it's superseded by the real manifest and removed below. + */ + if (bidir.enabled) + { + write_manifest(&bidir, subscriber_name, database_list[0], base_prov_connstr); + if (unlink(bidir_pending_path) != 0 && errno != ENOENT) + print_msg(VERBOSITY_NORMAL, + _("warning: could not remove superseded pending sidecar " + "%s: %s\n"), bidir_pending_path, strerror(errno)); + print_msg(VERBOSITY_NORMAL, + _("Bidirectional plumbing complete: %d peer(s) discovered, " + "source slot created, manifest written to %s.\n"), + bidir.num_peers, bidir.manifest_path); + } + restore_point_name = generate_restore_point_name(); print_msg(VERBOSITY_NORMAL, _("Creating restore point \"%s\" on remote node ...\n"), @@ -1251,12 +2321,22 @@ main(int argc, char **argv) /* * Start subscriber node with spock disabled, and wait until it starts * accepting connections which means it has caught up to the restore point. + * + * TODO: for --bidirectional this node should be network-quarantined + * (private socket/listen address, or a restrictive pg_hba.conf) from + * this first startup through the end of the join -- spock.readonly = + * 'local' (set later) blocks writes but not reads or peer probes. Not + * implemented: --subscriber-dsn must be directly reachable, and the + * tool's own connections use that same DSN throughout, so restricting + * listen_addresses here would also lock the tool itself out. */ pg_ctl_ret = run_pg_ctl("start -l \"spock_create_subscriber_postgres.log\" -o \"-c shared_preload_libraries=''\""); if (pg_ctl_ret != 0) die(_("Postgres startup for restore point catchup failed with %d. See spock_create_subscriber_postgres.log."), pg_ctl_ret); - wait_primary_connection(sub_connstr); + wait_primary_connection(sub_connstr, + bidir.enabled ? bidir.stall_timeout : 0, + bidir.enabled ? bidir.max_wait : 0); /* * Clean any per-node data that were copied by pg_basebackup. @@ -1264,22 +2344,114 @@ main(int argc, char **argv) print_msg(VERBOSITY_VERBOSE, _("Removing old spock configuration ...\n")); - for (dbnum = 0; dbnum < n_databases; dbnum++) + if (bidir.enabled) { - char *db = database_list[dbnum]; + Oid source_nodeid; + char *expected_sysid; + + /* + * Give n3 its own permanent identity now, right after promotion + * and before any catalog mutation: a physical backup preserves + * the source's system identifier, which risks stray WAL from one + * cluster being mistaken for the other's, and until reset makes + * system_identifier useless for proving a connection actually + * reaches n3 rather than the source. + */ + print_msg(VERBOSITY_NORMAL, + _("Assigning a new system identifier to the subscriber node...\n")); + pg_ctl_ret = run_pg_ctl("stop"); + if (pg_ctl_ret != 0) + die(_("Postgres stop before resetting system identifier failed with %d."), pg_ctl_ret); + wait_postmaster_shutdown(); - sub_connstr = get_connstr(base_sub_connstr, db); + { + sigset_t block_set, + old_set; + + /* + * Neither step below is safe to interrupt -- both write + * pg_control/WAL directly, and signal_handler() -> die() is + * not async-signal-safe. A signal landing mid-write could + * corrupt pg_control with no repair short of --cleanup + * --force. Block both signals across this pair of calls; + * any that arrives is deferred until right after. + */ + sigemptyset(&block_set); + sigaddset(&block_set, SIGINT); + sigaddset(&block_set, SIGTERM); + sigprocmask(SIG_BLOCK, &block_set, &old_set); + + expected_sysid = reset_subscriber_sysid(data_dir); + run_pg_resetwal(data_dir); + + sigprocmask(SIG_SETMASK, &old_set, NULL); + } - if (!sub_connstr || !strlen(sub_connstr)) - die(_("Subscriber connection string is not valid.\n")); + pg_ctl_ret = run_pg_ctl("start -l \"spock_create_subscriber_postgres.log\" -o \"-c shared_preload_libraries=''\""); + if (pg_ctl_ret != 0) + die(_("Postgres startup after resetting system identifier failed with %d."), pg_ctl_ret); + wait_postmaster_connection(sub_connstr); subscriber_conn = connectdb(sub_connstr); - remove_unwanted_data(subscriber_conn); + + /* + * --subscriber-dsn is expected to point directly at this node; + * verify that cheaply before running anything destructive, rather + * than trusting it silently. Now that n3 has just been given its + * own system identifier above, a straightforward comparison is a + * valid proof the connection reaches n3 and not the source or any + * other server -- unlike before the reset, nothing else could + * share it. + */ + { + PGresult *sysid_res = PQexec(subscriber_conn, "SELECT system_identifier FROM pg_control_system()"); + bool mismatch; + + if (PQresultStatus(sysid_res) != PGRES_TUPLES_OK || PQntuples(sysid_res) != 1) + { + PQclear(sysid_res); + die(_("could not verify --subscriber-dsn connects to this node: %s\n"), + PQerrorMessage(subscriber_conn)); + } + mismatch = strcmp(PQgetvalue(sysid_res, 0, 0), expected_sysid) != 0; + PQclear(sysid_res); + if (mismatch) + die(_("--subscriber-dsn does not connect to the node at \"%s\": " + "system identifier mismatch. This can happen if the DSN " + "routes to the source node or another server; refusing " + "to run catalog operations against it.\n"), data_dir); + } + free(expected_sysid); + + /* Capture repset/table/sequence state before the catalog strip. */ + source_nodeid = get_local_node_id(subscriber_conn); + capture_catalog_state(subscriber_conn, source_nodeid, &capture); + + /* Drop all origins, then guarded DROP EXTENSION. */ + remove_unwanted_data_bidir(subscriber_conn, &capture); + PQfinish(subscriber_conn); subscriber_conn = NULL; } + else + { + for (dbnum = 0; dbnum < n_databases; dbnum++) + { + char *db = database_list[dbnum]; + + sub_connstr = get_connstr(base_sub_connstr, db); + + if (!sub_connstr || !strlen(sub_connstr)) + die(_("Subscriber connection string is not valid.\n")); + + subscriber_conn = connectdb(sub_connstr); + remove_unwanted_data(subscriber_conn); + PQfinish(subscriber_conn); + subscriber_conn = NULL; + } + } - /* Stop Postgres so we can reset system id and start it with spock loaded. */ + /* Stop Postgres so we can start it again with spock (shared_preload_libraries) loaded. */ pg_ctl_ret = run_pg_ctl("stop"); if (pg_ctl_ret != 0) die(_("Postgres stop after restore point catchup failed with %d. See spock_create_subscriber_postgres.log."), pg_ctl_ret); @@ -1287,7 +2459,7 @@ main(int argc, char **argv) /* * Start the node again, now with spock active so that we can start the - * logical replication. This is final start, so don't log to to special log + * logical replication. This is final start, so don't log to to special log * file anymore. */ print_msg(VERBOSITY_NORMAL, @@ -1296,43 +2468,110 @@ main(int argc, char **argv) pg_ctl_ret = run_pg_ctl("start"); if (pg_ctl_ret != 0) die(_("Postgres restart with spock enabled failed with %d."), pg_ctl_ret); - wait_postmaster_connection(base_sub_connstr); + wait_postmaster_connection(bidir.enabled ? sub_connstr : base_sub_connstr); - for (dbnum = 0; dbnum < n_databases; dbnum++) + if (bidir.enabled) { - char *db = database_list[dbnum]; - - sub_connstr = get_connstr(base_sub_connstr, db); - prov_connstr = get_connstr(base_prov_connstr, db); + char *db = database_list[0]; subscriber_conn = connectdb(sub_connstr); - /* Create the extension. */ print_msg(VERBOSITY_VERBOSE, _("Creating spock extension for database %s...\n"), db); install_extension(subscriber_conn, "spock"); /* - * Create the identifier which is setup with the position to which we - * already caught up using physical replication. + * Create the local node, then immediately go read-only -- no + * window where n3 is reachable/writable before that lands. No + * origin creation here; the catchup subscription creates it + * later. + * + * dsn is --subscriber-dsn (sub_connstr) -- the externally-reachable + * address other nodes use to connect back, not a separate + * --node-dsn option. */ - print_msg(VERBOSITY_VERBOSE, - _("Creating replication origin for database %s...\n"), db); - initialize_replication_origin(subscriber_conn, slot_names[dbnum], remote_lsn); + print_msg(VERBOSITY_NORMAL, _("Creating local Spock node \"%s\"...\n"), + subscriber_name); + { + PQExpBuffer nodequery = createPQExpBuffer(); + PGresult *res; + + printfPQExpBuffer(nodequery, + "SELECT spock.node_create(node_name := %s, dsn := %s)", + PQescapeLiteral(subscriber_conn, subscriber_name, + strlen(subscriber_name)), + PQescapeLiteral(subscriber_conn, sub_connstr, + strlen(sub_connstr))); + res = PQexec(subscriber_conn, nodequery->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not create local node: %s\n"), + PQerrorMessage(subscriber_conn)); + } + PQclear(res); + destroyPQExpBuffer(nodequery); + } - /* - * And finally add the node to the cluster. - */ - print_msg(VERBOSITY_NORMAL, _("Creating subscriber %s for database %s...\n"), - subscriber_name, db); - print_msg(VERBOSITY_VERBOSE, _("Replication sets: %s\n"), replication_sets); + print_msg(VERBOSITY_NORMAL, _("Setting spock.readonly = 'local'...\n")); + set_readonly_local(subscriber_conn); + + /* Restore what was captured before the catalog strip. */ + print_msg(VERBOSITY_NORMAL, _("Restoring replication set state...\n")); + restore_replication_sets(subscriber_conn, &capture); - spock_subscribe(subscriber_conn, subscriber_name, sub_connstr, - prov_connstr, replication_sets, apply_delay, - force_text_transfer); + bidir.source_restore_lsn = pg_strdup(remote_lsn); + bidir.node_dsn = sub_connstr; + write_manifest(&bidir, subscriber_name, db, base_prov_connstr); PQfinish(subscriber_conn); subscriber_conn = NULL; + + print_msg(VERBOSITY_NORMAL, + _("Bidirectional join: physical backup, catalog strip, and " + "replication set restore complete. Node \"%s\" is " + "read-only pending the catchup subscription (a later " + "release).\n"), + subscriber_name); + } + else + { + for (dbnum = 0; dbnum < n_databases; dbnum++) + { + char *db = database_list[dbnum]; + + sub_connstr = get_connstr(base_sub_connstr, db); + prov_connstr = get_connstr(base_prov_connstr, db); + + subscriber_conn = connectdb(sub_connstr); + + /* Create the extension. */ + print_msg(VERBOSITY_VERBOSE, + _("Creating spock extension for database %s...\n"), db); + install_extension(subscriber_conn, "spock"); + + /* + * Create the identifier which is setup with the position to which we + * already caught up using physical replication. + */ + print_msg(VERBOSITY_VERBOSE, + _("Creating replication origin for database %s...\n"), db); + initialize_replication_origin(subscriber_conn, slot_names[dbnum], remote_lsn); + + /* + * And finally add the node to the cluster. + */ + print_msg(VERBOSITY_NORMAL, _("Creating subscriber %s for database %s...\n"), + subscriber_name, db); + print_msg(VERBOSITY_VERBOSE, _("Replication sets: %s\n"), replication_sets); + + spock_subscribe(subscriber_conn, subscriber_name, sub_connstr, + prov_connstr, replication_sets, apply_delay, + force_text_transfer); + + PQfinish(subscriber_conn); + subscriber_conn = NULL; + } } /* If user does not want the node to be running at the end, stop it. */ @@ -1367,7 +2606,10 @@ usage(void) printf(_(" pg_basebackup -X stream command\n")); printf(_(" --databases optional list of databases to replicate\n")); printf(_(" -n, --subscriber-name=NAME name of the newly created subscriber\n")); - printf(_(" --subscriber-dsn=CONNSTR connection string to the newly created subscriber\n")); + printf(_(" --subscriber-dsn=CONNSTR connection string to the newly created subscriber;\n")); + printf(_(" with --bidirectional, also the externally-\n")); + printf(_(" reachable address peers use to connect back\n")); + printf(_(" to this node once joined (required)\n")); printf(_(" --provider-dsn=CONNSTR connection string to the provider\n")); printf(_(" --replication-sets=SETS comma separated list of replication set names\n")); printf(_(" --apply-delay=DELAY apply delay in seconds (by default 0)\n")); @@ -1383,6 +2625,19 @@ usage(void) printf(_(" --hba-conf path to the new pg_hba.conf\n")); printf(_(" --postgresql-conf path to the new postgresql.conf\n")); printf(_(" --recovery-conf path to the template recovery configuration\n")); + printf(_("\nBidirectional join (joins an existing multi-master cluster):\n")); + printf(_(" --bidirectional enable bidirectional join plumbing\n")); + printf(_(" --stall-timeout=SECS once PostgreSQL accepts connections, seconds of no\n")); + printf(_(" replay progress before giving up (default 600); does\n")); + printf(_(" not bound PostgreSQL's own startup\n")); + printf(_(" --max-wait=SECS hard ceiling on post-connection catchup wait, seconds\n")); + printf(_(" (default: unbounded); does not bound PostgreSQL's own\n")); + printf(_(" startup\n")); + printf(_(" --cleanup idempotently remove partial join state and exit\n")); + printf(_(" --force with --cleanup, also remove the data directory\n")); + printf(_("\nDuring the join, this node must be network-quarantined (private address /\n")); + printf(_("restrictive pg_hba.conf) by the operator -- via --hba-conf/--postgresql-conf --\n")); + printf(_("until the join completes; the tool does not manage this for you.\n")); } /* @@ -1432,7 +2687,7 @@ print_msg(VerbosityLevelEnum level, const char *fmt,...) /* * Start pg_ctl with given argument(s) - used to start/stop postgres * - * Returns the exit code reported by pg_ctl. If pg_ctl exits due to a + * Returns the exit code reported by pg_ctl. If pg_ctl exits due to a * signal this call will die and not return. */ static int @@ -1442,7 +2697,7 @@ run_pg_ctl(const char *arg) PQExpBuffer cmd = createPQExpBuffer(); char *exec_path = find_other_exec_or_die(argv0, "pg_ctl"); - appendPQExpBuffer(cmd, "%s %s -D \"%s\"", exec_path, arg, data_dir); + appendPQExpBuffer(cmd, "\"%s\" %s -D \"%s\"", exec_path, arg, data_dir); /* Run pg_ctl in silent mode unless we run in debug mode. */ if (verbosity < VERBOSITY_DEBUG) @@ -1496,7 +2751,14 @@ run_basebackup(const char *provider_connstr, const char *data_dir, PQExpBuffer cmd = createPQExpBuffer(); char *exec_path = find_other_exec_or_die(argv0, "pg_basebackup"); - appendPQExpBuffer(cmd, "%s -D \"%s\" -d \"%s\" -X s -P", exec_path, data_dir, provider_connstr); + /* + * -c fast forces an immediate checkpoint. Without it, pg_basebackup + * requests the default "spread" checkpoint, which paces itself against + * checkpoint_timeout (5 minutes by default) regardless of how little + * data needs flushing -- an unpredictable, unnecessary stall for a + * tool whose entire job is this one backup. + */ + appendPQExpBuffer(cmd, "\"%s\" -D \"%s\" -d \"%s\" -X s -c fast -P", exec_path, data_dir, provider_connstr); /* Run pg_basebackup in verbose mode if we are running in verbose mode. */ if (verbosity >= VERBOSITY_VERBOSE) @@ -1672,115 +2934,707 @@ initialize_replication_slot(PGconn *conn, char *dbname, static RemoteInfo * get_remote_info(PGconn* conn) { - RemoteInfo *ri = (RemoteInfo *)pg_malloc0(sizeof(RemoteInfo)); - PGresult *res; + RemoteInfo *ri = (RemoteInfo *)pg_malloc0(sizeof(RemoteInfo)); + PGresult *res; + + if (!extension_exists(conn, "spock")) + die(_("The remote node is not configured as a spock provider.\n")); + + res = PQexec(conn, "SELECT node_id, node_name, sysid, dbname, replication_sets FROM spock.node_info()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not fetch remote node info: %s\n"), PQerrorMessage(conn)); + + /* No nodes found? */ + if (PQntuples(res) == 0) + die(_("The remote database is not configured as a spock node.\n")); + + if (PQntuples(res) > 1) + die(_("The remote database has multiple nodes configured. That is not supported with current version of spock.\n")); + +#define atooid(x) ((Oid) strtoul((x), NULL, 10)) + + ri->nodeid = atooid(PQgetvalue(res, 0, 0)); + ri->node_name = pstrdup(PQgetvalue(res, 0, 1)); + ri->sysid = pstrdup(PQgetvalue(res, 0, 2)); + ri->dbname = pstrdup(PQgetvalue(res, 0, 3)); + ri->replication_sets = pstrdup(PQgetvalue(res, 0, 4)); + + PQclear(res); + + return ri; +} + +/* + * Check if extension exists. + */ +static bool +extension_exists(PGconn *conn, const char *extname) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + bool ret; + + printfPQExpBuffer(query, "SELECT 1 FROM pg_catalog.pg_extension WHERE extname = %s;", + PQescapeLiteral(conn, extname, strlen(extname))); + res = PQexec(conn, query->data); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("Could not read extension info: %s\n"), PQerrorMessage(conn)); + } + + ret = PQntuples(res) == 1; + + PQclear(res); + destroyPQExpBuffer(query); + + return ret; +} + +/* + * Create extension. + */ +static void +install_extension(PGconn *conn, const char *extname) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + + printfPQExpBuffer(query, "CREATE EXTENSION IF NOT EXISTS %s;", + PQescapeIdentifier(conn, extname, strlen(extname))); + res = PQexec(conn, query->data); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + PQclear(res); + die(_("Could not install %s extension: %s\n"), extname, PQerrorMessage(conn)); + } + + PQclear(res); + destroyPQExpBuffer(query); +} + +/* + * Clean all the data that was copied from remote node but we don't + * want it here (currently shared security labels and replication identifiers). + */ +static void +remove_unwanted_data(PGconn *conn) +{ + PGresult *res; + + /* + * Remove replication identifiers (9.4 will get them removed by dropping + * the extension later as we emulate them there). + */ + res = PQexec(conn, "SELECT pg_replication_origin_drop(external_id) FROM pg_replication_origin_status;"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("Could not remove existing replication origins: %s\n"), PQerrorMessage(conn)); + } + PQclear(res); + + res = PQexec(conn, "DROP EXTENSION spock CASCADE;"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + die(_("Could not clean the spock extension, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); +} + +/* + * Return the connected node's own local node id, from spock.local_node -- + * a plain catalog table, so it works even with spock's shared memory not + * loaded (e.g. before DROP EXTENSION, while spock is disabled). + */ +static Oid +get_local_node_id(PGconn *conn) +{ + PGresult *res; + Oid nodeid; + + res = PQexec(conn, "SELECT node_id FROM spock.local_node"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("could not determine source local node id: %s\n"), + PQerrorMessage(conn)); + } + nodeid = (Oid) strtoul(PQgetvalue(res, 0, 0), NULL, 10); + PQclear(res); + return nodeid; +} + +/* + * Capture replication-set definitions, table memberships, and sequence + * state from the local catalog before DROP EXTENSION removes it -- this + * reflects what was replicated at backup time, unlike querying the live + * source afterward. Utility-side memory only; never written to the + * manifest. + */ +static void +capture_catalog_state(PGconn *conn, Oid source_nodeid, CatalogCapture *capture) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + int i; + + memset(capture, 0, sizeof(*capture)); + + /* Replication set definitions owned by the source node. */ + printfPQExpBuffer(query, + "SELECT set_name, replicate_insert, replicate_update," + " replicate_delete, replicate_truncate" + " FROM spock.replication_set" + " WHERE set_nodeid = %u", + source_nodeid); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not capture replication set definitions: %s\n"), + PQerrorMessage(conn)); + } + + capture->num_repsets = PQntuples(res); + capture->repsets = pg_malloc0(capture->num_repsets * sizeof(RepsetCapture)); + for (i = 0; i < capture->num_repsets; i++) + { + capture->repsets[i].set_name = pg_strdup(PQgetvalue(res, i, 0)); + capture->repsets[i].replicate_insert = (PQgetvalue(res, i, 1)[0] == 't'); + capture->repsets[i].replicate_update = (PQgetvalue(res, i, 2)[0] == 't'); + capture->repsets[i].replicate_delete = (PQgetvalue(res, i, 3)[0] == 't'); + capture->repsets[i].replicate_truncate = (PQgetvalue(res, i, 4)[0] == 't'); + } + PQclear(res); + + /* Table memberships across all of the source's sets. */ + printfPQExpBuffer(query, + "SELECT rs.set_name, rts.set_reloid::regclass AS qualified_table," + " rts.set_att_list AS columns," + " pg_get_expr(rts.set_row_filter, rts.set_reloid) AS row_filter" + " FROM spock.replication_set_table rts" + " JOIN spock.replication_set rs ON rts.set_id = rs.set_id" + " WHERE rs.set_nodeid = %u", + source_nodeid); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not capture replication set table memberships: %s\n"), + PQerrorMessage(conn)); + } + + capture->num_tables = PQntuples(res); + capture->tables = pg_malloc0(capture->num_tables * sizeof(RepsetTableCapture)); + for (i = 0; i < capture->num_tables; i++) + { + capture->tables[i].set_name = pg_strdup(PQgetvalue(res, i, 0)); + capture->tables[i].qualified_table = pg_strdup(PQgetvalue(res, i, 1)); + capture->tables[i].columns = PQgetisnull(res, i, 2) ? NULL : + pg_strdup(PQgetvalue(res, i, 2)); + capture->tables[i].row_filter = PQgetisnull(res, i, 3) ? NULL : + pg_strdup(PQgetvalue(res, i, 3)); + } + PQclear(res); + + /* + * Sequences and the sets they belong to (a sequence can be in more + * than one set, so this is captured per-membership like table rows, + * not deduplicated by sequence). + */ + printfPQExpBuffer(query, + "SELECT rs.set_name, rss.set_seqoid::regclass" + " FROM spock.replication_set_seq rss" + " JOIN spock.replication_set rs ON rss.set_id = rs.set_id" + " WHERE rs.set_nodeid = %u", + source_nodeid); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not capture replicated sequence list: %s\n"), + PQerrorMessage(conn)); + } + + capture->num_sequences = PQntuples(res); + capture->sequences = pg_malloc0(capture->num_sequences * sizeof(SequenceCapture)); + for (i = 0; i < capture->num_sequences; i++) + { + PQExpBuffer seq_query = createPQExpBuffer(); + PGresult *seq_res; + + capture->sequences[i].set_name = pg_strdup(PQgetvalue(res, i, 0)); + capture->sequences[i].qualified_seq = pg_strdup(PQgetvalue(res, i, 1)); + + /* + * Read last_value/is_called directly off the sequence relation + * (standard technique) rather than pg_sequence_last_value(), which + * conflates "never called" with is_called=false and loses the + * distinction setval()'s third argument needs to restore exactly. + */ + printfPQExpBuffer(seq_query, "SELECT last_value, is_called FROM %s", + capture->sequences[i].qualified_seq); + seq_res = PQexec(conn, seq_query->data); + if (PQresultStatus(seq_res) != PGRES_TUPLES_OK) + { + PQclear(seq_res); + destroyPQExpBuffer(seq_query); + die(_("could not read sequence state for \"%s\": %s\n"), + capture->sequences[i].qualified_seq, PQerrorMessage(conn)); + } + + capture->sequences[i].last_value = strtoll(PQgetvalue(seq_res, 0, 0), NULL, 10); + capture->sequences[i].is_called = (PQgetvalue(seq_res, 0, 1)[0] == 't'); + + PQclear(seq_res); + destroyPQExpBuffer(seq_query); + } + PQclear(res); + + destroyPQExpBuffer(query); + + print_msg(VERBOSITY_VERBOSE, + _("Captured %d replication set(s), %d table membership(s), " + "%d sequence(s) before catalog strip.\n"), + capture->num_repsets, capture->num_tables, capture->num_sequences); +} + +/* + * Bidirectional-mode catalog strip: drop ALL replication origins (not + * just ones with a status row, unlike remove_unwanted_data()), then + * guard DROP EXTENSION ... CASCADE with a one-hop pg_depend inventory -- + * any non-spock object depending on a spock member would otherwise be + * silently collaterally dropped. + */ +static void +remove_unwanted_data_bidir(PGconn *conn, CatalogCapture *capture) +{ + PGresult *res; - if (!extension_exists(conn, "spock")) - die(_("The remote node is not configured as a spock provider.\n")); + (void) capture; /* must already be populated before this runs */ - res = PQexec(conn, "SELECT node_id, node_name, sysid, dbname, replication_sets FROM spock.node_info()"); + /* + * Drop all replication origins copied by the basebackup. + * pg_replication_origin is a cluster-wide (not per-database) catalog, + * so this is scoped to spock's own "spk_..." naming convention + * (gen_slot_name(), shared with slot names) rather than dropping every + * row -- an unrelated database on the same instance with its own + * (non-spock) logical replication would otherwise lose its origins too. + */ + res = PQexec(conn, + "SELECT pg_replication_origin_drop(roname)" + " FROM pg_replication_origin" + " WHERE roname LIKE 'spk\\_%' ESCAPE '\\'"); if (PQresultStatus(res) != PGRES_TUPLES_OK) - die(_("could not fetch remote node info: %s\n"), PQerrorMessage(conn)); - - /* No nodes found? */ - if (PQntuples(res) == 0) - die(_("The remote database is not configured as a spock node.\n")); - - if (PQntuples(res) > 1) - die(_("The remote database has multiple nodes configured. That is not supported with current version of spock.\n")); + { + PQclear(res); + die(_("could not remove existing replication origins: %s\n"), + PQerrorMessage(conn)); + } + PQclear(res); -#define atooid(x) ((Oid) strtoul((x), NULL, 10)) + /* Guard against CASCADE collaterally dropping user objects. */ + res = PQexec(conn, + "WITH spock_ext AS (" + " SELECT oid FROM pg_extension WHERE extname = 'spock'" + "), ext_members AS (" + " SELECT classid, objid FROM pg_depend, spock_ext" + " WHERE refclassid = 'pg_extension'::regclass" + " AND refobjid = spock_ext.oid" + " AND deptype = 'e'" + "), spock_members AS (" + /* + * Extension members proper (tables, views, functions, ...) + * plus anything with an INTERNAL ('i') or AUTO ('a') + * dependency on one of them -- a view's own rules use 'i', + * while a table's own constraints (CHECK, FK, ...) use + * 'a'; both are linked to their owning relation this way, + * not directly to the extension, but are just as much + * spock's own objects. + */ + " SELECT classid, objid FROM ext_members" + " UNION" + " SELECT d.classid, d.objid FROM pg_depend d" + " JOIN ext_members m ON d.refclassid = m.classid AND d.refobjid = m.objid" + " WHERE d.deptype IN ('i', 'a')" + ")" + "SELECT DISTINCT pg_describe_object(d.classid, d.objid, d.objsubid)" + " FROM pg_depend d" + " JOIN spock_members m ON d.refclassid = m.classid AND d.refobjid = m.objid" + " WHERE d.deptype = 'n'" + " AND NOT EXISTS (" + " SELECT 1 FROM spock_members m2" + " WHERE m2.classid = d.classid AND m2.objid = d.objid)"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not inventory spock extension dependents: %s\n"), + PQerrorMessage(conn)); + } + if (PQntuples(res) > 0) + { + PQExpBuffer list = createPQExpBuffer(); + int i; - ri->nodeid = atooid(PQgetvalue(res, 0, 0)); - ri->node_name = pstrdup(PQgetvalue(res, 0, 1)); - ri->sysid = pstrdup(PQgetvalue(res, 0, 2)); - ri->dbname = pstrdup(PQgetvalue(res, 0, 3)); - ri->replication_sets = pstrdup(PQgetvalue(res, 0, 4)); + for (i = 0; i < PQntuples(res); i++) + appendPQExpBuffer(list, "\n - %s", PQgetvalue(res, i, 0)); + PQclear(res); + die(_("cannot drop the spock extension: the following object(s) " + "depend on it and would be collaterally dropped by CASCADE:%s\n" + "Resolve these dependencies manually before retrying; v1 does " + "not attempt to recreate them.\n"), + list->data); + } PQclear(res); - return ri; + res = PQexec(conn, "DROP EXTENSION spock CASCADE;"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + die(_("Could not clean the spock extension, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); } /* - * Check if extension exists. + * Immediately after node_create, make the new node read-only to + * non-superuser clients -- there must be no window where n3 is + * reachable/writable before this lands. */ -static bool -extension_exists(PGconn *conn, const char *extname) +static void +set_readonly_local(PGconn *conn) { - PQExpBuffer query = createPQExpBuffer(); - PGresult *res; - bool ret; + PGresult *res; - printfPQExpBuffer(query, "SELECT 1 FROM pg_catalog.pg_extension WHERE extname = %s;", - PQescapeLiteral(conn, extname, strlen(extname))); - res = PQexec(conn, query->data); + res = PQexec(conn, "ALTER SYSTEM SET spock.readonly = 'local'"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + die(_("could not set spock.readonly: status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); + res = PQexec(conn, "SELECT pg_reload_conf()"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { - PQclear(res); - die(_("Could not read extension info: %s\n"), PQerrorMessage(conn)); + die(_("could not reload configuration after setting spock.readonly: %s\n"), + PQerrorMessage(conn)); } - - ret = PQntuples(res) == 1; - PQclear(res); - destroyPQExpBuffer(query); - - return ret; } /* - * Create extension. + * Restore the replication-set definitions, table memberships, and + * sequence state captured before the catalog strip, now that + * node_create() has given this node an identity again. Without this, + * n3 would accept incoming changes but send nothing back once peers + * create reverse subscriptions later. */ static void -install_extension(PGconn *conn, const char *extname) +restore_replication_sets(PGconn *conn, CatalogCapture *capture) { - PQExpBuffer query = createPQExpBuffer(); - PGresult *res; + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + int i; - printfPQExpBuffer(query, "CREATE EXTENSION IF NOT EXISTS %s;", - PQescapeIdentifier(conn, extname, strlen(extname))); - res = PQexec(conn, query->data); + /* + * 1. Recreate custom sets. The three built-in sets already exist from + * node_create(), so apply the captured flags to them via + * repset_alter() instead, since the source may have altered them. + */ + for (i = 0; i < capture->num_repsets; i++) + { + RepsetCapture *s = &capture->repsets[i]; + bool builtin = (strcmp(s->set_name, "default") == 0 || + strcmp(s->set_name, "default_insert_only") == 0 || + strcmp(s->set_name, "ddl_sql") == 0); - if (PQresultStatus(res) != PGRES_COMMAND_OK) + if (builtin) + printfPQExpBuffer(query, + "SELECT spock.repset_alter(" + "set_name := %s, " + "replicate_insert := %s, " + "replicate_update := %s, " + "replicate_delete := %s, " + "replicate_truncate := %s)", + PQescapeLiteral(conn, s->set_name, strlen(s->set_name)), + s->replicate_insert ? "true" : "false", + s->replicate_update ? "true" : "false", + s->replicate_delete ? "true" : "false", + s->replicate_truncate ? "true" : "false"); + else + printfPQExpBuffer(query, + "SELECT spock.repset_create(" + "set_name := %s, " + "replicate_insert := %s, " + "replicate_update := %s, " + "replicate_delete := %s, " + "replicate_truncate := %s)", + PQescapeLiteral(conn, s->set_name, strlen(s->set_name)), + s->replicate_insert ? "true" : "false", + s->replicate_update ? "true" : "false", + s->replicate_delete ? "true" : "false", + s->replicate_truncate ? "true" : "false"); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not %s replication set \"%s\": %s\n"), + builtin ? "alter" : "recreate", s->set_name, PQerrorMessage(conn)); + } + PQclear(res); + } + + /* + * 2. Restore table memberships for all sets. Named arguments are + * required: repset_add_table's 3rd positional argument is + * synchronize_data, not the column list, so a positional call would + * misfire. + * + * include_partitions := false: the capture already has a separate row + * per partition. Restoring the parent with include_partitions := true + * would re-add every child, violating the (set_id, set_reloid) + * primary key against the child's own captured row. + */ + for (i = 0; i < capture->num_tables; i++) + { + RepsetTableCapture *t = &capture->tables[i]; + + printfPQExpBuffer(query, + "SELECT spock.repset_add_table(" + "set_name := %s, " + "relation := %s, " + "synchronize_data := false, " + "columns := %s, " + "row_filter := %s, " + "include_partitions := false)", + PQescapeLiteral(conn, t->set_name, strlen(t->set_name)), + PQescapeLiteral(conn, t->qualified_table, strlen(t->qualified_table)), + t->columns ? PQescapeLiteral(conn, t->columns, strlen(t->columns)) : "NULL", + t->row_filter ? PQescapeLiteral(conn, t->row_filter, strlen(t->row_filter)) : "NULL"); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not add table \"%s\" to replication set \"%s\": %s\n"), + t->qualified_table, t->set_name, PQerrorMessage(conn)); + } + PQclear(res); + } + + /* + * 3. Restore each sequence's replication-set membership, then its + * value, so n3 both publishes it and resumes it exactly. + */ + for (i = 0; i < capture->num_sequences; i++) { + SequenceCapture *sq = &capture->sequences[i]; + + printfPQExpBuffer(query, + "SELECT spock.repset_add_seq(" + "set_name := %s, relation := %s, " + "synchronize_data := false)", + PQescapeLiteral(conn, sq->set_name, strlen(sq->set_name)), + PQescapeLiteral(conn, sq->qualified_seq, strlen(sq->qualified_seq))); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not add sequence \"%s\" to replication set \"%s\": %s\n"), + sq->qualified_seq, sq->set_name, PQerrorMessage(conn)); + } + PQclear(res); + + printfPQExpBuffer(query, "SELECT setval(%s, " INT64_FORMAT ", %s)", + PQescapeLiteral(conn, sq->qualified_seq, strlen(sq->qualified_seq)), + sq->last_value, + sq->is_called ? "true" : "false"); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not restore sequence state for \"%s\": %s\n"), + sq->qualified_seq, PQerrorMessage(conn)); + } PQclear(res); - die(_("Could not install %s extension: %s\n"), extname, PQerrorMessage(conn)); } - PQclear(res); destroyPQExpBuffer(query); + + print_msg(VERBOSITY_VERBOSE, + _("Restored %d replication set(s), %d table membership(s), " + "%d sequence(s).\n"), + capture->num_repsets, capture->num_tables, capture->num_sequences); + + verify_replication_sets_restored(conn, capture); } /* - * Clean all the data that was copied from remote node but we don't - * want it here (currently shared security labels and replication identifiers). + * Round-trip check: compare what actually landed against the capture, + * rather than trusting that each individual repset_add_table()/ + * repset_add_seq() call succeeding means the final state matches. + * Catches per-row drift (a column list or row_filter that didn't + * re-parse identically) and aggregate drift (an accidental double-add). + * n3 is brand-new here, so an unqualified COUNT(*) is safe. */ static void -remove_unwanted_data(PGconn *conn) +verify_replication_sets_restored(PGconn *conn, CatalogCapture *capture) { - PGresult *res; + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + int i; + int count; /* - * Remove replication identifiers (9.4 will get them removed by dropping - * the extension later as we emulate them there). + * Verify the four replication-set flags landed correctly -- a bug in + * restore_replication_sets()'s argument binding would otherwise pass + * verification with wrong flags on the clone. */ - res = PQexec(conn, "SELECT pg_replication_origin_drop(external_id) FROM pg_replication_origin_status;"); + for (i = 0; i < capture->num_repsets; i++) + { + RepsetCapture *s = &capture->repsets[i]; + + printfPQExpBuffer(query, + "SELECT replicate_insert, replicate_update," + " replicate_delete, replicate_truncate" + " FROM spock.replication_set WHERE set_name = %s", + PQescapeLiteral(conn, s->set_name, strlen(s->set_name))); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("replication set restore verification failed: set \"%s\" " + "not found after restore\n"), s->set_name); + } + + if (strcmp(PQgetvalue(res, 0, 0), s->replicate_insert ? "t" : "f") != 0 || + strcmp(PQgetvalue(res, 0, 1), s->replicate_update ? "t" : "f") != 0 || + strcmp(PQgetvalue(res, 0, 2), s->replicate_delete ? "t" : "f") != 0 || + strcmp(PQgetvalue(res, 0, 3), s->replicate_truncate ? "t" : "f") != 0) + { + char *got_insert = pg_strdup(PQgetvalue(res, 0, 0)); + char *got_update = pg_strdup(PQgetvalue(res, 0, 1)); + char *got_delete = pg_strdup(PQgetvalue(res, 0, 2)); + char *got_truncate = pg_strdup(PQgetvalue(res, 0, 3)); + + PQclear(res); + die(_("replication set restore verification failed: flags for set " + "\"%s\" do not match capture (expected i=%s/u=%s/d=%s/t=%s, " + "got i=%s/u=%s/d=%s/t=%s)\n"), + s->set_name, + s->replicate_insert ? "t" : "f", s->replicate_update ? "t" : "f", + s->replicate_delete ? "t" : "f", s->replicate_truncate ? "t" : "f", + got_insert, got_update, got_delete, got_truncate); + } + PQclear(res); + } + + for (i = 0; i < capture->num_tables; i++) + { + RepsetTableCapture *t = &capture->tables[i]; + char *columns; + char *row_filter; + + printfPQExpBuffer(query, + "SELECT rts.set_att_list," + " pg_get_expr(rts.set_row_filter, rts.set_reloid)" + " FROM spock.replication_set_table rts" + " JOIN spock.replication_set rs ON rts.set_id = rs.set_id" + " WHERE rs.set_name = %s AND rts.set_reloid::regclass::text = %s", + PQescapeLiteral(conn, t->set_name, strlen(t->set_name)), + PQescapeLiteral(conn, t->qualified_table, strlen(t->qualified_table))); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("replication set restore verification failed: table \"%s\" " + "not found in set \"%s\" after restore\n"), + t->qualified_table, t->set_name); + } + + columns = PQgetisnull(res, 0, 0) ? NULL : pg_strdup(PQgetvalue(res, 0, 0)); + row_filter = PQgetisnull(res, 0, 1) ? NULL : pg_strdup(PQgetvalue(res, 0, 1)); + PQclear(res); + + if ((columns == NULL) != (t->columns == NULL) || + (columns && strcmp(columns, t->columns) != 0)) + die(_("replication set restore verification failed: column list for " + "table \"%s\" in set \"%s\" does not match capture " + "(expected %s, got %s)\n"), + t->qualified_table, t->set_name, + t->columns ? t->columns : "NULL", columns ? columns : "NULL"); + + if ((row_filter == NULL) != (t->row_filter == NULL) || + (row_filter && strcmp(row_filter, t->row_filter) != 0)) + die(_("replication set restore verification failed: row_filter for " + "table \"%s\" in set \"%s\" does not re-parse identically " + "(expected %s, got %s)\n"), + t->qualified_table, t->set_name, + t->row_filter ? t->row_filter : "NULL", + row_filter ? row_filter : "NULL"); + } + + printfPQExpBuffer(query, + "SELECT COUNT(*) FROM spock.replication_set_table"); + res = PQexec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); - die(_("Could not remove existing replication origins: %s\n"), PQerrorMessage(conn)); + die(_("could not verify table membership count: %s\n"), PQerrorMessage(conn)); } + count = atoi(PQgetvalue(res, 0, 0)); PQclear(res); + if (count != capture->num_tables) + die(_("replication set restore verification failed: expected %d table " + "membership(s), found %d\n"), capture->num_tables, count); - res = PQexec(conn, "DROP EXTENSION spock CASCADE;"); - if (PQresultStatus(res) != PGRES_COMMAND_OK) + for (i = 0; i < capture->num_sequences; i++) { - die(_("Could not clean the spock extension, status %s: %s\n"), - PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + SequenceCapture *sq = &capture->sequences[i]; + + printfPQExpBuffer(query, + "SELECT COUNT(*) FROM spock.replication_set_seq rss" + " JOIN spock.replication_set rs ON rss.set_id = rs.set_id" + " WHERE rs.set_name = %s AND rss.set_seqoid::regclass::text = %s", + PQescapeLiteral(conn, sq->set_name, strlen(sq->set_name)), + PQescapeLiteral(conn, sq->qualified_seq, strlen(sq->qualified_seq))); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not verify sequence membership for \"%s\": %s\n"), + sq->qualified_seq, PQerrorMessage(conn)); + } + count = atoi(PQgetvalue(res, 0, 0)); + PQclear(res); + if (count != 1) + die(_("replication set restore verification failed: sequence \"%s\" " + "not a member of set \"%s\" after restore\n"), + sq->qualified_seq, sq->set_name); + } + + printfPQExpBuffer(query, "SELECT COUNT(*) FROM spock.replication_set_seq"); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not verify sequence membership count: %s\n"), PQerrorMessage(conn)); } + count = atoi(PQgetvalue(res, 0, 0)); PQclear(res); + if (count != capture->num_sequences) + die(_("replication set restore verification failed: expected %d sequence " + "membership(s), found %d\n"), capture->num_sequences, count); + + destroyPQExpBuffer(query); + + print_msg(VERBOSITY_VERBOSE, + _("Verified replication set restore matches capture exactly.\n")); } /* @@ -1800,7 +3654,7 @@ initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn) if (PQresultStatus(res) != PGRES_TUPLES_OK) { die(_("Could not create replication origin \"%s\": status %s: %s\n"), - query->data, + origin_name, PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); } PQclear(res); @@ -1816,7 +3670,7 @@ initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn) if (PQresultStatus(res) != PGRES_TUPLES_OK) { die(_("Could not advance replication origin \"%s\": status %s: %s\n"), - query->data, + origin_name, PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); } PQclear(res); @@ -1837,7 +3691,8 @@ create_restore_point(PGconn *conn, char *restore_point_name) PGresult *res; char *remote_lsn = NULL; - printfPQExpBuffer(query, "SELECT pg_create_restore_point('%s')", restore_point_name); + printfPQExpBuffer(query, "SELECT pg_create_restore_point(%s)", + PQescapeLiteral(conn, restore_point_name, strlen(restore_point_name))); res = PQexec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { @@ -1937,7 +3792,7 @@ validate_replication_set_input(char *replication_sets) if (strlen(name) == 0) die(_("Replication set name \"%s\" is too short\n"), name); - if (strlen(name) > NAMEDATALEN) + if (strlen(name) >= NAMEDATALEN) die(_("Replication set name \"%s\" is too long\n"), name); for (cp = name; *cp; cp++) @@ -2090,25 +3945,97 @@ get_connstr(char *connstr, char *dbname) static char * read_sysid(const char *data_dir) { - ControlFileData ControlFile; - int fd; - char ControlFilePath[MAXPGPATH]; + ControlFileData *cf; + bool crc_ok; char *res = (char *) pg_malloc0(33); - snprintf(ControlFilePath, MAXPGPATH, "%s/global/pg_control", data_dir); + /* + * get_controlfile() validates the control file's CRC; a torn or + * corrupted control file must be rejected here rather than silently + * misread, since this result feeds directly into check_data_dir()'s + * "is this really a basebackup of the expected node" safety check. + */ + cf = get_controlfile(data_dir, &crc_ok); + if (!crc_ok) + die(_("control file of \"%s\" appears to be corrupt\n"), data_dir); + + snprintf(res, 33, UINT64_FORMAT, cf->system_identifier); + pg_free(cf); + return res; +} + +/* + * Assign data_dir a fresh system identifier, since a physical clone + * otherwise keeps the source's -- risking stray WAL from one cluster + * being mistaken for the other's, and leaving system_identifier useless + * for proving --subscriber-dsn actually reaches this node. Called with + * the subscriber stopped, right after promotion and before any catalog + * mutation. + * + * pg_resetwal alone does NOT do this: it only regenerates + * system_identifier when it can't read an existing control file at all + * (verified empirically against a valid, cleanly-shut-down cluster). + * The identifier is overwritten directly in the control file here; + * pg_resetwal is run afterward (run_pg_resetwal()) only to relabel the + * existing WAL segments to match. + * + * Returns the new identifier as a string (caller must free()), matching + * read_sysid()'s convention. + */ +static char * +reset_subscriber_sysid(const char *data_dir) +{ + ControlFileData *cf; + bool crc_ok; + struct timeval tv; + char *result = (char *) pg_malloc0(33); - if ((fd = open(ControlFilePath, O_RDONLY | PG_BINARY, 0)) == -1) - die(_("%s: could not open file \"%s\" for reading: %s\n"), - progname, ControlFilePath, strerror(errno)); + cf = get_controlfile(data_dir, &crc_ok); + if (!crc_ok) + die(_("control file of \"%s\" appears to be corrupt\n"), data_dir); - if (read(fd, &ControlFile, sizeof(ControlFileData)) != sizeof(ControlFileData)) - die(_("%s: could not read file \"%s\": %s\n"), - progname, ControlFilePath, strerror(errno)); + /* Same formula used to assign a system identifier at initdb time. */ + gettimeofday(&tv, NULL); + cf->system_identifier = ((uint64) tv.tv_sec) << 32; + cf->system_identifier |= ((uint64) tv.tv_usec) << 12; + cf->system_identifier |= getpid() & 0xFFF; - close(fd); + update_controlfile(data_dir, cf, true); - snprintf(res, 33, UINT64_FORMAT, ControlFile.system_identifier); - return res; + snprintf(result, 33, UINT64_FORMAT, cf->system_identifier); + pg_free(cf); + + return result; +} + +/* + * Relabel data_dir's existing WAL segments to match the system + * identifier reset_subscriber_sysid() just wrote to the control file + * (see that function's comment for why both steps are needed). Must + * run after reset_subscriber_sysid(), with the subscriber stopped. + */ +static void +run_pg_resetwal(const char *data_dir) +{ + int ret; + PQExpBuffer cmd = createPQExpBuffer(); + char *exec_path = find_other_exec_or_die(argv0, "pg_resetwal"); + + appendPQExpBuffer(cmd, "\"%s\" -D \"%s\"", exec_path, data_dir); + + print_msg(VERBOSITY_DEBUG, _("Running pg_resetwal: %s.\n"), cmd->data); + ret = system(cmd->data); + + destroyPQExpBuffer(cmd); + + if (WIFEXITED(ret) && WEXITSTATUS(ret) == 0) + return; + if (WIFEXITED(ret)) + die(_("pg_resetwal failed with exit status %d, cannot continue.\n"), WEXITSTATUS(ret)); + else if (WIFSIGNALED(ret)) + die(_("pg_resetwal exited with signal %d, cannot continue"), WTERMSIG(ret)); + else + die(_("pg_resetwal exited for an unknown reason (system() returned %d)"), ret); } /* @@ -2198,10 +4125,12 @@ appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str) bool needquotes; /* - * If the string consists entirely of plain ASCII characters, no need to - * quote it. This is quite conservative, but better safe than sorry. + * If the string is one or more plain ASCII characters, no need to quote + * it. An empty string must default to needing quotes -- an unquoted + * empty value doesn't parse as empty, it swallows the entire next + * "keyword=value" token. */ - needquotes = false; + needquotes = true; for (s = str; *s; s++) { if (!((*s >= 'a' && *s <= 'z') || (*s >= 'A' && *s <= 'Z') || @@ -2210,6 +4139,7 @@ appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str) needquotes = true; break; } + needquotes = false; } if (needquotes) @@ -2263,7 +4193,7 @@ wait_postmaster_connection(const char *connstr) break; /* - * Check if the process is still alive. This covers cases where the + * Check if the process is still alive. This covers cases where the * postmaster successfully created the pidfile but then crashed without * removing it. */ @@ -2280,14 +4210,25 @@ wait_postmaster_connection(const char *connstr) /* - * Wait for PostgreSQL to leave recovery/standby mode + * Wait for PostgreSQL to leave recovery/standby mode. + * + * stall_timeout/max_wait (seconds; 0 = disabled) bound replay catchup, + * but only once PostgreSQL first accepts connections -- they don't bound + * server startup itself. stall_timeout tracks pg_last_wal_replay_lsn() + * as a progress signal and fires only when replay stalls, not on total + * elapsed time, so a slow multi-GB catchup can still run. max_wait is a + * separate hard ceiling on total post-connection wait time. The + * unidirectional path passes 0/0 for unbounded waiting. */ static void -wait_primary_connection(const char *connstr) +wait_primary_connection(const char *connstr, int stall_timeout, int max_wait) { bool ispri = false; PGconn *conn = NULL; PGresult *res; + time_t start_time = time(NULL); + time_t last_progress_time = start_time; + char *last_lsn = NULL; wait_postmaster_connection(connstr); @@ -2305,16 +4246,51 @@ wait_primary_connection(const char *connstr) res = PQexec(conn, "SELECT pg_is_in_recovery()"); if (PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) == 1 && *PQgetvalue(res, 0, 0) == 'f') + { ispri = true; - else + PQclear(res); + break; + } + PQclear(res); + + if (stall_timeout > 0) { - pg_usleep(1000000); /* 1 sec */ - print_msg(VERBOSITY_VERBOSE, "."); + PGresult *lsn_res = PQexec(conn, "SELECT pg_last_wal_replay_lsn()"); + + if (PQresultStatus(lsn_res) == PGRES_TUPLES_OK && PQntuples(lsn_res) == 1 && + !PQgetisnull(lsn_res, 0, 0)) + { + char *cur_lsn = PQgetvalue(lsn_res, 0, 0); + + if (!last_lsn || strcmp(cur_lsn, last_lsn) != 0) + { + pg_free(last_lsn); + last_lsn = pg_strdup(cur_lsn); + last_progress_time = time(NULL); + } + } + PQclear(lsn_res); + + if ((time(NULL) - last_progress_time) >= stall_timeout) + { + PQfinish(conn); + die(_("recovery appears stalled: no WAL replay progress for " + "%d second(s) (--stall-timeout)\n"), stall_timeout); + } } - PQclear(res); + if (max_wait > 0 && (time(NULL) - start_time) >= max_wait) + { + PQfinish(conn); + die(_("timed out after %d second(s) waiting for recovery to " + "complete (--max-wait)\n"), max_wait); + } + + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_VERBOSE, "."); } + pg_free(last_lsn); PQfinish(conn); print_msg(VERBOSITY_VERBOSE, "\n"); } @@ -2325,19 +4301,33 @@ wait_primary_connection(const char *connstr) static void wait_postmaster_shutdown(void) { - long pid; + long pid; + int waited = 0; + const int max_wait_secs = 60; print_msg(VERBOSITY_VERBOSE, "Waiting for PostgreSQL to shutdown ..."); for (;;) { - if ((pid = get_pgpid()) != 0) - { - pg_usleep(1000000); /* 1 sec */ - print_msg(VERBOSITY_NORMAL, "."); - } - else + pid = get_pgpid(); + if (pid == 0) + break; + + /* + * A hard-killed postmaster can leave its pidfile behind (it's only + * removed on a normal exit) -- without this check a stale pidfile + * hangs here forever. Mirrors the same postmaster_is_alive() check + * wait_postmaster_connection() already does on the start side. + */ + if (!postmaster_is_alive((pid_t) pid)) break; + + if (++waited >= max_wait_secs) + die(_("timed out after %d second(s) waiting for PostgreSQL to " + "shut down\n"), max_wait_secs); + + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_NORMAL, "."); } print_msg(VERBOSITY_VERBOSE, "\n"); @@ -2385,7 +4375,7 @@ copy_file(char *fromfile, char *tofile, bool append) #define COPY_BUF_SIZE (8 * BLCKSZ) - buffer = malloc(COPY_BUF_SIZE); + buffer = pg_malloc(COPY_BUF_SIZE); /* * Open the files @@ -2425,7 +4415,7 @@ copy_file(char *fromfile, char *tofile, bool append) /* we don't care about errors here */ close(srcfd); - free(buffer); + pg_free(buffer); } From f62662782d0a7d0171e24dcb911a2cf5e8ce995a Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Wed, 5 Aug 2026 17:20:05 +0500 Subject: [PATCH 11/13] spock_create_subscriber: add --postgresql-auto-conf override pg_basebackup copies postgresql.auto.conf verbatim from the source, including any settings applied there via ALTER SYSTEM. Most of that (tuning, spock GUCs) is exactly what should carry over to the new node, but something like port may need to differ. postgresql.auto.conf is loaded after postgresql.conf and wins on conflicts, so such a setting can't be overridden via --postgresql-conf. Add --postgresql-auto-conf: its contents are appended to the inherited postgresql.auto.conf rather than replacing it, so only the settings it specifies override the source's, and everything else inherited stays in effect. --- .../spock_create_subscriber.c | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index b8349a15..59f217a7 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -198,8 +198,8 @@ static bool extension_exists(PGconn *conn, const char *extname); static void install_extension(PGconn *conn, const char *extname); static void initialize_data_dir(char *data_dir, char *connstr, - char *postgresql_conf, char *pg_hba_conf, - char *extra_basebackup_args); + char *postgresql_conf, char *postgresql_auto_conf, + char *pg_hba_conf, char *extra_basebackup_args); static bool check_data_dir(char *data_dir, RemoteInfo *remoteinfo); static char *read_sysid(const char *data_dir); @@ -1832,6 +1832,7 @@ main(int argc, char **argv) char *replication_sets = NULL; char *databases = NULL; char *postgresql_conf = NULL, + *postgresql_auto_conf = NULL, *pg_hba_conf = NULL, *recovery_conf = NULL; int apply_delay = 0; @@ -1872,6 +1873,7 @@ main(int argc, char **argv) {"max-wait", required_argument, NULL, 14}, {"cleanup", no_argument, NULL, 15}, {"force", no_argument, NULL, 16}, + {"postgresql-auto-conf", required_argument, NULL, 17}, {NULL, 0, NULL, 0} }; @@ -1982,6 +1984,13 @@ main(int argc, char **argv) case 16: bidir.force_cleanup = true; break; + case 17: + { + postgresql_auto_conf = pg_strdup(optarg); + if (postgresql_auto_conf != NULL && !file_exists(postgresql_auto_conf)) + die(_("The specified postgresql.auto.conf file does not exist.")); + break; + } default: fprintf(stderr, _("Unknown option\n")); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); @@ -2263,7 +2272,7 @@ main(int argc, char **argv) initialize_data_dir(data_dir, use_existing_data_dir ? NULL : prov_connstr, - postgresql_conf, pg_hba_conf, + postgresql_conf, postgresql_auto_conf, pg_hba_conf, extra_basebackup_args); snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); @@ -2624,6 +2633,7 @@ usage(void) printf(_("\nConfiguration files override:\n")); printf(_(" --hba-conf path to the new pg_hba.conf\n")); printf(_(" --postgresql-conf path to the new postgresql.conf\n")); + printf(_(" --postgresql-auto-conf settings to override in postgresql.auto.conf\n")); printf(_(" --recovery-conf path to the template recovery configuration\n")); printf(_("\nBidirectional join (joins an existing multi-master cluster):\n")); printf(_(" --bidirectional enable bidirectional join plumbing\n")); @@ -2793,8 +2803,8 @@ run_basebackup(const char *provider_connstr, const char *data_dir, */ static void initialize_data_dir(char *data_dir, char *connstr, - char *postgresql_conf, char *pg_hba_conf, - char *extra_basebackup_args) + char *postgresql_conf, char *postgresql_auto_conf, + char *pg_hba_conf, char *extra_basebackup_args) { if (connstr) { @@ -2805,6 +2815,32 @@ initialize_data_dir(char *data_dir, char *connstr, if (postgresql_conf) CopyConfFile(postgresql_conf, "postgresql.conf", false); + if (postgresql_auto_conf) + { + char auto_conf_path[MAXPGPATH]; + FILE *f; + + /* + * postgresql.auto.conf is copied verbatim from the source by + * pg_basebackup, and is loaded after postgresql.conf and wins + * on conflicts -- most of it (tuning, spock GUCs) is exactly + * what should carry over to this node, but a setting like port + * or listen_addresses may need to differ. Append rather than + * replace, so this node's overrides win (same-file, later + * setting wins) while everything else inherited stays in effect. + * A marker line makes any resulting duplicate settings obvious + * to whoever next reads the file. + */ + snprintf(auto_conf_path, sizeof(auto_conf_path), "%s/postgresql.auto.conf", data_dir); + f = fopen(auto_conf_path, "a"); + if (f == NULL) + die(_("could not open \"%s\": %s\n"), auto_conf_path, strerror(errno)); + fprintf(f, "# --- appended by spock_create_subscriber (--postgresql-auto-conf); " + "later settings override the inherited ones above ---\n"); + fclose(f); + + CopyConfFile(postgresql_auto_conf, "postgresql.auto.conf", true); + } if (pg_hba_conf) CopyConfFile(pg_hba_conf, "pg_hba.conf", false); } From 01eeb2ab4ae58c0d5d5720f3978d731f6a4008ea Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Thu, 6 Aug 2026 19:19:42 +0500 Subject: [PATCH 12/13] spock_create_subscriber: trace every query and step at -v -v Add debug_exec(), a drop-in PQexec() wrapper that logs the query text and result status at VERBOSITY_DEBUG (-v -v), applied to every query site. Each major action also logs the concrete subscription, slot, node, or LSN it's acting on, so a -v -v run shows exactly what the tool is doing, not just which phase it's in. --- .../spock_create_subscriber.c | 192 ++++++++++++------ 1 file changed, 134 insertions(+), 58 deletions(-) diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index 59f217a7..8c9a182a 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -166,6 +166,7 @@ static void die(const char *fmt,...) pg_attribute_printf(1, 2); static void print_msg(VerbosityLevelEnum level, const char *fmt,...) pg_attribute_printf(2, 3); +static PGresult *debug_exec(PGconn *conn, const char *query); static int run_pg_ctl(const char *arg); static void validate_extra_basebackup_args(const char *args); @@ -333,6 +334,7 @@ discover_peer_nodes(PGconn *source_conn, const char *source_node_name, int i; paramValues[0] = source_node_name; + print_msg(VERBOSITY_DEBUG, _(" > %s [$1=%s]\n"), discover_sql, source_node_name); res = PQexecParams(source_conn, discover_sql, 1, NULL, paramValues, NULL, NULL, 0); if (PQresultStatus(res) != PGRES_TUPLES_OK) @@ -364,6 +366,10 @@ discover_peer_nodes(PGconn *source_conn, const char *source_node_name, paramValues[0] = dbname; paramValues[1] = peers[i].node_name; paramValues[2] = peers[i].sub_name; + print_msg(VERBOSITY_DEBUG, + _(" > SELECT spock.spock_gen_slot_name($1::name, $2::name, " + "$3::name) [$1=%s, $2=%s, $3=%s]\n"), + dbname, peers[i].node_name, peers[i].sub_name); slot_res = PQexecParams(source_conn, "SELECT spock.spock_gen_slot_name" "($1::name, $2::name, $3::name)", @@ -395,7 +401,7 @@ check_spock_version_at_least_6(PGconn *conn, const char *node_label) { PGresult *res; - res = PQexec(conn, "SELECT extversion FROM pg_extension WHERE extname = 'spock'"); + res = debug_exec(conn, "SELECT extversion FROM pg_extension WHERE extname = 'spock'"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -446,7 +452,7 @@ check_mesh_edges(PGconn *conn, const char *this_node_name, * means "actually replicating" -- a worker that's down or still * initializing must not satisfy the mesh. */ - res = PQexec(conn, "SELECT provider_node, status FROM spock.sub_show_status()"); + res = debug_exec(conn, "SELECT provider_node, status FROM spock.sub_show_status()"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -525,7 +531,7 @@ check_peer_identity(PGconn *peer_conn, const char *expected_name) { PGresult *res; - res = PQexec(peer_conn, "SELECT node_name FROM spock.node_info()"); + res = debug_exec(peer_conn, "SELECT node_name FROM spock.node_info()"); if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) { PQclear(res); @@ -576,7 +582,7 @@ compute_repset_fingerprints(PGconn *conn, Oid node_id, const char *selected_filt " FROM spock.replication_set WHERE set_nodeid = %u" " AND (%s)" " ORDER BY set_name", node_id, selected_filter); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -610,7 +616,7 @@ compute_repset_fingerprints(PGconn *conn, Oid node_id, const char *selected_filt " ORDER BY rts.set_reloid::regclass::text", node_id, PQescapeLiteral(conn, entries[i].set_name, strlen(entries[i].set_name))); - tres = PQexec(conn, query->data); + tres = debug_exec(conn, query->data); if (PQresultStatus(tres) != PGRES_TUPLES_OK) { PQclear(tres); @@ -645,7 +651,7 @@ compute_repset_fingerprints(PGconn *conn, Oid node_id, const char *selected_filt "SELECT relkind::text, relreplident::text" " FROM pg_class WHERE oid = %s::regclass", PQescapeLiteral(conn, qualified_table, strlen(qualified_table))); - cres = PQexec(conn, schema_query->data); + cres = debug_exec(conn, schema_query->data); if (PQresultStatus(cres) != PGRES_TUPLES_OK || PQntuples(cres) != 1) { PQclear(cres); @@ -670,7 +676,7 @@ compute_repset_fingerprints(PGconn *conn, Oid node_id, const char *selected_filt " WHERE a.attrelid = %s::regclass AND a.attnum > 0" " AND NOT a.attisdropped ORDER BY a.attnum", PQescapeLiteral(conn, qualified_table, strlen(qualified_table))); - cres = PQexec(conn, schema_query->data); + cres = debug_exec(conn, schema_query->data); destroyPQExpBuffer(schema_query); if (PQresultStatus(cres) != PGRES_TUPLES_OK) { @@ -705,7 +711,7 @@ compute_repset_fingerprints(PGconn *conn, Oid node_id, const char *selected_filt " ORDER BY rss.set_seqoid::regclass::text", node_id, PQescapeLiteral(conn, entries[i].set_name, strlen(entries[i].set_name))); - sres = PQexec(conn, query->data); + sres = debug_exec(conn, query->data); if (PQresultStatus(sres) != PGRES_TUPLES_OK) { PQclear(sres); @@ -756,7 +762,7 @@ build_selected_set_name_filter(PGconn *conn) char *result; int i; - res = PQexec(conn, + res = debug_exec(conn, "SELECT DISTINCT s FROM spock.subscription," " unnest(sub_replication_sets) AS s ORDER BY 1"); if (PQresultStatus(res) != PGRES_TUPLES_OK) @@ -905,7 +911,7 @@ check_preconditions(PGconn *source_conn, const char *source_node_name, check_spock_version_at_least_6(source_conn, "source"); /* track_commit_timestamp must be on at the source */ - res = PQexec(source_conn, "SHOW track_commit_timestamp"); + res = debug_exec(source_conn, "SHOW track_commit_timestamp"); if (PQresultStatus(res) != PGRES_TUPLES_OK) die(_("could not check track_commit_timestamp: %s"), PQerrorMessage(source_conn)); @@ -921,7 +927,7 @@ check_preconditions(PGconn *source_conn, const char *source_node_name, * count is monotonically non-decreasing and is never zero on any node * that has replicated so much as a single DDL statement. */ - res = PQexec(source_conn, + res = debug_exec(source_conn, "SELECT COUNT(*) FROM pg_replication_slots" " WHERE slot_type = 'logical' AND plugin = 'spock_output'" " AND (confirmed_flush_lsn IS NULL" @@ -957,7 +963,7 @@ check_preconditions(PGconn *source_conn, const char *source_node_name, check_peer_identity(peer_conn, peers[i].node_name); check_spock_version_at_least_6(peer_conn, peers[i].node_name); - res = PQexec(peer_conn, "SHOW track_commit_timestamp"); + res = debug_exec(peer_conn, "SHOW track_commit_timestamp"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { /* @@ -1012,7 +1018,7 @@ check_single_spock_database(PGconn *conn, const char *base_prov_connstr, PQExpBuffer others = createPQExpBuffer(); int other_count = 0; - res = PQexec(conn, "SELECT datname FROM pg_database WHERE NOT datistemplate"); + res = debug_exec(conn, "SELECT datname FROM pg_database WHERE NOT datistemplate"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -1043,7 +1049,7 @@ check_single_spock_database(PGconn *conn, const char *base_prov_connstr, "\"%s\" to check: %s\n"), dbname, errmsg); } - ext_res = PQexec(db_conn, "SELECT 1 FROM pg_extension WHERE extname = 'spock'"); + ext_res = debug_exec(db_conn, "SELECT 1 FROM pg_extension WHERE extname = 'spock'"); if (PQresultStatus(ext_res) != PGRES_TUPLES_OK) { char *errmsg = pg_strdup(PQerrorMessage(db_conn)); @@ -1058,7 +1064,7 @@ check_single_spock_database(PGconn *conn, const char *base_prov_connstr, if (PQntuples(ext_res) > 0) { - node_res = PQexec(db_conn, "SELECT 1 FROM spock.local_node"); + node_res = debug_exec(db_conn, "SELECT 1 FROM spock.local_node"); if (PQresultStatus(node_res) != PGRES_TUPLES_OK) { char *errmsg = pg_strdup(PQerrorMessage(db_conn)); @@ -1104,7 +1110,7 @@ check_no_native_subscriptions(PGconn *conn) { PGresult *res; - res = PQexec(conn, + res = debug_exec(conn, "SELECT s.subname, d.datname" " FROM pg_subscription s" " JOIN pg_database d ON d.oid = s.subdbid" @@ -1594,7 +1600,7 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, " FROM pg_replication_slots" " WHERE slot_name = '%s'", state->source_slot_name); - res = PQexec(source_conn, query->data); + res = debug_exec(source_conn, query->data); if (PQresultStatus(res) == PGRES_TUPLES_OK) { if (PQntuples(res) > 0) @@ -1651,7 +1657,7 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, " FROM pg_replication_slots" " WHERE slot_name = '%s'", peer->slot_name); - res = PQexec(peer_conn, query->data); + res = debug_exec(peer_conn, query->data); if (PQresultStatus(res) == PGRES_TUPLES_OK) { if (PQntuples(res) > 0) @@ -1684,7 +1690,7 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, printfPQExpBuffer(query, "SELECT spock.sub_drop(%s, true)", PQescapeLiteral(peer_conn, reverse_sub, strlen(reverse_sub))); - res = PQexec(peer_conn, query->data); + res = debug_exec(peer_conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { print_msg(VERBOSITY_NORMAL, @@ -2187,6 +2193,15 @@ main(int argc, char **argv) remote_info->node_name, subscriber_name, db, &bidir.peers); + { + int pi; + + for (pi = 0; pi < bidir.num_peers; pi++) + print_msg(VERBOSITY_DEBUG, + _("Discovered peer \"%s\" (dsn \"%s\", slot \"%s\")\n"), + bidir.peers[pi].node_name, bidir.peers[pi].dsn, + bidir.peers[pi].slot_name); + } check_preconditions(provider_conn, remote_info->node_name, bidir.peers, bidir.num_peers); check_single_spock_database(provider_conn, base_prov_connstr, db); @@ -2208,11 +2223,16 @@ main(int argc, char **argv) print_msg(VERBOSITY_NORMAL, _("Creating source replication slot in database %s ...\n"), db); + print_msg(VERBOSITY_DEBUG, + _("Creating replication slot on source \"%s\" for future " + "subscription \"%s\"\n"), remote_info->node_name, source_sub_name); bidir.source_slot_name = initialize_replication_slot(provider_conn, remote_info->dbname, remote_info->node_name, source_sub_name, drop_slot_if_exists); + print_msg(VERBOSITY_DEBUG, _("Source replication slot created: \"%s\"\n"), + bidir.source_slot_name); bidir.source_origin_name = pg_strdup(bidir.source_slot_name); pg_free(source_sub_name); @@ -2270,6 +2290,14 @@ main(int argc, char **argv) prov_connstr = get_connstr(base_prov_connstr, database_list[0]); sub_connstr = get_connstr(base_sub_connstr, database_list[0]); + if (!use_existing_data_dir) + print_msg(VERBOSITY_DEBUG, + _("Taking a physical base backup from \"%s\" into \"%s\"\n"), + prov_connstr, data_dir); + else + print_msg(VERBOSITY_DEBUG, + _("Reusing existing data directory \"%s\" (already a basebackup " + "of this source)\n"), data_dir); initialize_data_dir(data_dir, use_existing_data_dir ? NULL : prov_connstr, postgresql_conf, postgresql_auto_conf, pg_hba_conf, @@ -2413,7 +2441,7 @@ main(int argc, char **argv) * share it. */ { - PGresult *sysid_res = PQexec(subscriber_conn, "SELECT system_identifier FROM pg_control_system()"); + PGresult *sysid_res = debug_exec(subscriber_conn, "SELECT system_identifier FROM pg_control_system()"); bool mismatch; if (PQresultStatus(sysid_res) != PGRES_TUPLES_OK || PQntuples(sysid_res) != 1) @@ -2434,9 +2462,20 @@ main(int argc, char **argv) /* Capture repset/table/sequence state before the catalog strip. */ source_nodeid = get_local_node_id(subscriber_conn); + print_msg(VERBOSITY_DEBUG, + _("Capturing replication-set/table/sequence membership for local " + "node id %u before dropping the spock extension\n"), source_nodeid); capture_catalog_state(subscriber_conn, source_nodeid, &capture); + print_msg(VERBOSITY_DEBUG, + _("Captured %d replication set(s), %d table membership(s), " + "%d sequence(s)\n"), + capture.num_repsets, capture.num_tables, capture.num_sequences); /* Drop all origins, then guarded DROP EXTENSION. */ + print_msg(VERBOSITY_DEBUG, + _("Dropping replication origins and the spock extension (checking " + "pg_depend first for non-spock objects CASCADE would collaterally " + "drop)\n")); remove_unwanted_data_bidir(subscriber_conn, &capture); PQfinish(subscriber_conn); @@ -2501,6 +2540,8 @@ main(int argc, char **argv) */ print_msg(VERBOSITY_NORMAL, _("Creating local Spock node \"%s\"...\n"), subscriber_name); + print_msg(VERBOSITY_DEBUG, _("Registering node \"%s\" with dsn \"%s\"\n"), + subscriber_name, sub_connstr); { PQExpBuffer nodequery = createPQExpBuffer(); PGresult *res; @@ -2511,7 +2552,7 @@ main(int argc, char **argv) strlen(subscriber_name)), PQescapeLiteral(subscriber_conn, sub_connstr, strlen(sub_connstr))); - res = PQexec(subscriber_conn, nodequery->data); + res = debug_exec(subscriber_conn, nodequery->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -2527,6 +2568,11 @@ main(int argc, char **argv) /* Restore what was captured before the catalog strip. */ print_msg(VERBOSITY_NORMAL, _("Restoring replication set state...\n")); + print_msg(VERBOSITY_DEBUG, + _("Restoring %d replication set(s), %d table membership(s), " + "%d sequence(s) onto node \"%s\"\n"), + capture.num_repsets, capture.num_tables, capture.num_sequences, + subscriber_name); restore_replication_sets(subscriber_conn, &capture); bidir.source_restore_lsn = pg_strdup(remote_lsn); @@ -2624,7 +2670,9 @@ usage(void) printf(_(" --apply-delay=DELAY apply delay in seconds (by default 0)\n")); printf(_(" --drop-slot-if-exists drop replication slot of conflicting name\n")); printf(_(" -s, --stop stop the server once the initialization is done\n")); - printf(_(" -v increase logging verbosity\n")); + printf(_(" -v increase logging verbosity; repeatable --\n")); + printf(_(" -v -v also traces every query this tool\n")); + printf(_(" runs, with its result status\n")); printf(_(" --extra-basebackup-args additional arguments to pass to pg_basebackup.\n")); printf(_(" Safe options: -T, -c, --xlogdir/--waldir\n")); printf(_(" --text-types transfer column values as text rather than binary\n")); @@ -2693,6 +2741,34 @@ print_msg(VerbosityLevelEnum level, const char *fmt,...) } } +/* + * PQexec() wrapper that logs the query text at VERBOSITY_DEBUG (-v -v) + * before running it, and the resulting status/row count after -- a + * drop-in replacement so every query this tool issues is traceable + * without a separate print_msg() call at each site. Callers still do + * their own PQresultStatus()/die() handling on the result exactly as + * with a plain PQexec() call. + */ +static PGresult * +debug_exec(PGconn *conn, const char *query) +{ + PGresult *res; + + print_msg(VERBOSITY_DEBUG, _(" > %s\n"), query); + res = PQexec(conn, query); + if (verbosity >= VERBOSITY_DEBUG) + { + if (PQresultStatus(res) == PGRES_TUPLES_OK) + print_msg(VERBOSITY_DEBUG, _(" < %s (%d row(s))\n"), + PQresStatus(PQresultStatus(res)), PQntuples(res)); + else + print_msg(VERBOSITY_DEBUG, _(" < %s\n"), + PQresStatus(PQresultStatus(res))); + } + + return res; +} + /* * Start pg_ctl with given argument(s) - used to start/stop postgres @@ -2900,7 +2976,7 @@ initialize_replication_slot(PGconn *conn, char *dbname, PQescapeLiteral(conn, subscription_name, strlen(subscription_name))); - res = PQexec(conn, query.data); + res = debug_exec(conn, query.data); if (PQresultStatus(res) != PGRES_TUPLES_OK) die(_("Could generate slot name: %s"), PQerrorMessage(conn)); @@ -2914,7 +2990,7 @@ initialize_replication_slot(PGconn *conn, char *dbname, "SELECT 1 FROM pg_catalog.pg_replication_slots WHERE slot_name = %s", PQescapeLiteral(conn, slot_name, strlen(slot_name))); - res = PQexec(conn, query.data); + res = debug_exec(conn, query.data); if (PQresultStatus(res) != PGRES_TUPLES_OK) die(_("Could not fetch existing slot information: %s"), PQerrorMessage(conn)); @@ -2935,7 +3011,7 @@ initialize_replication_slot(PGconn *conn, char *dbname, "SELECT pg_catalog.pg_drop_replication_slot(%s)", PQescapeLiteral(conn, slot_name, strlen(slot_name))); - res = PQexec(conn, query.data); + res = debug_exec(conn, query.data); if (PQresultStatus(res) != PGRES_TUPLES_OK) die(_("Could not drop existing slot %s: %s"), slot_name, PQerrorMessage(conn)); @@ -2949,7 +3025,7 @@ initialize_replication_slot(PGconn *conn, char *dbname, PQescapeLiteral(conn, slot_name, strlen(slot_name)), "spock_output"); - res = PQexec(conn, query.data); + res = debug_exec(conn, query.data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { die(_("Could not create replication slot, status %s: %s\n"), @@ -2976,7 +3052,7 @@ get_remote_info(PGconn* conn) if (!extension_exists(conn, "spock")) die(_("The remote node is not configured as a spock provider.\n")); - res = PQexec(conn, "SELECT node_id, node_name, sysid, dbname, replication_sets FROM spock.node_info()"); + res = debug_exec(conn, "SELECT node_id, node_name, sysid, dbname, replication_sets FROM spock.node_info()"); if (PQresultStatus(res) != PGRES_TUPLES_OK) die(_("could not fetch remote node info: %s\n"), PQerrorMessage(conn)); @@ -3012,7 +3088,7 @@ extension_exists(PGconn *conn, const char *extname) printfPQExpBuffer(query, "SELECT 1 FROM pg_catalog.pg_extension WHERE extname = %s;", PQescapeLiteral(conn, extname, strlen(extname))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { @@ -3039,7 +3115,7 @@ install_extension(PGconn *conn, const char *extname) printfPQExpBuffer(query, "CREATE EXTENSION IF NOT EXISTS %s;", PQescapeIdentifier(conn, extname, strlen(extname))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_COMMAND_OK) { @@ -3064,7 +3140,7 @@ remove_unwanted_data(PGconn *conn) * Remove replication identifiers (9.4 will get them removed by dropping * the extension later as we emulate them there). */ - res = PQexec(conn, "SELECT pg_replication_origin_drop(external_id) FROM pg_replication_origin_status;"); + res = debug_exec(conn, "SELECT pg_replication_origin_drop(external_id) FROM pg_replication_origin_status;"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3072,7 +3148,7 @@ remove_unwanted_data(PGconn *conn) } PQclear(res); - res = PQexec(conn, "DROP EXTENSION spock CASCADE;"); + res = debug_exec(conn, "DROP EXTENSION spock CASCADE;"); if (PQresultStatus(res) != PGRES_COMMAND_OK) { die(_("Could not clean the spock extension, status %s: %s\n"), @@ -3092,7 +3168,7 @@ get_local_node_id(PGconn *conn) PGresult *res; Oid nodeid; - res = PQexec(conn, "SELECT node_id FROM spock.local_node"); + res = debug_exec(conn, "SELECT node_id FROM spock.local_node"); if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) { PQclear(res); @@ -3127,7 +3203,7 @@ capture_catalog_state(PGconn *conn, Oid source_nodeid, CatalogCapture *capture) " FROM spock.replication_set" " WHERE set_nodeid = %u", source_nodeid); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3156,7 +3232,7 @@ capture_catalog_state(PGconn *conn, Oid source_nodeid, CatalogCapture *capture) " JOIN spock.replication_set rs ON rts.set_id = rs.set_id" " WHERE rs.set_nodeid = %u", source_nodeid); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3188,7 +3264,7 @@ capture_catalog_state(PGconn *conn, Oid source_nodeid, CatalogCapture *capture) " JOIN spock.replication_set rs ON rss.set_id = rs.set_id" " WHERE rs.set_nodeid = %u", source_nodeid); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3214,7 +3290,7 @@ capture_catalog_state(PGconn *conn, Oid source_nodeid, CatalogCapture *capture) */ printfPQExpBuffer(seq_query, "SELECT last_value, is_called FROM %s", capture->sequences[i].qualified_seq); - seq_res = PQexec(conn, seq_query->data); + seq_res = debug_exec(conn, seq_query->data); if (PQresultStatus(seq_res) != PGRES_TUPLES_OK) { PQclear(seq_res); @@ -3261,7 +3337,7 @@ remove_unwanted_data_bidir(PGconn *conn, CatalogCapture *capture) * row -- an unrelated database on the same instance with its own * (non-spock) logical replication would otherwise lose its origins too. */ - res = PQexec(conn, + res = debug_exec(conn, "SELECT pg_replication_origin_drop(roname)" " FROM pg_replication_origin" " WHERE roname LIKE 'spk\\_%' ESCAPE '\\'"); @@ -3274,7 +3350,7 @@ remove_unwanted_data_bidir(PGconn *conn, CatalogCapture *capture) PQclear(res); /* Guard against CASCADE collaterally dropping user objects. */ - res = PQexec(conn, + res = debug_exec(conn, "WITH spock_ext AS (" " SELECT oid FROM pg_extension WHERE extname = 'spock'" "), ext_members AS (" @@ -3328,7 +3404,7 @@ remove_unwanted_data_bidir(PGconn *conn, CatalogCapture *capture) } PQclear(res); - res = PQexec(conn, "DROP EXTENSION spock CASCADE;"); + res = debug_exec(conn, "DROP EXTENSION spock CASCADE;"); if (PQresultStatus(res) != PGRES_COMMAND_OK) { die(_("Could not clean the spock extension, status %s: %s\n"), @@ -3347,7 +3423,7 @@ set_readonly_local(PGconn *conn) { PGresult *res; - res = PQexec(conn, "ALTER SYSTEM SET spock.readonly = 'local'"); + res = debug_exec(conn, "ALTER SYSTEM SET spock.readonly = 'local'"); if (PQresultStatus(res) != PGRES_COMMAND_OK) { die(_("could not set spock.readonly: status %s: %s\n"), @@ -3355,7 +3431,7 @@ set_readonly_local(PGconn *conn) } PQclear(res); - res = PQexec(conn, "SELECT pg_reload_conf()"); + res = debug_exec(conn, "SELECT pg_reload_conf()"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { die(_("could not reload configuration after setting spock.readonly: %s\n"), @@ -3416,7 +3492,7 @@ restore_replication_sets(PGconn *conn, CatalogCapture *capture) s->replicate_update ? "true" : "false", s->replicate_delete ? "true" : "false", s->replicate_truncate ? "true" : "false"); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3453,7 +3529,7 @@ restore_replication_sets(PGconn *conn, CatalogCapture *capture) PQescapeLiteral(conn, t->qualified_table, strlen(t->qualified_table)), t->columns ? PQescapeLiteral(conn, t->columns, strlen(t->columns)) : "NULL", t->row_filter ? PQescapeLiteral(conn, t->row_filter, strlen(t->row_filter)) : "NULL"); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3477,7 +3553,7 @@ restore_replication_sets(PGconn *conn, CatalogCapture *capture) "synchronize_data := false)", PQescapeLiteral(conn, sq->set_name, strlen(sq->set_name)), PQescapeLiteral(conn, sq->qualified_seq, strlen(sq->qualified_seq))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3490,7 +3566,7 @@ restore_replication_sets(PGconn *conn, CatalogCapture *capture) PQescapeLiteral(conn, sq->qualified_seq, strlen(sq->qualified_seq)), sq->last_value, sq->is_called ? "true" : "false"); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3540,7 +3616,7 @@ verify_replication_sets_restored(PGconn *conn, CatalogCapture *capture) " replicate_delete, replicate_truncate" " FROM spock.replication_set WHERE set_name = %s", PQescapeLiteral(conn, s->set_name, strlen(s->set_name))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) { PQclear(res); @@ -3584,7 +3660,7 @@ verify_replication_sets_restored(PGconn *conn, CatalogCapture *capture) " WHERE rs.set_name = %s AND rts.set_reloid::regclass::text = %s", PQescapeLiteral(conn, t->set_name, strlen(t->set_name)), PQescapeLiteral(conn, t->qualified_table, strlen(t->qualified_table))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) { PQclear(res); @@ -3617,7 +3693,7 @@ verify_replication_sets_restored(PGconn *conn, CatalogCapture *capture) printfPQExpBuffer(query, "SELECT COUNT(*) FROM spock.replication_set_table"); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3639,7 +3715,7 @@ verify_replication_sets_restored(PGconn *conn, CatalogCapture *capture) " WHERE rs.set_name = %s AND rss.set_seqoid::regclass::text = %s", PQescapeLiteral(conn, sq->set_name, strlen(sq->set_name)), PQescapeLiteral(conn, sq->qualified_seq, strlen(sq->qualified_seq))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3655,7 +3731,7 @@ verify_replication_sets_restored(PGconn *conn, CatalogCapture *capture) } printfPQExpBuffer(query, "SELECT COUNT(*) FROM spock.replication_set_seq"); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3685,7 +3761,7 @@ initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn) printfPQExpBuffer(query, "SELECT pg_replication_origin_create(%s)", PQescapeLiteral(conn, origin_name, strlen(origin_name))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { @@ -3701,7 +3777,7 @@ initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn) PQescapeLiteral(conn, origin_name, strlen(origin_name)), remote_lsn); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { @@ -3729,7 +3805,7 @@ create_restore_point(PGconn *conn, char *restore_point_name) printfPQExpBuffer(query, "SELECT pg_create_restore_point(%s)", PQescapeLiteral(conn, restore_point_name, strlen(restore_point_name))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { die(_("Could not create restore point, status %s: %s\n"), @@ -3758,7 +3834,7 @@ spock_subscribe(PGconn *conn, char *subscriber_name, char *subscriber_dsn, PQescapeLiteral(conn, subscriber_name, strlen(subscriber_name)), PQescapeLiteral(conn, subscriber_dsn, strlen(subscriber_dsn))); - res = PQexec(conn, query.data); + res = debug_exec(conn, query.data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { die(_("Could not create local node, status %s: %s\n"), @@ -3783,7 +3859,7 @@ spock_subscribe(PGconn *conn, char *subscriber_name, char *subscriber_dsn, PQescapeLiteral(conn, repsets.data, repsets.len), apply_delay, (force_text_transfer ? "t" : "f")); - res = PQexec(conn, query.data); + res = debug_exec(conn, query.data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { die(_("Could not create subscription, status %s: %s\n"), @@ -3791,7 +3867,7 @@ spock_subscribe(PGconn *conn, char *subscriber_name, char *subscriber_dsn, } PQclear(res); - res = PQexec(conn, "UPDATE spock.local_sync_status SET sync_status = 'r'" + res = debug_exec(conn, "UPDATE spock.local_sync_status SET sync_status = 'r'" " WHERE sync_status != 'r'"); if (PQresultStatus(res) != PGRES_COMMAND_OK) { @@ -4280,7 +4356,7 @@ wait_primary_connection(const char *connstr, int stall_timeout, int max_wait) conn = connectdb(connstr); } - res = PQexec(conn, "SELECT pg_is_in_recovery()"); + res = debug_exec(conn, "SELECT pg_is_in_recovery()"); if (PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) == 1 && *PQgetvalue(res, 0, 0) == 'f') { ispri = true; @@ -4291,7 +4367,7 @@ wait_primary_connection(const char *connstr, int stall_timeout, int max_wait) if (stall_timeout > 0) { - PGresult *lsn_res = PQexec(conn, "SELECT pg_last_wal_replay_lsn()"); + PGresult *lsn_res = debug_exec(conn, "SELECT pg_last_wal_replay_lsn()"); if (PQresultStatus(lsn_res) == PGRES_TUPLES_OK && PQntuples(lsn_res) == 1 && !PQgetisnull(lsn_res, 0, 0)) From 9f2a788f480d0c2f7a7d4955f1c7622e48ac32e5 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Thu, 6 Aug 2026 19:48:19 +0500 Subject: [PATCH 13/13] spock_create_subscriber: catchup subscription phase Continue --bidirectional past replication-set restore into catchup: - Create n3's subscription to the source disabled-first (enabled := false), which sets it READY and creates its local replication origin atomically with no apply worker and no INIT window; advance that origin to the recorded restore LSN, confirm forward_origins landed as '{all}' (or forwarded peer changes would be silently dropped), then enable it. - Pre-create a disabled subscription to every peer, giving each its own local named origin the same way, without touching the peer at all -- no remote slot, no apply worker there yet. - Capture a catchup target via a single spock.sync_event() on the source. - Wait for the catchup subscription to reach that target: a progress watchdog (resets on any origin advance, not a flat wall-clock timeout, mirroring the existing WAL-replay wait), that aborts immediately if the subscription's own status reports disabled -- the signal an unresolvable apply exception leaves behind. --- tests/tap/schedule | 2 +- .../t/{048_bidir_pr3.pl => 048_bidir_join.pl} | 102 ++- .../spock_create_subscriber.c | 618 ++++++++++++++++-- 3 files changed, 656 insertions(+), 66 deletions(-) rename tests/tap/t/{048_bidir_pr3.pl => 048_bidir_join.pl} (85%) diff --git a/tests/tap/schedule b/tests/tap/schedule index 9b2c208f..8f53c533 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -55,7 +55,7 @@ test: 032_lolor_largeobject_repset test: 033_zodan_lolor_add_node test: 034_reserved_object_ddl test: 044_apply_change_logging -test: 048_bidir_pr3 +test: 048_bidir_join # Upgrade schema match test (builds from source, slow): #test: 018_upgrade_schema_match # diff --git a/tests/tap/t/048_bidir_pr3.pl b/tests/tap/t/048_bidir_join.pl similarity index 85% rename from tests/tap/t/048_bidir_pr3.pl rename to tests/tap/t/048_bidir_join.pl index a317e451..09a152d2 100644 --- a/tests/tap/t/048_bidir_pr3.pl +++ b/tests/tap/t/048_bidir_join.pl @@ -1,11 +1,14 @@ #!/usr/bin/perl # ============================================================================= -# Test: 048_bidir_pr3.pl - spock_create_subscriber --bidirectional +# Test: 048_bidir_join.pl - spock_create_subscriber --bidirectional # ============================================================================= -# Validates the bidirectional node-join procedure: physical backup, recovery -# to a restore point, catalog strip (capture + origin drop + guarded DROP -# EXTENSION), and replication-set/table/sequence restore -- stopping short -# of the catchup subscription (a later step). +# Validates the bidirectional node-join procedure end to end: physical +# backup, recovery to a restore point, catalog strip (capture + origin drop +# + guarded DROP EXTENSION), replication-set/table/sequence restore, and +# catchup (disabled-first subscription to the source, disabled placeholder +# subscriptions to every peer, and a wait for n3 to reach a target LSN +# captured on the source) -- stopping short of enabling any direct peer +# subscription (a later step). # # Topology: # n1 <-> n2 (full bidirectional Spock subscriptions, existing 2-node @@ -25,10 +28,12 @@ # 1 sequence advanced past its initial value on n1 (setval fidelity check) # 1 partitioned table (parent + 2 children) added to custom set on n1 # 1 sequence with apostrophe in name added to custom set on n1 +# 1 peer-forwarding test table created on n1 +# 1 peer-forwarding test table replicated to n2 # 1 --bidirectional exits 0 # 1 n3 postgres is running # 1 spock extension installed cleanly on n3 (exactly one row) -# 1 n3 has no leftover replication origins from the basebackup +# 1 n3 has exactly the catchup and peer origins, none leftover from the basebackup # 1 n3 was given its own system identifier (pg_resetwal), distinct from n1 # 1 spock.readonly is 'local' on n3 # 1 custom replication set restored on n3 with correct flags @@ -45,6 +50,11 @@ # 1 manifest: source_restore_lsn populated # 1 manifest: node_dsn populated # 1 source slot exists on n1 +# 1 catchup subscription sub_n3_n1 is replicating on n3 +# 1 disabled peer subscription sub_n3_n2 exists and is disabled +# 1 n3's origin for peer n2 starts at 0/0 before any post-join write +# 1 n2's post-join write reached n3 via forwarding through sub_n3_n1 +# 1 n3's origin for peer n2 advanced during catchup forwarding # 1 --cleanup --force exits 0 # 1 source slot removed from n1 after cleanup # 1 n3 data directory removed after cleanup --force @@ -69,12 +79,12 @@ # 1 pending sidecar removed once cleanup actually completed # 1 destroy_cluster # --- -# 57 total +# 64 total # ============================================================================= use strict; use warnings; -use Test::More tests => 57; +use Test::More tests => 64; use File::Path qw(remove_tree); use lib '.'; use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail @@ -183,6 +193,24 @@ q(SELECT setval('"weird''s_seq"', 7, true)); pass('sequence with apostrophe in name added to custom set on n1'); +# Table used later to verify n3's origin for peer n2 advances via forwarding. +# Created on n1 only and left to arrive on n2 via DDL replication (creating +# it directly on both sides races the already-established cross-wire DDL +# replay); spock.include_ddl_repset=on adds it to 'default' on each node +# once it lands there. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr4_peer_tbl (id serial primary key, val text)"; +pass('peer-forwarding test table created on n1'); + +my $tbl_on_n2 = '0'; +for (1 .. 15) { + $tbl_on_n2 = scalar_query(2, + "SELECT COUNT(*) FROM pg_tables WHERE tablename = 'pr4_peer_tbl'"); + last if $tbl_on_n2 eq '1'; + sleep(1); +} +is($tbl_on_n2, '1', 'peer-forwarding test table replicated to n2'); + # check_preconditions() requires all of n1's outbound replication to have # caught up (no unreplicated DDL/data still in flight to n2); wait for the # setup above to drain. @@ -248,9 +276,14 @@ $ext_count =~ s/\s+//g; is($ext_count, '1', 'spock extension installed cleanly on n3 (exactly one row)'); +# By this point the catchup subscription and the one disabled peer +# subscription (n2) have each created their own origin -- exactly 2, not +# more. Anything beyond that would mean an origin survived from the +# basebackup instead of being dropped by the catalog strip. my $origin_count = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM pg_replication_origin"`; $origin_count =~ s/\s+//g; -is($origin_count, '0', 'n3 has no leftover replication origins from the basebackup'); +is($origin_count, '2', + 'n3 has exactly the catchup and peer origins, none leftover from the basebackup'); my $n3_sysid = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT system_identifier FROM pg_control_system()"`; $n3_sysid =~ s/\s+//g; @@ -343,6 +376,57 @@ sub psql_capture { "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); ok($source_slot_exists >= 1, 'source slot exists on n1'); +# ============================================================================= +# TEST: catchup subscription created, enabled, and caught up; disabled peer +# subscription's origin advances via forwarding once n2 writes post-join. +# ============================================================================= +my $sub_status = ''; +for (1 .. 30) { + $sub_status = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT status FROM spock.sub_show_status('sub_n3_n1')"); + last if $sub_status eq 'replicating'; + sleep(1); +} +is($sub_status, 'replicating', 'catchup subscription sub_n3_n1 is replicating on n3'); + +my $peer_sub_status = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT status FROM spock.sub_show_status('sub_n3_n2')"); +is($peer_sub_status, 'disabled', 'disabled peer subscription sub_n3_n2 exists and is disabled'); + +# Origin name matches what create_disabled_peer_subscriptions() computed for +# sub_n3_n2 (spock_gen_slot_name(dbname, 'n2', 'sub_n3_n2')). +my $n2_origin_name = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT spock.spock_gen_slot_name('$dbname', 'n2', 'sub_n3_n2')"); + +my $n2_origin_query = + "SELECT COALESCE(s.remote_lsn::text, '0/0') FROM pg_replication_origin o " . + "LEFT JOIN pg_replication_origin_status s ON o.roident = s.local_id " . + "WHERE o.roname = '$n2_origin_name'"; + +my $n2_origin_lsn_initial = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', $n2_origin_query); +is($n2_origin_lsn_initial, '0/0', + "n3's origin for peer n2 starts at 0/0 before any post-join write"); + +# Write on n2 after the join; n1 forwards it to n3 via sub_n3_n1's +# forward_origins = '{all}', and maybe_advance_forwarded_origin() should move +# n3's origin for n2 off 0/0 even though the direct sub_n3_n2 stays disabled. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[1], '-d', $dbname, '-c', + "INSERT INTO pr4_peer_tbl (val) VALUES ('from_n2_post_join')"; + +my $row_on_n3 = '0'; +for (1 .. 30) { + $row_on_n3 = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT COUNT(*) FROM pr4_peer_tbl WHERE val = 'from_n2_post_join'"); + last if $row_on_n3 eq '1'; + sleep(1); +} +is($row_on_n3, '1', "n2's post-join write reached n3 via forwarding through sub_n3_n1"); + +my $n2_origin_lsn = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', $n2_origin_query); +isnt($n2_origin_lsn, '0/0', "n3's origin for peer n2 advanced during catchup forwarding"); + # ============================================================================= # TEST: --cleanup --force removes source slot, data directory, and manifest # ============================================================================= diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index 8c9a182a..b2a0062d 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -70,6 +71,9 @@ typedef struct RemoteInfo { char *sysid; char *dbname; char *replication_sets; + TimeLineID timeline_id; /* current TLI, for detecting a data_dir + * left over from an already-promoted + * earlier attempt (see check_data_dir()) */ } RemoteInfo; typedef struct PeerNodeInfo @@ -202,6 +206,7 @@ static void initialize_data_dir(char *data_dir, char *connstr, char *postgresql_conf, char *postgresql_auto_conf, char *pg_hba_conf, char *extra_basebackup_args); static bool check_data_dir(char *data_dir, RemoteInfo *remoteinfo); +static void check_reused_data_dir_is_safe(const char *data_dir, RemoteInfo *remoteinfo); static char *read_sysid(const char *data_dir); @@ -214,6 +219,7 @@ static char *PQconninfoParamsToConnstr(const char *const * keywords, const char static void appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str); static bool file_exists(const char *path); +static char *expand_tilde(char *path); static bool is_pg_dir(const char *path); static void copy_file(char *fromfile, char *tofile, bool append); static char *find_other_exec_or_die(const char *argv0, const char *target); @@ -242,6 +248,8 @@ static bool read_manifest(const char *manifest_path, BidirectionalState *state, static bool cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, const char *dbname, const char *source_dsn, bool force_rm_datadir); +static void stop_postgres_in_data_dir(void); +static bool remove_data_dir_if_forced(bool force); static void append_json_string(PQExpBuffer buf, const char *str); static void check_single_spock_database(PGconn *conn, const char *base_prov_connstr, @@ -252,6 +260,15 @@ static void capture_catalog_state(PGconn *conn, Oid source_nodeid, static void remove_unwanted_data_bidir(PGconn *conn, CatalogCapture *capture); static void restore_replication_sets(PGconn *conn, CatalogCapture *capture); static void verify_replication_sets_restored(PGconn *conn, CatalogCapture *capture); +static void create_catchup_subscription(PGconn *subscriber_conn, const char *source_sub_name, + const char *source_dsn, const char *replication_sets, + const char *source_slot_name, const char *source_restore_lsn); +static void create_disabled_peer_subscriptions(PGconn *subscriber_conn, PeerNodeInfo *peers, + int num_peers, const char *replication_sets); +static char *get_catchup_target_lsn(const char *source_dsn); +static void wait_for_catchup(PGconn *subscriber_conn, const char *source_sub_name, + const char *source_slot_name, const char *target_lsn, + int stall_timeout, int max_wait); static void set_readonly_local(PGconn *conn); static Oid get_local_node_id(PGconn *conn); @@ -1552,6 +1569,65 @@ read_manifest(const char *manifest_path, BidirectionalState *state, return true; } +/* + * If data_dir holds a running postmaster, stop it (fast mode) and wait + * for shutdown. No-op if data_dir is unset, doesn't exist, or has no + * postmaster.pid. + */ +static void +stop_postgres_in_data_dir(void) +{ + struct stat st; + + if (data_dir == NULL || !data_dir[0] || !file_exists(data_dir)) + return; + + snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); + if (stat(pid_file, &st) == 0) + { + print_msg(VERBOSITY_NORMAL, + _(" stopping postgres in %s ...\n"), data_dir); + run_pg_ctl("stop -m fast"); + wait_postmaster_shutdown(); + } +} + +/* + * If data_dir exists, remove it when force is true (stopping postgres in + * it first, defensively, in case the caller hasn't already); if force is + * false, leave it in place with a hint. Returns false only when removal + * was attempted and actually failed; a missing data_dir, an unset one, + * or force being false are all "nothing to report" and return true. + */ +static bool +remove_data_dir_if_forced(bool force) +{ + if (data_dir == NULL || !data_dir[0] || !file_exists(data_dir)) + return true; + + if (!force) + { + print_msg(VERBOSITY_NORMAL, + _(" data directory %s was left in place; pass --force " + "to remove it, or clean it up manually.\n"), data_dir); + return true; + } + + stop_postgres_in_data_dir(); + + print_msg(VERBOSITY_NORMAL, + _(" removing data directory %s ...\n"), data_dir); + if (!rmtree(data_dir, true)) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not fully remove data directory " + "%s; remove it manually\n"), data_dir); + return false; + } + + return true; +} + /* * Idempotently remove bidirectional join state from all reachable nodes. * Connects to the source and each peer; drops replication slots and @@ -1577,6 +1653,18 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, print_msg(VERBOSITY_NORMAL, _("Cleaning up partial bidirectional join state ...\n")); + /* + * If n3 got far enough to have a live, replicating catchup + * subscription, its apply worker holds the source slot active -- + * pg_drop_replication_slot() below would fail against it. Stop n3 + * first, unconditionally (not gated by --force, which only governs + * removing the data directory): leaving it running against a source + * slot that this function is about to drop would just have it error + * out repeatedly instead of sitting in a clean, inspectable stopped + * state. + */ + stop_postgres_in_data_dir(); + source_conn = PQconnectdb(source_dsn); if (PQstatus(source_conn) != CONNECTION_OK) { @@ -1716,39 +1804,8 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, * The data directory a partial run may have created via basebackup. * Never touch it without --force. */ - if (data_dir != NULL && data_dir[0] && file_exists(data_dir)) - { - if (force_rm_datadir) - { - struct stat st; - - snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); - if (stat(pid_file, &st) == 0) - { - print_msg(VERBOSITY_NORMAL, - _(" stopping postgres in %s before removing it ...\n"), - data_dir); - run_pg_ctl("stop -m fast"); - wait_postmaster_shutdown(); - } - - print_msg(VERBOSITY_NORMAL, - _(" removing data directory %s ...\n"), data_dir); - if (!rmtree(data_dir, true)) - { - print_msg(VERBOSITY_NORMAL, - _("warning: could not fully remove data directory " - "%s; remove it manually\n"), data_dir); - fully_cleaned = false; - } - } - else - { - print_msg(VERBOSITY_NORMAL, - _(" data directory %s was left in place; pass --force " - "to remove it, or clean it up manually.\n"), data_dir); - } - } + if (!remove_data_dir_if_forced(force_rm_datadir)) + fully_cleaned = false; if (!fully_cleaned) { @@ -1909,7 +1966,7 @@ main(int argc, char **argv) switch (c) { case 'D': - data_dir = pg_strdup(optarg); + data_dir = expand_tilde(pg_strdup(optarg)); break; case 'n': subscriber_name = pg_strdup(optarg); @@ -1925,21 +1982,21 @@ main(int argc, char **argv) break; case 4: { - postgresql_conf = pg_strdup(optarg); + postgresql_conf = expand_tilde(pg_strdup(optarg)); if (postgresql_conf != NULL && !file_exists(postgresql_conf)) die(_("The specified postgresql.conf file does not exist.")); break; } case 5: { - pg_hba_conf = pg_strdup(optarg); + pg_hba_conf = expand_tilde(pg_strdup(optarg)); if (pg_hba_conf != NULL && !file_exists(pg_hba_conf)) die(_("The specified pg_hba.conf file does not exist.")); break; } case 6: { - recovery_conf = pg_strdup(optarg); + recovery_conf = expand_tilde(pg_strdup(optarg)); if (recovery_conf != NULL && !file_exists(recovery_conf)) die(_("The specified recovery configuration file does not exist.")); break; @@ -1992,7 +2049,7 @@ main(int argc, char **argv) break; case 17: { - postgresql_auto_conf = pg_strdup(optarg); + postgresql_auto_conf = expand_tilde(pg_strdup(optarg)); if (postgresql_auto_conf != NULL && !file_exists(postgresql_auto_conf)) die(_("The specified postgresql.auto.conf file does not exist.")); break; @@ -2086,6 +2143,25 @@ main(int argc, char **argv) exit(cleanup_partial_state(&bidir, sub_name, db, src_dsn, bidir.force_cleanup) ? 0 : 1); + /* + * Neither record exists -- there's no slot/subscription bookkeeping + * to act on, e.g. because the run died before the pending sidecar + * was even written. But an orphaned data_dir can still be sitting + * there from that attempt, and --force is an explicit instruction + * to remove it: don't leave it behind just because there was + * nothing to read. + */ + if (bidir.force_cleanup && data_dir != NULL && data_dir[0] && + file_exists(data_dir)) + { + fprintf(stderr, + _("No manifest found at %s or %s; no slot/subscription " + "state to clean up, but --force was given -- removing " + "data directory %s.\n"), + bidir.manifest_path, bidir_pending_path, data_dir); + exit(remove_data_dir_if_forced(true) ? 0 : 1); + } + fprintf(stderr, _("No manifest found at %s or %s; nothing to clean up.\n"), bidir.manifest_path, bidir_pending_path); exit(0); @@ -2208,13 +2284,7 @@ main(int argc, char **argv) check_no_native_subscriptions(provider_conn); use_existing_data_dir = check_data_dir(data_dir, remote_info); if (use_existing_data_dir) - { - char *local_sysid = read_sysid(data_dir); - bool mismatch = strcmp(remote_info->sysid, local_sysid) != 0; - free(local_sysid); - if (mismatch) - die(_("Subscriber data directory is not basebackup of remote node.\n")); - } + check_reused_data_dir_is_safe(data_dir, remote_info); appendPQExpBuffer(sub_name_buf, "sub_%s_%s", subscriber_name, remote_info->node_name); @@ -2261,13 +2331,7 @@ main(int argc, char **argv) use_existing_data_dir = check_data_dir(data_dir, remote_info); if (use_existing_data_dir) - { - char *local_sysid = read_sysid(data_dir); - bool mismatch = strcmp(remote_info->sysid, local_sysid) != 0; - free(local_sysid); - if (mismatch) - die(_("Subscriber data directory is not basebackup of remote node.\n")); - } + check_reused_data_dir_is_safe(data_dir, remote_info); } /* @@ -2579,14 +2643,54 @@ main(int argc, char **argv) bidir.node_dsn = sub_connstr; write_manifest(&bidir, subscriber_name, db, base_prov_connstr); + { + PQExpBuffer sub_name_buf = createPQExpBuffer(); + char *source_sub_name; + char *target_lsn; + + appendPQExpBuffer(sub_name_buf, "sub_%s_%s", + subscriber_name, remote_info->node_name); + source_sub_name = pg_strdup(sub_name_buf->data); + destroyPQExpBuffer(sub_name_buf); + + print_msg(VERBOSITY_NORMAL, _("Creating catchup subscription to the source...\n")); + print_msg(VERBOSITY_DEBUG, + _("Creating subscription \"%s\" to source \"%s\" using slot " + "\"%s\", forward_origins={all}, enabled=false\n"), + source_sub_name, prov_connstr, bidir.source_slot_name); + create_catchup_subscription(subscriber_conn, source_sub_name, prov_connstr, + replication_sets, bidir.source_slot_name, + bidir.source_restore_lsn); + print_msg(VERBOSITY_DEBUG, + _("Subscription \"%s\" created, origin advanced to %s, and " + "enabled\n"), source_sub_name, bidir.source_restore_lsn); + + print_msg(VERBOSITY_NORMAL, _("Creating disabled peer subscriptions...\n")); + create_disabled_peer_subscriptions(subscriber_conn, bidir.peers, + bidir.num_peers, replication_sets); + + print_msg(VERBOSITY_NORMAL, _("Getting catchup target from the source...\n")); + target_lsn = get_catchup_target_lsn(prov_connstr); + print_msg(VERBOSITY_DEBUG, _("Catchup target LSN: %s\n"), target_lsn); + + print_msg(VERBOSITY_NORMAL, _("Waiting for catchup to the source...\n")); + print_msg(VERBOSITY_DEBUG, + _("Waiting for subscription \"%s\" (origin \"%s\") to reach " + "LSN %s\n"), source_sub_name, bidir.source_slot_name, target_lsn); + wait_for_catchup(subscriber_conn, source_sub_name, bidir.source_slot_name, + target_lsn, bidir.stall_timeout, bidir.max_wait); + + pg_free(target_lsn); + pg_free(source_sub_name); + } + PQfinish(subscriber_conn); subscriber_conn = NULL; print_msg(VERBOSITY_NORMAL, - _("Bidirectional join: physical backup, catalog strip, and " - "replication set restore complete. Node \"%s\" is " - "read-only pending the catchup subscription (a later " - "release).\n"), + _("Bidirectional join: catchup complete. Node \"%s\" has caught " + "up to the source and forward-tracked every peer's origin; " + "ready for the next phase.\n"), subscriber_name); } else @@ -2954,6 +3058,44 @@ check_data_dir(char *data_dir, RemoteInfo *remoteinfo) return false; } +/* + * Called whenever check_data_dir() approves reusing an existing + * data_dir. The sysid check alone doesn't catch every unsafe reuse: if + * an earlier attempt already reached promotion (recovery_target_action = + * promote) before failing or being interrupted, but before + * reset_subscriber_sysid() ran, the sysid still matches, yet this + * data_dir's own timeline has advanced past whatever the source has. + * Re-entering recovery against the source at that point can never + * succeed: the source has no way to supply WAL for a timeline it never + * had, so streaming fails permanently with "highest timeline N of the + * primary is behind recovery timeline M" and this tool would otherwise + * wait forever for WAL that will never arrive. Refuse reuse instead. + */ +static void +check_reused_data_dir_is_safe(const char *data_dir, RemoteInfo *remoteinfo) +{ + char *local_sysid = read_sysid(data_dir); + bool mismatch = strcmp(remoteinfo->sysid, local_sysid) != 0; + ControlFileData *cf; + bool crc_ok; + + free(local_sysid); + if (mismatch) + die(_("Subscriber data directory is not basebackup of remote node.\n")); + + cf = get_controlfile(data_dir, &crc_ok); + if (!crc_ok) + die(_("control file of \"%s\" appears to be corrupt\n"), data_dir); + if (cf->checkPointCopy.ThisTimeLineID > remoteinfo->timeline_id) + die(_("data directory \"%s\" is already on timeline %u, past the " + "source's current timeline %u -- it was already promoted by " + "an earlier, incomplete attempt and can never resume " + "recovery from this source again; run --cleanup --force and " + "retry with a fresh base backup\n"), + data_dir, cf->checkPointCopy.ThisTimeLineID, remoteinfo->timeline_id); + pg_free(cf); +} + /* * Initialize replication slots */ @@ -3073,6 +3215,12 @@ get_remote_info(PGconn* conn) PQclear(res); + res = debug_exec(conn, "SELECT timeline_id FROM pg_control_checkpoint()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not fetch remote node's current timeline: %s\n"), PQerrorMessage(conn)); + ri->timeline_id = (TimeLineID) strtoul(PQgetvalue(res, 0, 0), NULL, 10); + PQclear(res); + return ri; } @@ -3749,6 +3897,317 @@ verify_replication_sets_restored(PGconn *conn, CatalogCapture *capture) _("Verified replication set restore matches capture exactly.\n")); } +/* + * Create n3's subscription to the source disabled-first (enabled := + * false) -- sub_create() sets SYNC_STATUS_READY and creates the local + * replication origin atomically, with no apply worker and no INIT + * window, before this advances that origin to source_restore_lsn and + * enables it. + */ +static void +create_catchup_subscription(PGconn *subscriber_conn, const char *source_sub_name, + const char *source_dsn, const char *replication_sets, + const char *source_slot_name, const char *source_restore_lsn) +{ + PQExpBuffer query = createPQExpBuffer(); + PQExpBuffer repsets = createPQExpBuffer(); + PGresult *res; + PGconn *source_conn; + + /* Re-confirm the source slot is still there before relying on it. */ + source_conn = connectdb(source_dsn); + printfPQExpBuffer(query, "SELECT 1 FROM pg_replication_slots WHERE slot_name = %s", + PQescapeLiteral(source_conn, source_slot_name, strlen(source_slot_name))); + res = debug_exec(source_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check source replication slot \"%s\": %s\n"), + source_slot_name, PQerrorMessage(source_conn)); + } + if (PQntuples(res) != 1) + { + PQclear(res); + die(_("source replication slot \"%s\" is missing on the source; " + "cannot start catchup\n"), source_slot_name); + } + PQclear(res); + PQfinish(source_conn); + + printfPQExpBuffer(repsets, "{%s}", replication_sets); + printfPQExpBuffer(query, + "SELECT spock.sub_create(" + "subscription_name := %s, provider_dsn := %s, " + "replication_sets := %s, " + "synchronize_structure := false, " + "synchronize_data := false, " + "forward_origins := '{all}', " + "enabled := false)", + PQescapeLiteral(subscriber_conn, source_sub_name, strlen(source_sub_name)), + PQescapeLiteral(subscriber_conn, source_dsn, strlen(source_dsn)), + PQescapeLiteral(subscriber_conn, repsets->data, repsets->len)); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not create catchup subscription \"%s\": %s\n"), + source_sub_name, PQerrorMessage(subscriber_conn)); + } + PQclear(res); + + printfPQExpBuffer(query, "SELECT pg_replication_origin_advance(%s, %s)", + PQescapeLiteral(subscriber_conn, source_slot_name, strlen(source_slot_name)), + PQescapeLiteral(subscriber_conn, source_restore_lsn, strlen(source_restore_lsn))); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not advance catchup origin to the recovery point: %s\n"), + PQerrorMessage(subscriber_conn)); + } + PQclear(res); + + /* + * Confirm forward_origins landed as '{all}' -- otherwise forwarded peer + * changes are silently dropped during catchup instead of reaching n3. + */ + printfPQExpBuffer(query, "SELECT forward_origins FROM spock.sub_show_status(%s)", + PQescapeLiteral(subscriber_conn, source_sub_name, strlen(source_sub_name))); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("could not verify forward_origins on \"%s\": %s\n"), + source_sub_name, PQerrorMessage(subscriber_conn)); + } + if (strcmp(PQgetvalue(res, 0, 0), "{all}") != 0) + { + char *got = pg_strdup(PQgetvalue(res, 0, 0)); + + PQclear(res); + die(_("catchup subscription \"%s\" has forward_origins = %s, expected " + "{all}; forwarded peer changes would be silently dropped\n"), + source_sub_name, got); + } + PQclear(res); + + printfPQExpBuffer(query, "SELECT spock.sub_enable(%s)", + PQescapeLiteral(subscriber_conn, source_sub_name, strlen(source_sub_name))); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not enable catchup subscription \"%s\": %s\n"), + source_sub_name, PQerrorMessage(subscriber_conn)); + } + PQclear(res); + + destroyPQExpBuffer(query); + destroyPQExpBuffer(repsets); +} + +/* + * Pre-create a disabled subscription to every peer on n3, giving each a + * local named origin (sub_create(enabled := false), same mechanism as + * the catchup subscription) without creating anything on the peer + * itself -- no remote slot, no apply worker. Forwarding through the + * catchup subscription is what advances these origins; the direct peer + * subscriptions stay disabled until a later phase. + */ +static void +create_disabled_peer_subscriptions(PGconn *subscriber_conn, PeerNodeInfo *peers, + int num_peers, const char *replication_sets) +{ + PQExpBuffer query = createPQExpBuffer(); + PQExpBuffer repsets = createPQExpBuffer(); + int i; + + printfPQExpBuffer(repsets, "{%s}", replication_sets); + + for (i = 0; i < num_peers; i++) + { + PeerNodeInfo *peer = &peers[i]; + PGresult *res; + + print_msg(VERBOSITY_DEBUG, + _("Creating disabled subscription \"%s\" to peer \"%s\" (dsn " + "\"%s\"); its origin will be \"%s\"\n"), + peer->sub_name, peer->node_name, peer->dsn, peer->slot_name); + printfPQExpBuffer(query, + "SELECT spock.sub_create(" + "subscription_name := %s, provider_dsn := %s, " + "replication_sets := %s, " + "synchronize_structure := false, " + "synchronize_data := false, " + "enabled := false)", + PQescapeLiteral(subscriber_conn, peer->sub_name, strlen(peer->sub_name)), + PQescapeLiteral(subscriber_conn, peer->dsn, strlen(peer->dsn)), + PQescapeLiteral(subscriber_conn, repsets->data, repsets->len)); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not create disabled subscription \"%s\" to peer \"%s\": %s\n"), + peer->sub_name, peer->node_name, PQerrorMessage(subscriber_conn)); + } + PQclear(res); + + printfPQExpBuffer(query, "SELECT 1 FROM pg_replication_origin WHERE roname = %s", + PQescapeLiteral(subscriber_conn, peer->slot_name, strlen(peer->slot_name))); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not verify replication origin for peer \"%s\": %s\n"), + peer->node_name, PQerrorMessage(subscriber_conn)); + } + if (PQntuples(res) != 1) + { + PQclear(res); + die(_("expected replication origin \"%s\" for peer \"%s\" was not " + "created\n"), peer->slot_name, peer->node_name); + } + PQclear(res); + + peer->disabled_sub_created = true; + } + + destroyPQExpBuffer(query); + destroyPQExpBuffer(repsets); +} + +/* + * A single spock.sync_event() on the source is the catchup target -- it + * flushes durably before returning, so the LSN is guaranteed to arrive + * at n3 via the replication stream with no per-peer flush needed. + * Caller must free the result. + */ +static char * +get_catchup_target_lsn(const char *source_dsn) +{ + PGconn *source_conn; + PGresult *res; + char *target_lsn; + + source_conn = connectdb(source_dsn); + res = debug_exec(source_conn, "SELECT spock.sync_event()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("could not get catchup target LSN: %s\n"), PQerrorMessage(source_conn)); + } + target_lsn = pg_strdup(PQgetvalue(res, 0, 0)); + PQclear(res); + PQfinish(source_conn); + + return target_lsn; +} + +/* + * Wait for n3's catchup subscription to reach target_lsn. Progress + * watchdog, not a flat wall-clock timeout -- reset the stall clock + * whenever remote_lsn advances at all, since a legitimately large + * catchup can take hours (same shape as wait_primary_connection(), which + * does this for WAL replay). Aborts immediately, without waiting out + * the timeout, if the subscription's own status reports 'disabled' -- + * the signal an unresolvable apply exception leaves behind under + * spock.exception_behaviour = 'sub_disable'; catchup must not be allowed + * to silently stall forever behind a stopped apply worker. + */ +static void +wait_for_catchup(PGconn *subscriber_conn, const char *source_sub_name, + const char *source_slot_name, const char *target_lsn, + int stall_timeout, int max_wait) +{ + PQExpBuffer query = createPQExpBuffer(); + time_t start_time = time(NULL); + time_t last_progress_time = start_time; + char *last_lsn = NULL; + + print_msg(VERBOSITY_VERBOSE, "Waiting for catchup to reach %s...", target_lsn); + + for (;;) + { + PGresult *res; + bool reached; + + printfPQExpBuffer(query, + "SELECT (remote_lsn >= %s::pg_lsn), remote_lsn::text" + " FROM pg_replication_origin_status WHERE external_id = %s", + PQescapeLiteral(subscriber_conn, target_lsn, strlen(target_lsn)), + PQescapeLiteral(subscriber_conn, source_slot_name, strlen(source_slot_name))); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check catchup progress: %s\n"), PQerrorMessage(subscriber_conn)); + } + + reached = PQntuples(res) == 1 && !PQgetisnull(res, 0, 0) && + PQgetvalue(res, 0, 0)[0] == 't'; + if (reached) + { + PQclear(res); + break; + } + + if (PQntuples(res) == 1 && !PQgetisnull(res, 0, 1)) + { + char *cur_lsn = PQgetvalue(res, 0, 1); + + if (!last_lsn || strcmp(cur_lsn, last_lsn) != 0) + { + pg_free(last_lsn); + last_lsn = pg_strdup(cur_lsn); + last_progress_time = time(NULL); + } + } + PQclear(res); + + /* + * spock.sub_show_status() is the same primitive check_mesh_edges() + * relies on for subscription health; 'disabled' here means the + * apply worker hit an unresolvable exception and + * spock.exception_behaviour disabled it -- catchup cannot recover + * from that on its own, so abort now rather than waiting out + * stall_timeout/max_wait behind a subscription that will never + * move again. + */ + printfPQExpBuffer(query, "SELECT status FROM spock.sub_show_status(%s)", + PQescapeLiteral(subscriber_conn, source_sub_name, strlen(source_sub_name))); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check catchup subscription status: %s\n"), + PQerrorMessage(subscriber_conn)); + } + if (PQntuples(res) == 1 && strcmp(PQgetvalue(res, 0, 0), "disabled") == 0) + { + PQclear(res); + die(_("catchup subscription \"%s\" was disabled during catchup, " + "likely by an unresolvable apply exception; this is a hard " + "join failure -- run --cleanup and retry\n"), source_sub_name); + } + PQclear(res); + + if (stall_timeout > 0 && (time(NULL) - last_progress_time) >= stall_timeout) + die(_("catchup appears stalled: no origin progress for %d second(s) " + "(--stall-timeout)\n"), stall_timeout); + + if (max_wait > 0 && (time(NULL) - start_time) >= max_wait) + die(_("timed out after %d second(s) waiting for catchup to " + "complete (--max-wait)\n"), max_wait); + + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_VERBOSE, "."); + } + + pg_free(last_lsn); + destroyPQExpBuffer(query); + print_msg(VERBOSITY_VERBOSE, "\n"); +} + /* * Initialize new remote identifier to specific position. */ @@ -4456,6 +4915,53 @@ file_exists(const char *path) return true; } +/* + * Replace a leading "~" or "~username" with that user's home directory, + * in place. Shell tilde expansion never happens for a quoted argument, + * so a path like "~/n3.auto.conf" otherwise reaches file_exists() + * literally and fails. get_home_path() (port.h, already linked) + * resolves "~"/"~/..." for the current user; getpwnam() handles + * "~user"/"~user/...". + */ +static char * +expand_tilde(char *path) +{ + char *slash; + char home[MAXPGPATH]; + char *result; + + if (path == NULL || path[0] != '~') + return path; + + slash = strchr(path, '/'); + + if (slash == path + 1 || path[1] == '\0') + { + if (!get_home_path(home)) + return path; + } + else + { + char username[MAXPGPATH]; + struct passwd *pw; + size_t len = slash ? (size_t) (slash - (path + 1)) : strlen(path + 1); + + if (len >= sizeof(username)) + return path; + memcpy(username, path + 1, len); + username[len] = '\0'; + + pw = getpwnam(username); + if (pw == NULL) + return path; + strlcpy(home, pw->pw_dir, sizeof(home)); + } + + result = psprintf("%s%s", home, slash ? slash : ""); + pg_free(path); + return result; +} + static bool is_pg_dir(const char *path) {