Skip to content

test(e2e): add WaitForReplication after first switchover/failover #390

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tsivaprasad merged 2 commits into from
Jun 1, 2026
Merged

test(e2e): add WaitForReplication after first switchover/failover #390

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2cf6520
test(e2e): add WaitForReplication after first switchover/failover
tsivaprasad May 15, 2026
78670c3
addressing review comments
tsivaprasad Jun 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 15 additions & 6 deletions e2e/failover_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@ import (
func TestFailoverScenarios(t *testing.T) {
t.Parallel()

host1 := fixture.HostIDs()[0]
host2 := fixture.HostIDs()[1]
host3 := fixture.HostIDs()[2]
hostIDs := fixture.HostIDs()
require.GreaterOrEqual(t, len(hostIDs), 3, "fixture must provide at least 3 hosts")
host1 := hostIDs[0]
host2 := hostIDs[1]
host3 := hostIDs[2]

ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute)
defer cancel()
Expand All @@ -42,6 +44,10 @@ func TestFailoverScenarios(t *testing.T) {
Name: "n1",
HostIds: []controlplane.Identifier{controlplane.Identifier(host1), controlplane.Identifier(host2), controlplane.Identifier(host3)},
},
{
Name: "n2",
HostIds: []controlplane.Identifier{controlplane.Identifier(host1)},
},
},
},
})
Expand All @@ -56,15 +62,17 @@ func TestFailoverScenarios(t *testing.T) {
return inst.ID
}

waitFor(func() bool {
db.Refresh(ctx)
require.True(t, waitFor(func() bool {
require.NoError(t, db.Refresh(ctx))
for _, inst := range db.Instances {
if inst.NodeName == "n1" && (inst.State == "modifying" || inst.State == "creating") {
return false
}
}
return true
}, 60*time.Second)
}, 60*time.Second))

waitForFailoverSlots(ctx, t, db)

// Returns a non-primary instance that's ready/available, or "" if none found within timeout.
waitForReadyReplica := func(timeout time.Duration) string {
Expand Down Expand Up @@ -138,6 +146,7 @@ func TestFailoverScenarios(t *testing.T) {
"[auto] primary did not change within timeout (still %s)", origPrimary)
newPrimary := getPrimaryInstanceID()
t.Logf("[auto] new primary: %s", newPrimary)
db.WaitForReplication(ctx, t, "admin", "password")
})

t.Run("failover to a specific candidate", func(t *testing.T) {
Expand Down
49 changes: 43 additions & 6 deletions e2e/switchover_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,21 @@ import (
"time"

"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/stretchr/testify/require"

controlplane "github.com/pgEdge/control-plane/api/apiv1/gen/control_plane"
"github.com/pgEdge/control-plane/client"
)

func TestSwitchoverScenarios(t *testing.T) {
t.Parallel()

host1 := fixture.HostIDs()[0]
host2 := fixture.HostIDs()[1]
host3 := fixture.HostIDs()[2]
hostIDs := fixture.HostIDs()
require.GreaterOrEqual(t, len(hostIDs), 3, "fixture must provide at least 3 hosts")
host1 := hostIDs[0]
host2 := hostIDs[1]
host3 := hostIDs[2]

ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute)
defer cancel()
Expand All @@ -42,6 +46,10 @@ func TestSwitchoverScenarios(t *testing.T) {
Name: "n1",
HostIds: []controlplane.Identifier{controlplane.Identifier(host1), controlplane.Identifier(host2), controlplane.Identifier(host3)},
},
{
Name: "n2",
HostIds: []controlplane.Identifier{controlplane.Identifier(host1)},
},
},
},
})
Expand All @@ -60,15 +68,17 @@ func TestSwitchoverScenarios(t *testing.T) {
return inst.ID
}

waitFor(func() bool {
db.Refresh(ctx)
require.True(t, waitFor(func() bool {
require.NoError(t, db.Refresh(ctx))
for _, inst := range db.Instances {
if inst.NodeName == "n1" && (inst.State == "modifying" || inst.State == "creating") {
return false
}
}
return true
}, 60*time.Second)
}, 60*time.Second))

waitForFailoverSlots(ctx, t, db)

// Returns a non-primary instance that's ready/available, or "" if none found within timeout.
waitForReadyReplica := func(curPrimary string, timeout time.Duration) string {
Expand Down Expand Up @@ -140,6 +150,7 @@ func TestSwitchoverScenarios(t *testing.T) {
"[auto] primary did not change within timeout (still %s)", origPrimary)
newPrimary := getPrimaryInstanceID()
t.Logf("[auto] new primary: %s", newPrimary)
db.WaitForReplication(ctx, t, "admin", "password")
})

t.Run("switchover to a specific candidate", func(t *testing.T) {
Expand Down Expand Up @@ -261,6 +272,32 @@ func waitFor(cond func() bool, timeout time.Duration) bool {
return false
}

func waitForFailoverSlots(ctx context.Context, t *testing.T, db *DatabaseFixture) {
require.NoError(t, db.Refresh(ctx))
replicas := db.GetInstances(And(WithRole(client.RoleReplica)))
require.NotEmpty(t, replicas)
for replica := range replicas {
opts := ConnectionOptions{
Username: "admin",
Password: "password",
Instance: replica,
}
db.WithConnection(ctx, opts, t, func(conn *pgx.Conn) {
tLogf(t, "waiting for failover slot to exist on replica instance %s", replica.ID)
start := time.Now()

require.True(t, waitFor(func() bool {
var failoverSlotExists bool
row := conn.QueryRow(ctx, `select exists (select 1 from pg_replication_slots where plugin = 'spock_output')`)
require.NoError(t, row.Scan(&failoverSlotExists))
return failoverSlotExists
}, 60*time.Second))

tLogf(t, "failover slot exists in replica instance %s after %.2f seconds", replica.ID, time.Since(start).Seconds())
})
}
}

func serverNowUTC(t *testing.T, baseURL string) time.Time {
req, err := http.NewRequest(http.MethodHead, baseURL+"/v1/version", nil)
require.NoError(t, err)
Expand Down