perf: reduce E2E CI runtime via parallel jobs and test tuning

.circleci/config.yml

@@ -61,25 +61,66 @@ commands:
               docker-compose-plugin
             sudo systemctl start docker
 jobs:
-  test:
+  # basic_check: unit tests + lint + license check (make ci)
+  basic_check:
     executor: common
     steps:
       - common_setup
       - run:
           name: Run CI checks
           command: make ci TEST_RERUN_FAILS=2
+      - store_test_results:
+          path: .
+
+  test_cluster:
+    executor: common
+    steps:
+      - common_setup
       - run:
           name: Run cluster tests
           command: |
             make docker-swarm-init
             make start-local-registry
             make buildx-init
             make test-cluster CONTROL_PLANE_IMAGE_REPO=172.17.0.1:5000/control-plane TEST_RERUN_FAILS=2
+      - store_test_results:
+          path: .
+
+  test_e2e:
+    executor: common
+    parallelism: 3
+    steps:
+      - common_setup
+      - run:
+          name: Select this container's e2e tests
+          command: |
+            # Split e2e test files (excluding main_test.go, whose only test
+            # is the special TestMain entrypoint and can't be filtered via
+            # -run) across the 3 parallel containers, falling back to an
+            # even file-count split until enough --junitfile timing history
+            # (test-e2e-results.xml) has accumulated across runs.
+            TEST_FILES=$(grep -l '^func Test' e2e/*_test.go | grep -v 'e2e/main_test.go' \
+              | circleci tests split --split-by=timings --timings-type=filename)
+            if [ -z "$TEST_FILES" ]; then
+              TEST_NAMES=""
+            else
+              TEST_NAMES=$(grep -h '^func Test' $TEST_FILES \
+                | sed -E 's/^func (Test[A-Za-z0-9_]*).*/\1/' | paste -sd '|')
+            fi
+            # $$ (not $) so that Make's own recursive variable expansion of
+            # $(E2E_RUN) in the Makefile collapses it back to a single
+            # literal $ instead of silently dropping a lone trailing $.
+            echo "export E2E_RUN='^(${TEST_NAMES})\$\$'" >> "$BASH_ENV"
+            echo "Running tests: ${TEST_NAMES}"
       - run:
           name: Run e2e tests with Docker Compose
           command: |
-            make ci-compose-detached
-            make test-e2e E2E_PARALLEL=4 E2E_DEBUG=1 E2E_FIXTURE=ci TEST_RERUN_FAILS=2
+            if [ -z "$E2E_RUN" ] || [ "$E2E_RUN" = '^()$' ]; then
+              echo "No tests assigned to this container, skipping"
+            else
+              make ci-compose-detached
+              make test-e2e E2E_PARALLEL=4 E2E_DEBUG=1 E2E_FIXTURE=ci TEST_RERUN_FAILS=2
+            fi
       - run:
           name: Archive debug output
           command: |
@@ -172,7 +213,9 @@ workflows:
   test:
     unless: << pipeline.parameters.build_ref >>
     jobs:
-      - test
+      - basic_check
+      - test_cluster
+      - test_e2e
 
   release:
     jobs:

Makefile

@@ -38,7 +38,7 @@ docker_compose_dev=WORKSPACE_DIR=$(shell pwd) \
 docker_compose_ci=docker compose -f ./docker/control-plane-ci/docker-compose.yaml
 e2e_args=-tags=e2e_test -count=1 -timeout=45m \
 	$(if $(E2E_PARALLEL),-parallel $(E2E_PARALLEL)) \
-	$(if $(E2E_RUN),-run $(E2E_RUN)) \
+	$(if $(E2E_RUN),-run "$(E2E_RUN)") \
 	-args \
 	$(if $(E2E_FIXTURE),-fixture $(E2E_FIXTURE)) \
 	$(if $(filter 1,$(E2E_SKIP_CLEANUP)),-skip-cleanup) \

clustertest/add_remove_host_test.go

@@ -154,7 +154,7 @@ func TestForcedHostRemovalWithDatabase(t *testing.T) {
 	cluster.Host("host-3").Stop(t)
 
 	// Allow cluster some time to detect the container termination
-	time.Sleep(5 * time.Second)
+	time.Sleep(15 * time.Second)
 
 	tLog(t, "removing host-3 from cluster")
 	resp, err := cluster.Client().RemoveHost(ctx, &api.RemoveHostPayload{
@@ -175,7 +175,7 @@ func TestForcedHostRemovalWithDatabase(t *testing.T) {
 	require.NoError(t, err, "database not healthy with 2 nodes")
 
 	tLog(t, "verifying cluster has 2 hosts")
-	err = waitForHostCount(ctx, cluster.Client(), 2, 30*time.Second)
+	err = waitForHostCount(ctx, cluster.Client(), 2, 60*time.Second)
 	require.NoError(t, err, "cluster should have 2 hosts")
 
 	// Ensure host-3 is not in the list

clustertest/etcd_mode_change_test.go

@@ -11,6 +11,7 @@ import (
 )
 
 func TestPromoteClientToServer(t *testing.T) {
+	t.Parallel()
 
 	tLog(t, "initializing cluster with 3 etcd servers and 1 etcd client")
 
@@ -55,6 +56,7 @@ func TestPromoteClientToServer(t *testing.T) {
 }
 
 func TestDemoteServerToClient(t *testing.T) {
+	t.Parallel()
 
 	tLog(t, "initializing cluster with 4 etcd servers")
 

docs/development/supported-services.md

@@ -807,8 +807,8 @@ for state transitions. Key patterns to replicate for a new service type:
 |---------|-------------|-------------------|
 | Single-host provision | `TestProvisionMCPService` | Service reaches `"running"` state |
 | Multi-host provision | `TestProvisionMultiHostMCPService` | One instance per host, all reach `"running"` |
-| Add to existing DB | `TestUpdateDatabaseAddService` | Service added without affecting database |
-| Remove from DB | `TestUpdateDatabaseRemoveService` | Empty `Services` array removes the service |
+| Add to existing DB | `TestUpdateDatabaseService/Add` | Service added without affecting database |
+| Remove from DB | `TestUpdateDatabaseService/Remove` | Empty `Services` array removes the service |
 | Stability | `TestUpdateDatabaseServiceStable` | Unrelated DB update doesn't recreate service (checks `created_at` and `container_id` unchanged) |
 | Bad version | `TestProvisionMCPServiceUnsupportedVersion` | Unregistered version fails task, DB goes to `"failed"` |
 | Recovery | `TestProvisionMCPServiceRecovery` | Failed DB recovered by updating with valid version |

e2e/add_replica_test.go

@@ -38,7 +38,11 @@ func TestAddReplica(t *testing.T) {
 			Port:        pointerTo(0),
 			PatroniPort: pointerTo(0),
 			Nodes: []*api.DatabaseNodeSpec{
-				{Name: "n1", HostIds: []api.Identifier{api.Identifier(host1)}},
+				{
+					Name:           "n1",
+					HostIds:        []api.Identifier{api.Identifier(host1)},
+					PostgresqlConf: fastCheckpointConf,
+				},
 				{Name: "n2", HostIds: []api.Identifier{api.Identifier(host2)}},
 			},
 		},
@@ -78,6 +82,7 @@ func TestAddReplica(t *testing.T) {
 						api.Identifier(host1),
 						api.Identifier(host3),
 					},
+					PostgresqlConf: fastCheckpointConf,
 				},
 				{Name: "n2", HostIds: []api.Identifier{api.Identifier(host2)}},
 			},

e2e/database_test.go

@@ -405,7 +405,7 @@ func (f *DatabaseFixture) WaitForReplication(ctx context.Context, t testing.TB,
 			f.WithConnection(ctx, primaryOpts, t, func(conn *pgx.Conn) {
 				var synced bool
 
-				row := conn.QueryRow(ctx, "CALL spock.wait_for_sync_event(true, $1, $2::pg_lsn, 30);", peerNode, lsn)
+				row := conn.QueryRow(ctx, "CALL spock.wait_for_sync_event(true, $1, $2::pg_lsn, 60);", peerNode, lsn)
 				require.NoError(t, row.Scan(&synced))
 				assert.True(t, synced)
 			})

e2e/db_create_test.go

@@ -57,10 +57,14 @@ func testCreateDB(t *testing.T, nodeCount int, deployReplicas bool) {
 		if deployReplicas {
 			hosts = append(hosts, controlplane.Identifier(host3))
 		}
-		nodes = append(nodes, &controlplane.DatabaseNodeSpec{
+		node := &controlplane.DatabaseNodeSpec{
 			Name:    fmt.Sprintf("n%d", i),
 			HostIds: hosts,
-		})
+		}
+		if deployReplicas {
+			node.PostgresqlConf = fastCheckpointConf
+		}
+		nodes = append(nodes, node)
 	}
 
 	db := fixture.NewDatabaseFixture(ctx, t, &controlplane.CreateDatabaseRequest{
@@ -141,12 +145,12 @@ func testCreateDB(t *testing.T, nodeCount int, deployReplicas bool) {
 				row := conn.QueryRow(ctx, "SELECT pg_current_wal_lsn() AS commit_lsn")
 				require.NoError(t, row.Scan(&commitLSN))
 
-				// Wait for up to 10 seconds and verify commit_lsn has replayed
+				// Wait for up to 20 seconds and verify commit_lsn has replayed
 				// This should prevent flaky tests
 				var replayLSN string
 				var hasReplayed bool
 
-				for i := 0; i < 10; i++ {
+				for i := 0; i < 20; i++ {
 					rows, err := conn.Query(ctx, "SELECT replay_lsn, (replay_lsn >= $1::pg_lsn) AS has_replayed FROM pg_stat_replication", commitLSN)
 					require.NoError(t, err)
 					defer rows.Close()

e2e/db_update_add_node_test.go

@@ -175,7 +175,7 @@ func verifyWALReplay(ctx context.Context, t *testing.T, conn *pgx.Conn) {
 	var commitLSN string
 	require.NoError(t, conn.QueryRow(ctx, "SELECT pg_current_wal_lsn()").Scan(&commitLSN))
 
-	for i := 0; i < 10; i++ {
+	for i := 0; i < 20; i++ {
 		rows, err := conn.Query(ctx, "SELECT (replay_lsn >= $1::pg_lsn) AS has_replayed FROM pg_stat_replication", commitLSN)
 		require.NoError(t, err)
 		defer rows.Close()

e2e/main_test.go

@@ -71,3 +71,11 @@ func TestMain(m *testing.M) {
 func pointerTo[T any](v T) *T {
 	return &v
 }
+
+// fastCheckpointConf shortens checkpoint_timeout/checkpoint_completion_target
+// so that replica bootstrap (pg_basebackup) on the primary isn't delayed by
+// the production checkpoint defaults in short-lived test databases.
+var fastCheckpointConf = map[string]any{
+	"checkpoint_timeout":           "30s",
+	"checkpoint_completion_target": "0",
+}