From 537f46e8f035dd13c73d912fff358d0d67106e4b Mon Sep 17 00:00:00 2001
From: Wenting Wu <wentingwu@microsoft.com>
Date: Fri, 10 Jul 2026 12:39:51 -0400
Subject: [PATCH 1/6] test: add kill-operator-pod and kill-primary-pod chaos
 operations

Add the first two chaos fault-injection operations to the long-haul
operation scheduler:

- kill-operator-pod: deletes the operator pod via its Deployment selector
  and waits for the Deployment to become Available again. Asserts the CNPG
  data plane is unaffected by an operator restart (small write-failure
  budget).
- kill-primary-pod: deletes the CNPG primary pod to exercise the automatic
  failover path, guarded by an HA precondition (instancesPerNode>=2).

Both plug into the existing Operation interface and are selected by the
weighted scheduler (one disruptive op at a time, cooldown + steady-state
gate). Recovery is judged by the health monitor and workload verifier.

Adds ClusterClient.GetPrimaryInstance (reads CNPG Cluster
status.currentPrimary) and DeletePod, a LONGHAUL_OPERATOR_NAMESPACE
config knob (default documentdb-operator), unit tests, and README
operations + RBAC notes.

Controlled failover was intentionally excluded: DocumentDB exposes no
single-cluster manual switchover, so it would only test upstream CNPG.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Signed-off-by: Wenting Wu <wentingwu@microsoft.com>
---
 test/longhaul/README.md                       |  23 +++
 test/longhaul/cmd/longhaul/main.go            |   2 +
 test/longhaul/config/config.go                |  38 +++--
 test/longhaul/config/config_test.go           |  16 ++
 test/longhaul/go.mod                          |   2 +-
 test/longhaul/monitor/health.go               |   9 +
 test/longhaul/monitor/health_test.go          |   8 +-
 test/longhaul/monitor/k8sclient.go            |  27 ++-
 test/longhaul/operations/kill_operator.go     | 156 ++++++++++++++++++
 .../longhaul/operations/kill_operator_test.go | 118 +++++++++++++
 test/longhaul/operations/kill_primary.go      |  76 +++++++++
 test/longhaul/operations/kill_primary_test.go |  75 +++++++++
 test/longhaul/operations/scale_test.go        |  18 ++
 13 files changed, 552 insertions(+), 16 deletions(-)
 create mode 100644 test/longhaul/operations/kill_operator.go
 create mode 100644 test/longhaul/operations/kill_operator_test.go
 create mode 100644 test/longhaul/operations/kill_primary.go
 create mode 100644 test/longhaul/operations/kill_primary_test.go

diff --git a/test/longhaul/README.md b/test/longhaul/README.md
index a7d311e94..c9fde0247 100644
--- a/test/longhaul/README.md
+++ b/test/longhaul/README.md
@@ -125,6 +125,7 @@ All configuration is via environment variables.
 | `LONGHAUL_DOCUMENTDB_URI` | Yes | — | Connection string to the DocumentDB gateway. |
 | `LONGHAUL_CLUSTER_NAME` | Yes | — | Name of the target DocumentDB cluster CR. |
 | `LONGHAUL_NAMESPACE` | No | `default` | Kubernetes namespace of the target cluster. |
+| `LONGHAUL_OPERATOR_NAMESPACE` | No | `documentdb-operator` | Namespace of the DocumentDB operator Deployment (target of the `kill-operator-pod` chaos op). |
 | `LONGHAUL_MAX_DURATION` | No | `30m` | Max test duration. Use `0s` for run-until-failure. |
 | `LONGHAUL_NUM_WRITERS` | No | `5` | Number of concurrent writers. |
 | `LONGHAUL_OP_COOLDOWN` | No | `5m` | Cooldown between management operations. |
@@ -134,6 +135,28 @@ All configuration is via environment variables.
 | `LONGHAUL_REPORT_INTERVAL` | No | `1h` | How often to write checkpoint reports to ConfigMap. |
 | `LONGHAUL_RESET_DATA` | No | `false` | If `true`, drop the workload collection on startup. Off by default so a Deployment pod restart preserves durability history. |
 
+## Operations
+
+The scheduler runs one disruptive operation at a time, gated by steady state and
+a global cooldown. Current operations:
+
+| Operation | Kind | Notes |
+|-----------|------|-------|
+| `scale-up` / `scale-down` | Topology | Adjusts `spec.instancesPerNode` within `[MIN, MAX]`. |
+| `upgrade-documentdb` | Topology | In-place version upgrade; requires HA (`instancesPerNode>=2`). |
+| `kill-operator-pod` | Chaos | Deletes the operator pod; asserts the data plane keeps serving (small write-failure budget). |
+| `kill-primary-pod` | Chaos | Deletes the CNPG primary pod to exercise automatic failover; requires HA (`instancesPerNode>=2`). |
+
+### RBAC for chaos operations
+
+Beyond the base RBAC the driver already needs, the chaos operations require the
+driver ServiceAccount to be granted (in `deploy/rbac.yaml`):
+
+- **`kill-primary-pod`** — `get`/`list` on `clusters.postgresql.cnpg.io` (to read
+  `status.currentPrimary`) and `delete` on `pods` in the cluster namespace.
+- **`kill-operator-pod`** — `get` on `deployments` and `get`/`list`/`delete` on
+  `pods` in the operator namespace (`LONGHAUL_OPERATOR_NAMESPACE`).
+
 ## CI Safety
 
 The long haul test binary is deployed as a Kubernetes Deployment on a dedicated AKS
diff --git a/test/longhaul/cmd/longhaul/main.go b/test/longhaul/cmd/longhaul/main.go
index c7f375bd8..5195d49d7 100644
--- a/test/longhaul/cmd/longhaul/main.go
+++ b/test/longhaul/cmd/longhaul/main.go
@@ -140,6 +140,8 @@ func run(cfg config.Config) int {
 		operations.NewScaleUp(clusterClient, healthMon, cfg.MaxInstances, cfg.RecoveryTimeout),
 		operations.NewScaleDown(clusterClient, healthMon, cfg.MinInstances, cfg.RecoveryTimeout),
 		operations.NewUpgradeDocumentDB(clusterClient, k8sClientset, healthMon, j, cfg.Namespace, cfg.RecoveryTimeout),
+		operations.NewKillOperatorPod(k8sClientset, cfg.OperatorNamespace, cfg.RecoveryTimeout),
+		operations.NewKillPrimaryPod(clusterClient, healthMon, cfg.RecoveryTimeout),
 	}
 
 	// Start operation scheduler.
diff --git a/test/longhaul/config/config.go b/test/longhaul/config/config.go
index 5665bcd77..918431c35 100644
--- a/test/longhaul/config/config.go
+++ b/test/longhaul/config/config.go
@@ -18,6 +18,10 @@ const (
 	EnvNamespace   = "LONGHAUL_NAMESPACE"
 	EnvClusterName = "LONGHAUL_CLUSTER_NAME"
 
+	// EnvOperatorNamespace is the namespace where the DocumentDB operator
+	// Deployment runs (target of the kill-operator-pod chaos op).
+	EnvOperatorNamespace = "LONGHAUL_OPERATOR_NAMESPACE"
+
 	// Workload and operation tuning.
 	EnvDocumentDBURI   = "LONGHAUL_DOCUMENTDB_URI"
 	EnvNumWriters      = "LONGHAUL_NUM_WRITERS"
@@ -47,6 +51,10 @@ type Config struct {
 	// ClusterName is the name of the target DocumentDB cluster CR.
 	ClusterName string
 
+	// OperatorNamespace is the namespace of the DocumentDB operator Deployment,
+	// targeted by the kill-operator-pod chaos operation.
+	OperatorNamespace string
+
 	// DocumentDBURI is the DocumentDB connection string for data-plane workload.
 	DocumentDBURI string
 
@@ -82,17 +90,18 @@ type Config struct {
 // DefaultConfig returns a Config with safe defaults for local development.
 func DefaultConfig() Config {
 	return Config{
-		MaxDuration:     30 * time.Minute,
-		Namespace:       "default",
-		ClusterName:     "",
-		DocumentDBURI:   "",
-		NumWriters:      5,
-		OpCooldown:      5 * time.Minute,
-		RecoveryTimeout: 5 * time.Minute,
-		SteadyStateWait: 60 * time.Second,
-		MinInstances:    1,
-		MaxInstances:    3,
-		ReportInterval:  1 * time.Hour,
+		MaxDuration:       30 * time.Minute,
+		Namespace:         "default",
+		ClusterName:       "",
+		OperatorNamespace: "documentdb-operator",
+		DocumentDBURI:     "",
+		NumWriters:        5,
+		OpCooldown:        5 * time.Minute,
+		RecoveryTimeout:   5 * time.Minute,
+		SteadyStateWait:   60 * time.Second,
+		MinInstances:      1,
+		MaxInstances:      3,
+		ReportInterval:    1 * time.Hour,
 	}
 }
 
@@ -117,6 +126,10 @@ func LoadFromEnv() (Config, error) {
 		cfg.ClusterName = v
 	}
 
+	if v := os.Getenv(EnvOperatorNamespace); v != "" {
+		cfg.OperatorNamespace = v
+	}
+
 	if v := os.Getenv(EnvDocumentDBURI); v != "" {
 		cfg.DocumentDBURI = v
 	}
@@ -195,6 +208,9 @@ func (c *Config) Validate() error {
 	if c.ClusterName == "" {
 		return fmt.Errorf("cluster name must not be empty")
 	}
+	if c.OperatorNamespace == "" {
+		return fmt.Errorf("operator namespace must not be empty")
+	}
 	if c.NumWriters < 1 {
 		return fmt.Errorf("num writers must be at least 1, got %d", c.NumWriters)
 	}
diff --git a/test/longhaul/config/config_test.go b/test/longhaul/config/config_test.go
index 506803f52..c02196b87 100644
--- a/test/longhaul/config/config_test.go
+++ b/test/longhaul/config/config_test.go
@@ -17,6 +17,7 @@ var _ = Describe("Config", func() {
 			Expect(cfg.MaxDuration).To(Equal(30 * time.Minute))
 			Expect(cfg.Namespace).To(Equal("default"))
 			Expect(cfg.ClusterName).To(BeEmpty())
+			Expect(cfg.OperatorNamespace).To(Equal("documentdb-operator"))
 			Expect(cfg.NumWriters).To(Equal(5))
 			Expect(cfg.OpCooldown).To(Equal(5 * time.Minute))
 			Expect(cfg.RecoveryTimeout).To(Equal(5 * time.Minute))
@@ -32,6 +33,7 @@ var _ = Describe("Config", func() {
 		BeforeEach(func() {
 			for _, k := range []string{
 				EnvEnabled, EnvMaxDuration, EnvNamespace, EnvClusterName,
+				EnvOperatorNamespace,
 				EnvDocumentDBURI, EnvNumWriters,
 				EnvOpCooldown, EnvRecoveryTimeout, EnvSteadyStateWait,
 				EnvMinInstances, EnvMaxInstances, EnvReportInterval,
@@ -69,6 +71,13 @@ var _ = Describe("Config", func() {
 			Expect(cfg.ClusterName).To(Equal("my-cluster"))
 		})
 
+		It("parses OperatorNamespace from env", func() {
+			GinkgoT().Setenv(EnvOperatorNamespace, "custom-operator-ns")
+			cfg, err := LoadFromEnv()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(cfg.OperatorNamespace).To(Equal("custom-operator-ns"))
+		})
+
 		It("returns error for invalid MaxDuration", func() {
 			GinkgoT().Setenv(EnvMaxDuration, "not-a-duration")
 			_, err := LoadFromEnv()
@@ -124,6 +133,13 @@ var _ = Describe("Config", func() {
 			Expect(cfg.Validate()).To(MatchError(ContainSubstring("cluster name")))
 		})
 
+		It("fails when OperatorNamespace is empty", func() {
+			cfg := DefaultConfig()
+			cfg.ClusterName = "test"
+			cfg.OperatorNamespace = ""
+			Expect(cfg.Validate()).To(MatchError(ContainSubstring("operator namespace")))
+		})
+
 		It("fails when MaxDuration is negative", func() {
 			cfg := DefaultConfig()
 			cfg.ClusterName = "test"
diff --git a/test/longhaul/go.mod b/test/longhaul/go.mod
index 061577178..dd41c0997 100644
--- a/test/longhaul/go.mod
+++ b/test/longhaul/go.mod
@@ -3,6 +3,7 @@ module github.com/documentdb/documentdb-operator/test/longhaul
 go 1.25.11
 
 require (
+	github.com/cloudnative-pg/cloudnative-pg v1.28.1
 	github.com/documentdb/documentdb-operator v0.0.0-00010101000000-000000000000
 	github.com/documentdb/documentdb-operator/test/shared v0.0.0-00010101000000-000000000000
 	github.com/onsi/ginkgo/v2 v2.28.1
@@ -24,7 +25,6 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudnative-pg/barman-cloud v0.4.1-0.20260108104508-ced266c145f5 // indirect
-	github.com/cloudnative-pg/cloudnative-pg v1.28.1 // indirect
 	github.com/cloudnative-pg/cnpg-i v0.3.1 // indirect
 	github.com/cloudnative-pg/machinery v0.3.3 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
diff --git a/test/longhaul/monitor/health.go b/test/longhaul/monitor/health.go
index 0c5e95163..939f48056 100644
--- a/test/longhaul/monitor/health.go
+++ b/test/longhaul/monitor/health.go
@@ -49,6 +49,15 @@ type ClusterClient interface {
 
 	// UpgradeDocumentDB patches spec.documentDBVersion and spec.schemaVersion="auto".
 	UpgradeDocumentDB(ctx context.Context, version string) error
+
+	// GetPrimaryInstance returns the name of the pod currently serving as the
+	// CNPG primary (from Cluster.status.currentPrimary). The pod name equals
+	// the CNPG instance name. Returns an error if no primary is known yet.
+	GetPrimaryInstance(ctx context.Context) (string, error)
+
+	// DeletePod deletes the named pod in the cluster namespace. Used by chaos
+	// operations to inject pod-loss faults.
+	DeletePod(ctx context.Context, name string) error
 }
 
 // HealthMonitor continuously monitors cluster health and tracks steady-state.
diff --git a/test/longhaul/monitor/health_test.go b/test/longhaul/monitor/health_test.go
index 1ad903fd9..4d5385e1e 100644
--- a/test/longhaul/monitor/health_test.go
+++ b/test/longhaul/monitor/health_test.go
@@ -41,9 +41,11 @@ func (f *fakeClusterClient) GetClusterHealth(_ context.Context) (ClusterHealth,
 func (f *fakeClusterClient) GetCurrentDocumentDBImageTag(_ context.Context) (string, error) {
 	return "", nil
 }
-func (f *fakeClusterClient) GetInstancesPerNode(_ context.Context) (int, error)  { return 1, nil }
-func (f *fakeClusterClient) ScaleCluster(_ context.Context, _ int) error         { return nil }
-func (f *fakeClusterClient) UpgradeDocumentDB(_ context.Context, _ string) error { return nil }
+func (f *fakeClusterClient) GetInstancesPerNode(_ context.Context) (int, error)   { return 1, nil }
+func (f *fakeClusterClient) ScaleCluster(_ context.Context, _ int) error          { return nil }
+func (f *fakeClusterClient) UpgradeDocumentDB(_ context.Context, _ string) error  { return nil }
+func (f *fakeClusterClient) GetPrimaryInstance(_ context.Context) (string, error) { return "", nil }
+func (f *fakeClusterClient) DeletePod(_ context.Context, _ string) error          { return nil }
 
 var _ = Describe("HealthMonitor", func() {
 	Describe("IsSteadyState", func() {
diff --git a/test/longhaul/monitor/k8sclient.go b/test/longhaul/monitor/k8sclient.go
index b51ffdd82..f5b9807df 100644
--- a/test/longhaul/monitor/k8sclient.go
+++ b/test/longhaul/monitor/k8sclient.go
@@ -19,6 +19,8 @@ import (
 	metricsv "k8s.io/metrics/pkg/client/clientset/versioned"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
+
 	previewv1 "github.com/documentdb/documentdb-operator/api/preview"
 	shareddb "github.com/documentdb/documentdb-operator/test/shared/documentdb"
 	sharedk8s "github.com/documentdb/documentdb-operator/test/shared/k8s"
@@ -61,7 +63,7 @@ func NewK8sClusterClient(cfg K8sClientConfig) (*K8sClusterClient, error) {
 		return nil, fmt.Errorf("failed to create clientset: %w", err)
 	}
 
-	scheme, err := shareddb.NewScheme()
+	scheme, err := shareddb.NewScheme(cnpgv1.AddToScheme)
 	if err != nil {
 		return nil, fmt.Errorf("failed to build scheme: %w", err)
 	}
@@ -227,6 +229,29 @@ func (k *K8sClusterClient) UpgradeDocumentDB(ctx context.Context, version string
 	return nil
 }
 
+// GetPrimaryInstance reads status.currentPrimary from the CNPG Cluster that
+// backs this DocumentDB. The CNPG Cluster name equals the DocumentDB CR name,
+// and the returned instance name equals the primary pod name.
+func (k *K8sClusterClient) GetPrimaryInstance(ctx context.Context) (string, error) {
+	var cluster cnpgv1.Cluster
+	key := types.NamespacedName{Namespace: k.namespace, Name: k.clusterName}
+	if err := k.crClient.Get(ctx, key, &cluster); err != nil {
+		return "", fmt.Errorf("failed to get CNPG Cluster: %w", err)
+	}
+	if cluster.Status.CurrentPrimary == "" {
+		return "", fmt.Errorf("CNPG Cluster %s has no current primary yet", k.clusterName)
+	}
+	return cluster.Status.CurrentPrimary, nil
+}
+
+// DeletePod deletes the named pod in the cluster namespace.
+func (k *K8sClusterClient) DeletePod(ctx context.Context, name string) error {
+	if err := k.clientset.CoreV1().Pods(k.namespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil {
+		return fmt.Errorf("failed to delete pod %s/%s: %w", k.namespace, name, err)
+	}
+	return nil
+}
+
 // GetPodMetrics queries metrics-server for pod resource usage.
 // Returns nil, nil if metrics-server is not available.
 func (k *K8sClusterClient) GetPodMetrics(ctx context.Context) ([]PodMetrics, error) {
diff --git a/test/longhaul/operations/kill_operator.go b/test/longhaul/operations/kill_operator.go
new file mode 100644
index 000000000..306503f63
--- /dev/null
+++ b/test/longhaul/operations/kill_operator.go
@@ -0,0 +1,156 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+package operations
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/documentdb/documentdb-operator/test/longhaul/journal"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/kubernetes"
+)
+
+// OperatorDeploymentName is the fixed name of the operator Deployment. The
+// operator is a cluster singleton, so this name is stable across installs.
+const OperatorDeploymentName = "documentdb-operator"
+
+// KillOperatorPod deletes the running operator pod to verify that an operator
+// restart does not disrupt the data plane. The CNPG-managed database keeps
+// serving reads and writes while the Deployment reschedules the control plane,
+// so the workload verifier should observe (near) zero write failures. Recovery
+// is asserted by the Deployment returning to Available.
+type KillOperatorPod struct {
+	clientset  kubernetes.Interface
+	namespace  string
+	deployment string
+	recovery   time.Duration
+}
+
+// NewKillOperatorPod creates a KillOperatorPod operation targeting the operator
+// Deployment in the given namespace.
+func NewKillOperatorPod(clientset kubernetes.Interface, namespace string, recovery time.Duration) *KillOperatorPod {
+	return &KillOperatorPod{
+		clientset:  clientset,
+		namespace:  namespace,
+		deployment: OperatorDeploymentName,
+		recovery:   recovery,
+	}
+}
+
+func (k *KillOperatorPod) Name() string { return "kill-operator-pod" }
+
+func (k *KillOperatorPod) Weight() int { return 2 }
+
+// Precondition requires the operator Deployment to exist and currently be
+// Available, so the fault isn't stacked on an already-restarting operator.
+func (k *KillOperatorPod) Precondition(ctx context.Context) (bool, string) {
+	dep, err := k.getDeployment(ctx)
+	if err != nil {
+		return false, fmt.Sprintf("cannot get operator deployment: %v", err)
+	}
+	if !isDeploymentAvailable(dep) {
+		return false, "operator deployment not currently available"
+	}
+	return true, ""
+}
+
+func (k *KillOperatorPod) Execute(ctx context.Context) error {
+	dep, err := k.getDeployment(ctx)
+	if err != nil {
+		return fmt.Errorf("get operator deployment: %w", err)
+	}
+
+	// Resolve the pod set from the Deployment's own selector so we don't
+	// depend on the release-name-derived "app" label value.
+	selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels).String()
+	pods, err := k.clientset.CoreV1().Pods(k.namespace).List(ctx, metav1.ListOptions{LabelSelector: selector})
+	if err != nil {
+		return fmt.Errorf("list operator pods: %w", err)
+	}
+
+	target := oldestRunningPod(pods.Items)
+	if target == "" {
+		return fmt.Errorf("no running operator pod found for selector %q", selector)
+	}
+	if err := k.clientset.CoreV1().Pods(k.namespace).Delete(ctx, target, metav1.DeleteOptions{}); err != nil {
+		return fmt.Errorf("delete operator pod %s: %w", target, err)
+	}
+
+	// Wait for the Deployment to reschedule and become Available again.
+	recoveryCtx, cancel := context.WithTimeout(ctx, k.recovery)
+	defer cancel()
+	return k.waitForDeploymentAvailable(recoveryCtx)
+}
+
+// OutagePolicy tolerates only a small number of write failures: an operator
+// restart must not take down the data plane. A non-zero budget absorbs
+// coincidental blips (e.g. a scrape or client reconnect) without flagging a
+// false policy violation.
+func (k *KillOperatorPod) OutagePolicy() journal.OutagePolicy {
+	return journal.OutagePolicy{
+		AllowedWriteFailures: 5,
+		MustRecoverWithin:    k.recovery,
+	}
+}
+
+func (k *KillOperatorPod) getDeployment(ctx context.Context) (*appsv1.Deployment, error) {
+	return k.clientset.AppsV1().Deployments(k.namespace).Get(ctx, k.deployment, metav1.GetOptions{})
+}
+
+func (k *KillOperatorPod) waitForDeploymentAvailable(ctx context.Context) error {
+	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
+	for {
+		if dep, err := k.getDeployment(ctx); err == nil && isDeploymentAvailable(dep) {
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("timed out waiting for operator deployment to become available: %w", ctx.Err())
+		case <-ticker.C:
+		}
+	}
+}
+
+// isDeploymentAvailable reports whether the Deployment has its full desired
+// replica count ready with none unavailable and the observed generation caught
+// up to the latest spec.
+func isDeploymentAvailable(dep *appsv1.Deployment) bool {
+	if dep == nil {
+		return false
+	}
+	desired := int32(1)
+	if dep.Spec.Replicas != nil {
+		desired = *dep.Spec.Replicas
+	}
+	if dep.Status.ObservedGeneration < dep.Generation {
+		return false
+	}
+	return dep.Status.ReadyReplicas >= desired && dep.Status.UnavailableReplicas == 0
+}
+
+// oldestRunningPod returns the name of the oldest pod in the Running phase, or
+// "" if none are running. Targeting the oldest makes the choice deterministic.
+func oldestRunningPod(pods []corev1.Pod) string {
+	name := ""
+	var oldest time.Time
+	for i := range pods {
+		p := &pods[i]
+		if p.Status.Phase != corev1.PodRunning || p.DeletionTimestamp != nil {
+			continue
+		}
+		ts := p.CreationTimestamp.Time
+		if name == "" || ts.Before(oldest) {
+			name = p.Name
+			oldest = ts
+		}
+	}
+	return name
+}
diff --git a/test/longhaul/operations/kill_operator_test.go b/test/longhaul/operations/kill_operator_test.go
new file mode 100644
index 000000000..f05b04b9b
--- /dev/null
+++ b/test/longhaul/operations/kill_operator_test.go
@@ -0,0 +1,118 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+package operations
+
+import (
+	"context"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+)
+
+const opNS = "documentdb-operator"
+
+func operatorDeployment(desired, ready, unavailable int32, gen, observed int64) *appsv1.Deployment {
+	return &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       OperatorDeploymentName,
+			Namespace:  opNS,
+			Generation: gen,
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: &desired,
+			Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "documentdb-operator"}},
+		},
+		Status: appsv1.DeploymentStatus{
+			ReadyReplicas:       ready,
+			UnavailableReplicas: unavailable,
+			ObservedGeneration:  observed,
+		},
+	}
+}
+
+func operatorPod(name string, phase corev1.PodPhase, ageSeconds int) *corev1.Pod {
+	return &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:              name,
+			Namespace:         opNS,
+			Labels:            map[string]string{"app": "documentdb-operator"},
+			CreationTimestamp: metav1.NewTime(time.Now().Add(-time.Duration(ageSeconds) * time.Second)),
+		},
+		Status: corev1.PodStatus{Phase: phase},
+	}
+}
+
+var _ = Describe("KillOperatorPod", func() {
+	It("Name is kill-operator-pod and Weight is 2", func() {
+		k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, time.Minute)
+		Expect(k.Name()).To(Equal("kill-operator-pod"))
+		Expect(k.Weight()).To(Equal(2))
+	})
+
+	It("OutagePolicy tolerates only a small write-failure budget", func() {
+		k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, 2*time.Minute)
+		p := k.OutagePolicy()
+		Expect(p.AllowedWriteFailures).To(Equal(int64(5)))
+		Expect(p.MustRecoverWithin).To(Equal(2 * time.Minute))
+	})
+
+	Describe("Precondition", func() {
+		It("skips when the deployment is missing", func() {
+			k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, time.Minute)
+			ok, reason := k.Precondition(context.Background())
+			Expect(ok).To(BeFalse())
+			Expect(reason).To(ContainSubstring("cannot get operator deployment"))
+		})
+
+		It("skips when the deployment is not available", func() {
+			dep := operatorDeployment(1, 0, 1, 1, 1)
+			k := NewKillOperatorPod(fake.NewSimpleClientset(dep), opNS, time.Minute)
+			ok, reason := k.Precondition(context.Background())
+			Expect(ok).To(BeFalse())
+			Expect(reason).To(ContainSubstring("not currently available"))
+		})
+
+		It("is eligible when the deployment is available", func() {
+			dep := operatorDeployment(1, 1, 0, 1, 1)
+			k := NewKillOperatorPod(fake.NewSimpleClientset(dep), opNS, time.Minute)
+			ok, _ := k.Precondition(context.Background())
+			Expect(ok).To(BeTrue())
+		})
+	})
+
+	Describe("Execute", func() {
+		It("deletes the oldest running operator pod and returns once available", func() {
+			dep := operatorDeployment(1, 1, 0, 1, 1)
+			newer := operatorPod("op-new", corev1.PodRunning, 10)
+			older := operatorPod("op-old", corev1.PodRunning, 100)
+			cs := fake.NewSimpleClientset(dep, newer, older)
+			k := NewKillOperatorPod(cs, opNS, time.Minute)
+
+			err := k.Execute(context.Background())
+			Expect(err).NotTo(HaveOccurred())
+
+			_, getErr := cs.CoreV1().Pods(opNS).Get(context.Background(), "op-old", metav1.GetOptions{})
+			Expect(getErr).To(HaveOccurred(), "oldest pod should have been deleted")
+			_, getErr = cs.CoreV1().Pods(opNS).Get(context.Background(), "op-new", metav1.GetOptions{})
+			Expect(getErr).NotTo(HaveOccurred(), "newer pod should be untouched")
+		})
+
+		It("fails when no running pod matches the selector", func() {
+			dep := operatorDeployment(1, 1, 0, 1, 1)
+			pending := operatorPod("op-pending", corev1.PodPending, 10)
+			cs := fake.NewSimpleClientset(dep, pending)
+			k := NewKillOperatorPod(cs, opNS, time.Minute)
+
+			err := k.Execute(context.Background())
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("no running operator pod"))
+		})
+	})
+})
diff --git a/test/longhaul/operations/kill_primary.go b/test/longhaul/operations/kill_primary.go
new file mode 100644
index 000000000..aa2199db8
--- /dev/null
+++ b/test/longhaul/operations/kill_primary.go
@@ -0,0 +1,76 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+package operations
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/documentdb/documentdb-operator/test/longhaul/journal"
+	"github.com/documentdb/documentdb-operator/test/longhaul/monitor"
+)
+
+// KillPrimaryPod deletes the CNPG primary pod to exercise the automatic
+// failover path: CNPG must promote a standby, and the cluster must return to
+// steady state within the recovery budget. The continuous workload verifier
+// independently catches any data loss caused by the failover.
+type KillPrimaryPod struct {
+	client    monitor.ClusterClient
+	healthMon *monitor.HealthMonitor
+	recovery  time.Duration
+}
+
+// NewKillPrimaryPod creates a KillPrimaryPod operation.
+func NewKillPrimaryPod(client monitor.ClusterClient, health *monitor.HealthMonitor, recovery time.Duration) *KillPrimaryPod {
+	return &KillPrimaryPod{
+		client:    client,
+		healthMon: health,
+		recovery:  recovery,
+	}
+}
+
+func (k *KillPrimaryPod) Name() string { return "kill-primary-pod" }
+
+func (k *KillPrimaryPod) Weight() int { return 2 }
+
+// Precondition requires at least one standby (instancesPerNode>=2). Killing the
+// sole instance of a single-instance cluster would cause guaranteed downtime
+// with no failover target — a true-but-useless policy violation. The same guard
+// (and rationale) is used by UpgradeDocumentDB; skips don't consume the
+// scheduler cooldown, so this is free to re-evaluate on the next tick.
+func (k *KillPrimaryPod) Precondition(ctx context.Context) (bool, string) {
+	ipn, err := k.client.GetInstancesPerNode(ctx)
+	if err != nil {
+		return false, fmt.Sprintf("cannot read instancesPerNode: %v", err)
+	}
+	if ipn < 2 {
+		return false, fmt.Sprintf("instancesPerNode=%d (no HA standby) — killing primary would cause real downtime; skipping", ipn)
+	}
+	return true, ""
+}
+
+func (k *KillPrimaryPod) Execute(ctx context.Context) error {
+	primary, err := k.client.GetPrimaryInstance(ctx)
+	if err != nil {
+		return fmt.Errorf("get primary instance: %w", err)
+	}
+	if err := k.client.DeletePod(ctx, primary); err != nil {
+		return fmt.Errorf("delete primary pod %s: %w", primary, err)
+	}
+
+	// Wait for CNPG to elect a new primary and the cluster to settle.
+	recoveryCtx, cancel := context.WithTimeout(ctx, k.recovery)
+	defer cancel()
+	return k.healthMon.WaitForSteadyState(recoveryCtx)
+}
+
+// OutagePolicy allows a moderate write-failure budget: failover briefly
+// interrupts writes while a standby is promoted, similar to scale-down.
+func (k *KillPrimaryPod) OutagePolicy() journal.OutagePolicy {
+	return journal.OutagePolicy{
+		AllowedWriteFailures: 50,
+		MustRecoverWithin:    k.recovery,
+	}
+}
diff --git a/test/longhaul/operations/kill_primary_test.go b/test/longhaul/operations/kill_primary_test.go
new file mode 100644
index 000000000..aab917591
--- /dev/null
+++ b/test/longhaul/operations/kill_primary_test.go
@@ -0,0 +1,75 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+package operations
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/documentdb/documentdb-operator/test/longhaul/journal"
+	"github.com/documentdb/documentdb-operator/test/longhaul/monitor"
+)
+
+var _ = Describe("KillPrimaryPod", func() {
+	It("Name is kill-primary-pod and Weight is 2", func() {
+		k := NewKillPrimaryPod(&fakeClient{}, nil, time.Minute)
+		Expect(k.Name()).To(Equal("kill-primary-pod"))
+		Expect(k.Weight()).To(Equal(2))
+	})
+
+	It("OutagePolicy allows a moderate failover write-failure budget", func() {
+		k := NewKillPrimaryPod(&fakeClient{}, nil, 3*time.Minute)
+		p := k.OutagePolicy()
+		Expect(p.AllowedWriteFailures).To(Equal(int64(50)))
+		Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute))
+	})
+
+	DescribeTable("Precondition",
+		func(ipn int, ipnErr error, wantOK bool, wantReasonHas string) {
+			c := &fakeClient{instancesPerNode: ipn, ipnErr: ipnErr}
+			k := NewKillPrimaryPod(c, nil, time.Minute)
+
+			ok, reason := k.Precondition(context.Background())
+			Expect(ok).To(Equal(wantOK), "reason=%q", reason)
+			if wantReasonHas != "" {
+				Expect(reason).To(ContainSubstring(wantReasonHas))
+			}
+		},
+		Entry("single-instance: ipn=1 -> skip", 1, nil, false, "no HA standby"),
+		Entry("read error -> skip", 0, errors.New("boom"), false, "cannot read instancesPerNode"),
+		Entry("HA: ipn=2 -> eligible", 2, nil, true, ""),
+		Entry("HA: ipn=3 -> eligible", 3, nil, true, ""),
+	)
+
+	It("Execute deletes the reported primary pod", func() {
+		c := &fakeClient{instancesPerNode: 2, primary: "cluster-1"}
+		// The health monitor never reaches steady state here (its Run loop
+		// isn't started), so Execute times out on WaitForSteadyState — but the
+		// primary delete side-effect has already happened, which is what we
+		// assert. A short recovery keeps the test fast.
+		hm := monitor.NewHealthMonitor(c, journal.New(), time.Hour)
+		k := NewKillPrimaryPod(c, hm, 500*time.Millisecond)
+
+		_ = k.Execute(context.Background())
+		c.mu.Lock()
+		defer c.mu.Unlock()
+		Expect(c.deletedPods).To(ConsistOf("cluster-1"))
+	})
+
+	It("Execute fails without deleting when the primary is unknown", func() {
+		c := &fakeClient{instancesPerNode: 2, primaryErr: errors.New("no primary")}
+		k := NewKillPrimaryPod(c, nil, time.Second)
+
+		err := k.Execute(context.Background())
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("get primary instance"))
+		c.mu.Lock()
+		defer c.mu.Unlock()
+		Expect(c.deletedPods).To(BeEmpty())
+	})
+})
diff --git a/test/longhaul/operations/scale_test.go b/test/longhaul/operations/scale_test.go
index 3df032c05..a0d46381e 100644
--- a/test/longhaul/operations/scale_test.go
+++ b/test/longhaul/operations/scale_test.go
@@ -23,6 +23,10 @@ type fakeClient struct {
 	imageTag         string
 	scaleCalls       []int
 	upgradeCalls     []string
+	primary          string
+	primaryErr       error
+	deleteErr        error
+	deletedPods      []string
 }
 
 func (f *fakeClient) GetClusterHealth(_ context.Context) (monitor.ClusterHealth, error) {
@@ -51,6 +55,20 @@ func (f *fakeClient) UpgradeDocumentDB(_ context.Context, v string) error {
 	f.upgradeCalls = append(f.upgradeCalls, v)
 	return nil
 }
+func (f *fakeClient) GetPrimaryInstance(_ context.Context) (string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.primary, f.primaryErr
+}
+func (f *fakeClient) DeletePod(_ context.Context, name string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.deleteErr != nil {
+		return f.deleteErr
+	}
+	f.deletedPods = append(f.deletedPods, name)
+	return nil
+}
 
 var _ = Describe("ScaleUp", func() {
 	DescribeTable("clamps maxInstances to the CRD upper bound",

From 68bf8d5b9d7972a107e9e533a9a7702391f685cf Mon Sep 17 00:00:00 2001
From: Wenting Wu <wentingwu@microsoft.com>
Date: Fri, 10 Jul 2026 13:10:49 -0400
Subject: [PATCH 2/6] test: add shared NoOutagePolicy for no-write-failure
 operations
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Introduce journal.NoOutagePolicy (and the NoOutageWriteFailureCushion
constant) as the single budget for operations that keep the write path
(client -> gateway -> primary) up throughout, so they must not cause write
failures. Apply it to:

- kill-operator-pod (control-plane fault; was an inline budget of 5), and
- scale-up / scale-down, which only add/remove a standby replica — the
  primary is never touched, so their previous ad-hoc budgets (20 / 50) were
  too lenient and could mask a regression that disrupted writes.

The cushion is a small non-zero value (5) that absorbs unrelated background
noise without tolerating a real outage: at the default workload rate
(~50 writes/s) it is well under a second of stray errors. Centralizing it
lets the value be recalibrated against real long-haul runs in one place.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Signed-off-by: Wenting Wu <wentingwu@microsoft.com>
---
 test/longhaul/README.md                       |  9 +++++--
 test/longhaul/journal/policy.go               | 26 +++++++++++++++++++
 test/longhaul/journal/policy_test.go          | 10 +++++++
 test/longhaul/operations/kill_operator.go     | 11 +++-----
 .../longhaul/operations/kill_operator_test.go |  6 +++--
 test/longhaul/operations/scale.go             | 17 ++++++------
 test/longhaul/operations/scale_test.go        |  9 ++++---
 7 files changed, 64 insertions(+), 24 deletions(-)

diff --git a/test/longhaul/README.md b/test/longhaul/README.md
index c9fde0247..a4cef45a6 100644
--- a/test/longhaul/README.md
+++ b/test/longhaul/README.md
@@ -142,11 +142,16 @@ a global cooldown. Current operations:
 
 | Operation | Kind | Notes |
 |-----------|------|-------|
-| `scale-up` / `scale-down` | Topology | Adjusts `spec.instancesPerNode` within `[MIN, MAX]`. |
+| `scale-up` / `scale-down` | Topology | Adjusts `spec.instancesPerNode` within `[MIN, MAX]`. Only adds/removes a standby, so the primary write path is untouched (near-zero outage budget). |
 | `upgrade-documentdb` | Topology | In-place version upgrade; requires HA (`instancesPerNode>=2`). |
-| `kill-operator-pod` | Chaos | Deletes the operator pod; asserts the data plane keeps serving (small write-failure budget). |
+| `kill-operator-pod` | Chaos | Deletes the operator pod; asserts the data plane keeps serving (near-zero outage budget). |
 | `kill-primary-pod` | Chaos | Deletes the CNPG primary pod to exercise automatic failover; requires HA (`instancesPerNode>=2`). |
 
+Operations that keep the write path up throughout — the scale ops and
+`kill-operator-pod` — share the near-zero `journal.NoOutagePolicy` budget instead
+of ad-hoc per-op numbers, so a regression that unexpectedly disrupts writes
+during a "safe" operation trips the policy.
+
 ### RBAC for chaos operations
 
 Beyond the base RBAC the driver already needs, the chaos operations require the
diff --git a/test/longhaul/journal/policy.go b/test/longhaul/journal/policy.go
index 25d28e826..8ac106b0b 100644
--- a/test/longhaul/journal/policy.go
+++ b/test/longhaul/journal/policy.go
@@ -22,6 +22,32 @@ func DefaultOutagePolicy() OutagePolicy {
 	}
 }
 
+// NoOutageWriteFailureCushion is the small write-failure budget granted to
+// operations that are expected NOT to disrupt the data plane. It is not a
+// tolerance for real outages: at the default workload rate (~50 writes/s
+// aggregate, 5 writers x 100ms) this corresponds to well under a second of
+// stray errors, so any genuine primary disruption still trips the policy. The
+// non-zero value only absorbs unrelated background noise (a client reconnect,
+// service-endpoint churn) that would otherwise cause flaky false positives
+// against a strict 0. Centralized here so the single value can be recalibrated
+// against real long-haul runs.
+const NoOutageWriteFailureCushion int64 = 5
+
+// NoOutagePolicy is the outage budget for operations that keep the write path
+// (client -> gateway -> primary) up throughout and therefore must not cause
+// write failures. It is shared by every "no data-plane impact" operation:
+//   - control-plane faults, e.g. an operator pod restart, and
+//   - scaling that only adds or removes a standby replica (the primary, and
+//     thus the write path, is never touched).
+//
+// recovery bounds how long the cluster may take to return to steady state.
+func NoOutagePolicy(recovery time.Duration) OutagePolicy {
+	return OutagePolicy{
+		AllowedWriteFailures: NoOutageWriteFailureCushion,
+		MustRecoverWithin:    recovery,
+	}
+}
+
 // DisruptionWindow represents an active or closed disruption period.
 type DisruptionWindow struct {
 	// OperationName identifies which operation opened this window.
diff --git a/test/longhaul/journal/policy_test.go b/test/longhaul/journal/policy_test.go
index 4fe26b1a4..cac082d45 100644
--- a/test/longhaul/journal/policy_test.go
+++ b/test/longhaul/journal/policy_test.go
@@ -81,4 +81,14 @@ var _ = Describe("DisruptionWindow", func() {
 		Expect(p.MustRecoverWithin).NotTo(BeZero())
 		Expect(p.AllowedWriteFailures).NotTo(BeZero())
 	})
+
+	It("NoOutagePolicy grants the near-zero cushion and echoes recovery", func() {
+		p := NoOutagePolicy(3 * time.Minute)
+		Expect(p.AllowedWriteFailures).To(Equal(NoOutageWriteFailureCushion))
+		Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute))
+	})
+
+	It("NoOutagePolicy is far tighter than DefaultOutagePolicy", func() {
+		Expect(NoOutageWriteFailureCushion).To(BeNumerically("<", DefaultOutagePolicy().AllowedWriteFailures))
+	})
 })
diff --git a/test/longhaul/operations/kill_operator.go b/test/longhaul/operations/kill_operator.go
index 306503f63..6018aab98 100644
--- a/test/longhaul/operations/kill_operator.go
+++ b/test/longhaul/operations/kill_operator.go
@@ -89,15 +89,10 @@ func (k *KillOperatorPod) Execute(ctx context.Context) error {
 	return k.waitForDeploymentAvailable(recoveryCtx)
 }
 
-// OutagePolicy tolerates only a small number of write failures: an operator
-// restart must not take down the data plane. A non-zero budget absorbs
-// coincidental blips (e.g. a scrape or client reconnect) without flagging a
-// false policy violation.
+// OutagePolicy: an operator restart is a control-plane fault that must not take
+// down the data plane, so it shares the near-zero NoOutagePolicy budget.
 func (k *KillOperatorPod) OutagePolicy() journal.OutagePolicy {
-	return journal.OutagePolicy{
-		AllowedWriteFailures: 5,
-		MustRecoverWithin:    k.recovery,
-	}
+	return journal.NoOutagePolicy(k.recovery)
 }
 
 func (k *KillOperatorPod) getDeployment(ctx context.Context) (*appsv1.Deployment, error) {
diff --git a/test/longhaul/operations/kill_operator_test.go b/test/longhaul/operations/kill_operator_test.go
index f05b04b9b..e690702ec 100644
--- a/test/longhaul/operations/kill_operator_test.go
+++ b/test/longhaul/operations/kill_operator_test.go
@@ -10,6 +10,8 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
+	"github.com/documentdb/documentdb-operator/test/longhaul/journal"
+
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -56,10 +58,10 @@ var _ = Describe("KillOperatorPod", func() {
 		Expect(k.Weight()).To(Equal(2))
 	})
 
-	It("OutagePolicy tolerates only a small write-failure budget", func() {
+	It("OutagePolicy uses the near-zero NoOutagePolicy budget", func() {
 		k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, 2*time.Minute)
 		p := k.OutagePolicy()
-		Expect(p.AllowedWriteFailures).To(Equal(int64(5)))
+		Expect(p.AllowedWriteFailures).To(Equal(journal.NoOutageWriteFailureCushion))
 		Expect(p.MustRecoverWithin).To(Equal(2 * time.Minute))
 	})
 
diff --git a/test/longhaul/operations/scale.go b/test/longhaul/operations/scale.go
index 9a1672959..f002fdc61 100644
--- a/test/longhaul/operations/scale.go
+++ b/test/longhaul/operations/scale.go
@@ -77,6 +77,9 @@ type ScaleUp struct{ scaleOp }
 
 // NewScaleUp creates a ScaleUp operation. maxInstances is clamped to the
 // CRD upper bound (3) to avoid admission rejections.
+//
+// Scaling up only adds a standby replica (the primary and thus the write path
+// is untouched), so it uses the near-zero NoOutagePolicy budget.
 func NewScaleUp(client monitor.ClusterClient, health *monitor.HealthMonitor, maxInstances int, recovery time.Duration) *ScaleUp {
 	if maxInstances > 3 {
 		maxInstances = 3
@@ -90,10 +93,7 @@ func NewScaleUp(client monitor.ClusterClient, health *monitor.HealthMonitor, max
 		bound:     maxInstances,
 		boundKind: "max",
 		recovery:  recovery,
-		policy: journal.OutagePolicy{
-			AllowedWriteFailures: 20,
-			MustRecoverWithin:    recovery,
-		},
+		policy:    journal.NoOutagePolicy(recovery),
 	}}
 }
 
@@ -105,6 +105,10 @@ type ScaleDown struct{ scaleOp }
 
 // NewScaleDown creates a ScaleDown operation. minInstances is clamped to the
 // CRD lower bound (1) to avoid admission rejections.
+//
+// Scaling down removes the highest-ordinal standby (CNPG never removes the
+// primary), so the write path stays up and it uses the same near-zero
+// NoOutagePolicy budget as scale-up.
 func NewScaleDown(client monitor.ClusterClient, health *monitor.HealthMonitor, minInstances int, recovery time.Duration) *ScaleDown {
 	if minInstances < 1 {
 		minInstances = 1
@@ -118,10 +122,7 @@ func NewScaleDown(client monitor.ClusterClient, health *monitor.HealthMonitor, m
 		bound:     minInstances,
 		boundKind: "min",
 		recovery:  recovery,
-		policy: journal.OutagePolicy{
-			AllowedWriteFailures: 50,
-			MustRecoverWithin:    recovery,
-		},
+		policy:    journal.NoOutagePolicy(recovery),
 	}}
 }
 
diff --git a/test/longhaul/operations/scale_test.go b/test/longhaul/operations/scale_test.go
index a0d46381e..222d215a5 100644
--- a/test/longhaul/operations/scale_test.go
+++ b/test/longhaul/operations/scale_test.go
@@ -12,6 +12,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
+	"github.com/documentdb/documentdb-operator/test/longhaul/journal"
 	"github.com/documentdb/documentdb-operator/test/longhaul/monitor"
 )
 
@@ -105,10 +106,10 @@ var _ = Describe("ScaleUp", func() {
 		Entry("blocked: ipn read error", 0, errors.New("apiserver down"), 3, false, "cannot get instancesPerNode"),
 	)
 
-	It("OutagePolicy uses tighter budgets and echoes MustRecoverWithin", func() {
+	It("OutagePolicy uses the near-zero NoOutagePolicy budget and echoes MustRecoverWithin", func() {
 		s := NewScaleUp(&fakeClient{}, nil, 3, 5*time.Minute)
 		p := s.OutagePolicy()
-		Expect(p.AllowedWriteFailures).To(Equal(int64(20)))
+		Expect(p.AllowedWriteFailures).To(Equal(journal.NoOutageWriteFailureCushion))
 		Expect(p.MustRecoverWithin).To(Equal(5 * time.Minute))
 	})
 })
@@ -148,9 +149,9 @@ var _ = Describe("ScaleDown", func() {
 		Entry("blocked: ipn read error", 0, errors.New("apiserver down"), 1, false, "cannot get instancesPerNode"),
 	)
 
-	It("OutagePolicy is more lenient than scale-up", func() {
+	It("OutagePolicy shares the near-zero NoOutagePolicy budget with scale-up", func() {
 		s := NewScaleDown(&fakeClient{}, nil, 1, 5*time.Minute)
 		p := s.OutagePolicy()
-		Expect(p.AllowedWriteFailures).To(Equal(int64(50)))
+		Expect(p.AllowedWriteFailures).To(Equal(journal.NoOutageWriteFailureCushion))
 	})
 })

From 4f0d632915b5ebee948485c6b7aadcc7c707d3ad Mon Sep 17 00:00:00 2001
From: Wenting Wu <wentingwu@microsoft.com>
Date: Fri, 10 Jul 2026 13:19:58 -0400
Subject: [PATCH 3/6] refactor(longhaul): make outage budgets duration-based

Express OutagePolicy as a wall-clock write-outage duration
(MaxWriteOutage) instead of a raw write-failure count. The journal
converts the observed failure count into an estimated outage using the
workload's aggregate write rate, so budgets no longer scale with
LONGHAUL_NUM_WRITERS.

- policy: OutagePolicy.AllowedWriteFailures -> MaxWriteOutage;
  DisruptionWindow gains WritesPerSecond + EstimatedWriteOutage();
  NoOutageWriteFailureCushion -> NoOutageWriteOutageCushion (300ms).
- journal: New() defaults to DefaultWritesPerSecond; SetWriteRate lets
  main.go supply the real rate; OpenDisruptionWindow stamps the rate.
- workload: expose AggregateWriteRate(numWriters).
- ops: kill-primary 30s, upgrade 45s (was 50/200 failures).
- report: surface Est. Write Outage column.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Signed-off-by: Wenting Wu <wentingwu@microsoft.com>
---
 test/longhaul/README.md                       |  8 +++
 test/longhaul/cmd/longhaul/main.go            |  1 +
 test/longhaul/journal/journal.go              | 35 +++++++++--
 test/longhaul/journal/journal_test.go         | 10 ++--
 test/longhaul/journal/policy.go               | 59 +++++++++++++------
 test/longhaul/journal/policy_test.go          | 59 +++++++++++--------
 .../longhaul/operations/kill_operator_test.go |  2 +-
 test/longhaul/operations/kill_primary.go      | 11 ++--
 test/longhaul/operations/kill_primary_test.go |  4 +-
 test/longhaul/operations/scale_test.go        |  4 +-
 test/longhaul/operations/upgrade.go           | 11 ++--
 test/longhaul/operations/upgrade_test.go      |  4 +-
 test/longhaul/report/report.go                |  9 +--
 test/longhaul/report/report_test.go           | 11 ++--
 test/longhaul/workload/writer.go              | 12 ++++
 15 files changed, 166 insertions(+), 74 deletions(-)

diff --git a/test/longhaul/README.md b/test/longhaul/README.md
index a4cef45a6..452fe3785 100644
--- a/test/longhaul/README.md
+++ b/test/longhaul/README.md
@@ -152,6 +152,14 @@ Operations that keep the write path up throughout — the scale ops and
 of ad-hoc per-op numbers, so a regression that unexpectedly disrupts writes
 during a "safe" operation trips the policy.
 
+Outage budgets are expressed as **wall-clock write-outage durations**
+(`OutagePolicy.MaxWriteOutage`), not raw write-failure counts. The journal
+converts the observed failure count into an estimated outage using the workload's
+aggregate write rate (`workload.AggregateWriteRate(NumWriters)`), so the budgets
+are independent of `LONGHAUL_NUM_WRITERS`: `NoOutagePolicy` ≈ 300ms (noise
+cushion), `kill-primary-pod` = 30s (single automatic failover), and
+`upgrade-documentdb` = 45s (primary switchover during the rolling upgrade).
+
 ### RBAC for chaos operations
 
 Beyond the base RBAC the driver already needs, the chaos operations require the
diff --git a/test/longhaul/cmd/longhaul/main.go b/test/longhaul/cmd/longhaul/main.go
index 5195d49d7..a93870dbd 100644
--- a/test/longhaul/cmd/longhaul/main.go
+++ b/test/longhaul/cmd/longhaul/main.go
@@ -57,6 +57,7 @@ func run(cfg config.Config) int {
 
 	// Initialize components.
 	j := journal.New()
+	j.SetWriteRate(workload.AggregateWriteRate(cfg.NumWriters))
 	metrics := workload.NewMetrics()
 
 	// Connect to DocumentDB.
diff --git a/test/longhaul/journal/journal.go b/test/longhaul/journal/journal.go
index bfddebc76..2ad071682 100644
--- a/test/longhaul/journal/journal.go
+++ b/test/longhaul/journal/journal.go
@@ -52,13 +52,39 @@ type Journal struct {
 
 	// All closed disruption windows.
 	closedWindows []DisruptionWindow
+
+	// writesPerSecond is the workload's aggregate write rate, stamped onto each
+	// disruption window so ExceededPolicy can convert write-failure counts into
+	// an estimated outage duration. Defaults to DefaultWritesPerSecond; override
+	// with SetWriteRate once the real writer count is known.
+	writesPerSecond float64
 }
 
+// DefaultWritesPerSecond is the assumed aggregate write rate used until
+// SetWriteRate is called. It matches the default workload (5 writers at one
+// write per 100ms = 50 writes/s) so tests and un-configured journals still
+// evaluate write-outage budgets sensibly.
+const DefaultWritesPerSecond = 50.0
+
 // New creates a new empty Journal.
 func New() *Journal {
 	return &Journal{
-		events: make([]Event, 0, 256),
+		events:          make([]Event, 0, 256),
+		writesPerSecond: DefaultWritesPerSecond,
+	}
+}
+
+// SetWriteRate records the workload's aggregate write rate (writes/second across
+// all writers) so disruption windows can translate write-failure counts into an
+// estimated outage duration. Non-positive values are ignored, preserving the
+// default. Safe for concurrent use.
+func (j *Journal) SetWriteRate(writesPerSecond float64) {
+	if writesPerSecond <= 0 {
+		return
 	}
+	j.mu.Lock()
+	defer j.mu.Unlock()
+	j.writesPerSecond = writesPerSecond
 }
 
 // Record appends a new event to the journal. Safe for concurrent use.
@@ -112,9 +138,10 @@ func (j *Journal) OpenDisruptionWindow(operationName string, policy OutagePolicy
 	}
 
 	j.activeWindow = &DisruptionWindow{
-		OperationName: operationName,
-		StartTime:     time.Now(),
-		Policy:        policy,
+		OperationName:   operationName,
+		StartTime:       time.Now(),
+		Policy:          policy,
+		WritesPerSecond: j.writesPerSecond,
 	}
 
 	j.events = append(j.events, Event{
diff --git a/test/longhaul/journal/journal_test.go b/test/longhaul/journal/journal_test.go
index 02e2b3c6c..43c159db6 100644
--- a/test/longhaul/journal/journal_test.go
+++ b/test/longhaul/journal/journal_test.go
@@ -43,7 +43,7 @@ var _ = Describe("Journal", func() {
 	Describe("DisruptionWindow lifecycle", func() {
 		It("opens, records failures, and closes correctly", func() {
 			j := New()
-			policy := OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 10}
+			policy := OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}
 
 			Expect(j.ActiveWindow()).To(BeNil())
 
@@ -89,14 +89,14 @@ var _ = Describe("Journal", func() {
 
 		It("returns false on a closed window within budget", func() {
 			j := New()
-			j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 10})
+			j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second})
 			j.CloseDisruptionWindow()
 			Expect(j.HasPolicyViolation()).To(BeFalse())
 		})
 
-		It("returns true on a closed window over write-failure budget", func() {
+		It("returns true on a closed window over write-outage budget", func() {
 			j := New()
-			j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 1})
+			j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: 10 * time.Millisecond})
 			j.RecordWriteFailure()
 			j.RecordWriteFailure()
 			j.CloseDisruptionWindow()
@@ -105,7 +105,7 @@ var _ = Describe("Journal", func() {
 
 		It("returns true on an active window over time budget", func() {
 			j := New()
-			j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Nanosecond, AllowedWriteFailures: 10})
+			j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Nanosecond, MaxWriteOutage: time.Second})
 			time.Sleep(1 * time.Millisecond)
 			Expect(j.HasPolicyViolation()).To(BeTrue())
 		})
diff --git a/test/longhaul/journal/policy.go b/test/longhaul/journal/policy.go
index 8ac106b0b..738e86438 100644
--- a/test/longhaul/journal/policy.go
+++ b/test/longhaul/journal/policy.go
@@ -7,8 +7,13 @@ import "time"
 
 // OutagePolicy defines acceptable disruption bounds for an operation.
 type OutagePolicy struct {
-	// AllowedWriteFailures is the maximum number of write failures during the window.
-	AllowedWriteFailures int64
+	// MaxWriteOutage bounds how long the write path (client -> gateway ->
+	// primary) may be unavailable during the window. It is evaluated from the
+	// observed write-failure count normalized by the workload's aggregate write
+	// rate (see DisruptionWindow.EstimatedWriteOutage), so the budget is
+	// expressed in wall-clock outage time and is independent of how many writer
+	// goroutines (LONGHAUL_NUM_WRITERS) are configured.
+	MaxWriteOutage time.Duration
 
 	// MustRecoverWithin is the maximum time from operation start to full recovery.
 	MustRecoverWithin time.Duration
@@ -17,25 +22,24 @@ type OutagePolicy struct {
 // DefaultOutagePolicy returns a conservative policy suitable for most operations.
 func DefaultOutagePolicy() OutagePolicy {
 	return OutagePolicy{
-		AllowedWriteFailures: 50,
-		MustRecoverWithin:    5 * time.Minute,
+		MaxWriteOutage:    5 * time.Second,
+		MustRecoverWithin: 5 * time.Minute,
 	}
 }
 
-// NoOutageWriteFailureCushion is the small write-failure budget granted to
+// NoOutageWriteOutageCushion is the tiny write-outage budget granted to
 // operations that are expected NOT to disrupt the data plane. It is not a
-// tolerance for real outages: at the default workload rate (~50 writes/s
-// aggregate, 5 writers x 100ms) this corresponds to well under a second of
-// stray errors, so any genuine primary disruption still trips the policy. The
-// non-zero value only absorbs unrelated background noise (a client reconnect,
-// service-endpoint churn) that would otherwise cause flaky false positives
-// against a strict 0. Centralized here so the single value can be recalibrated
-// against real long-haul runs.
-const NoOutageWriteFailureCushion int64 = 5
+// tolerance for real outages: one fully-failed write tick (every configured
+// writer failing once) maps to exactly one writeInterval of estimated outage
+// (~100ms) regardless of writer count, so this ~3-tick cushion absorbs unrelated
+// background noise (a client reconnect, service-endpoint churn) without
+// tolerating a genuine primary outage. Centralized so it can be recalibrated
+// against real long-haul runs in one place.
+const NoOutageWriteOutageCushion = 300 * time.Millisecond
 
 // NoOutagePolicy is the outage budget for operations that keep the write path
-// (client -> gateway -> primary) up throughout and therefore must not cause
-// write failures. It is shared by every "no data-plane impact" operation:
+// up throughout and therefore must not cause a write outage. It is shared by
+// every "no data-plane impact" operation:
 //   - control-plane faults, e.g. an operator pod restart, and
 //   - scaling that only adds or removes a standby replica (the primary, and
 //     thus the write path, is never touched).
@@ -43,8 +47,8 @@ const NoOutageWriteFailureCushion int64 = 5
 // recovery bounds how long the cluster may take to return to steady state.
 func NoOutagePolicy(recovery time.Duration) OutagePolicy {
 	return OutagePolicy{
-		AllowedWriteFailures: NoOutageWriteFailureCushion,
-		MustRecoverWithin:    recovery,
+		MaxWriteOutage:    NoOutageWriteOutageCushion,
+		MustRecoverWithin: recovery,
 	}
 }
 
@@ -64,6 +68,25 @@ type DisruptionWindow struct {
 
 	// WriteFailures counts failures observed during this window.
 	WriteFailures int64
+
+	// WritesPerSecond is the workload's aggregate write rate at the time the
+	// window opened. It is used to convert the raw WriteFailures count into an
+	// estimated write-outage duration (see EstimatedWriteOutage). A real outage
+	// makes every writer fail on every tick, so failures accrue at the full
+	// aggregate rate and count/rate recovers the wall-clock outage duration
+	// regardless of writer count. Zero disables the write-outage check.
+	WritesPerSecond float64
+}
+
+// EstimatedWriteOutage converts the observed write-failure count into an
+// approximate duration for which the write path was unavailable, using the
+// aggregate write rate captured when the window opened. Returns 0 when the rate
+// is unknown (<= 0), which disables the write-outage portion of the policy.
+func (w *DisruptionWindow) EstimatedWriteOutage() time.Duration {
+	if w.WritesPerSecond <= 0 {
+		return 0
+	}
+	return time.Duration(float64(w.WriteFailures) / w.WritesPerSecond * float64(time.Second))
 }
 
 // IsActive returns true if the disruption window has not been closed.
@@ -85,7 +108,7 @@ func (w *DisruptionWindow) ExceededPolicy() bool {
 	if w.Duration() > w.Policy.MustRecoverWithin {
 		return true
 	}
-	if w.WriteFailures > w.Policy.AllowedWriteFailures {
+	if w.EstimatedWriteOutage() > w.Policy.MaxWriteOutage {
 		return true
 	}
 	return false
diff --git a/test/longhaul/journal/policy_test.go b/test/longhaul/journal/policy_test.go
index cac082d45..dec577337 100644
--- a/test/longhaul/journal/policy_test.go
+++ b/test/longhaul/journal/policy_test.go
@@ -43,52 +43,65 @@ var _ = Describe("DisruptionWindow", func() {
 		},
 		Entry("within all budgets",
 			DisruptionWindow{
-				StartTime:     time.Now().Add(-10 * time.Second),
-				EndTime:       time.Now(),
-				WriteFailures: 5,
-				Policy:        OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50},
+				StartTime:       time.Now().Add(-10 * time.Second),
+				EndTime:         time.Now(),
+				WriteFailures:   5, // 5/50 = 0.1s < 1s
+				WritesPerSecond: 50,
+				Policy:          OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second},
 			}, false),
 		Entry("exceeds MustRecoverWithin",
 			DisruptionWindow{
-				StartTime:     time.Now().Add(-2 * time.Minute),
-				EndTime:       time.Now(),
-				WriteFailures: 1,
-				Policy:        OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50},
+				StartTime:       time.Now().Add(-2 * time.Minute),
+				EndTime:         time.Now(),
+				WriteFailures:   1,
+				WritesPerSecond: 50,
+				Policy:          OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second},
 			}, true),
-		Entry("exceeds AllowedWriteFailures",
+		Entry("exceeds MaxWriteOutage",
 			DisruptionWindow{
-				StartTime:     time.Now().Add(-10 * time.Second),
-				EndTime:       time.Now(),
-				WriteFailures: 100,
-				Policy:        OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50},
+				StartTime:       time.Now().Add(-10 * time.Second),
+				EndTime:         time.Now(),
+				WriteFailures:   100, // 100/50 = 2s > 1s
+				WritesPerSecond: 50,
+				Policy:          OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second},
 			}, true),
-		Entry("boundary: equal to write-failure budget is allowed",
+		Entry("boundary: estimated outage equal to budget is allowed",
 			DisruptionWindow{
-				StartTime:     time.Now().Add(-10 * time.Second),
-				EndTime:       time.Now(),
-				WriteFailures: 50,
-				Policy:        OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50},
+				StartTime:       time.Now().Add(-10 * time.Second),
+				EndTime:         time.Now(),
+				WriteFailures:   50, // 50/50 = exactly 1s
+				WritesPerSecond: 50,
+				Policy:          OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second},
+			}, false),
+		Entry("unknown write rate disables the write-outage check",
+			DisruptionWindow{
+				StartTime:       time.Now().Add(-10 * time.Second),
+				EndTime:         time.Now(),
+				WriteFailures:   100000,
+				WritesPerSecond: 0,
+				Policy:          OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second},
 			}, false),
 		Entry("active window also evaluated against MustRecoverWithin",
 			DisruptionWindow{
-				StartTime: time.Now().Add(-2 * time.Minute),
-				Policy:    OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50},
+				StartTime:       time.Now().Add(-2 * time.Minute),
+				WritesPerSecond: 50,
+				Policy:          OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second},
 			}, true),
 	)
 
 	It("DefaultOutagePolicy returns no zero-valued field", func() {
 		p := DefaultOutagePolicy()
 		Expect(p.MustRecoverWithin).NotTo(BeZero())
-		Expect(p.AllowedWriteFailures).NotTo(BeZero())
+		Expect(p.MaxWriteOutage).NotTo(BeZero())
 	})
 
 	It("NoOutagePolicy grants the near-zero cushion and echoes recovery", func() {
 		p := NoOutagePolicy(3 * time.Minute)
-		Expect(p.AllowedWriteFailures).To(Equal(NoOutageWriteFailureCushion))
+		Expect(p.MaxWriteOutage).To(Equal(NoOutageWriteOutageCushion))
 		Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute))
 	})
 
 	It("NoOutagePolicy is far tighter than DefaultOutagePolicy", func() {
-		Expect(NoOutageWriteFailureCushion).To(BeNumerically("<", DefaultOutagePolicy().AllowedWriteFailures))
+		Expect(NoOutageWriteOutageCushion).To(BeNumerically("<", DefaultOutagePolicy().MaxWriteOutage))
 	})
 })
diff --git a/test/longhaul/operations/kill_operator_test.go b/test/longhaul/operations/kill_operator_test.go
index e690702ec..d409b5ab4 100644
--- a/test/longhaul/operations/kill_operator_test.go
+++ b/test/longhaul/operations/kill_operator_test.go
@@ -61,7 +61,7 @@ var _ = Describe("KillOperatorPod", func() {
 	It("OutagePolicy uses the near-zero NoOutagePolicy budget", func() {
 		k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, 2*time.Minute)
 		p := k.OutagePolicy()
-		Expect(p.AllowedWriteFailures).To(Equal(journal.NoOutageWriteFailureCushion))
+		Expect(p.MaxWriteOutage).To(Equal(journal.NoOutageWriteOutageCushion))
 		Expect(p.MustRecoverWithin).To(Equal(2 * time.Minute))
 	})
 
diff --git a/test/longhaul/operations/kill_primary.go b/test/longhaul/operations/kill_primary.go
index aa2199db8..15084085d 100644
--- a/test/longhaul/operations/kill_primary.go
+++ b/test/longhaul/operations/kill_primary.go
@@ -66,11 +66,14 @@ func (k *KillPrimaryPod) Execute(ctx context.Context) error {
 	return k.healthMon.WaitForSteadyState(recoveryCtx)
 }
 
-// OutagePolicy allows a moderate write-failure budget: failover briefly
-// interrupts writes while a standby is promoted, similar to scale-down.
+// OutagePolicy bounds the write outage of an automatic failover. Killing the
+// primary interrupts writes until CNPG detects the loss and promotes a standby;
+// a healthy single failover should restore the write path well within this
+// budget. Expressed as wall-clock outage time, so it is independent of the
+// configured writer count.
 func (k *KillPrimaryPod) OutagePolicy() journal.OutagePolicy {
 	return journal.OutagePolicy{
-		AllowedWriteFailures: 50,
-		MustRecoverWithin:    k.recovery,
+		MaxWriteOutage:    30 * time.Second,
+		MustRecoverWithin: k.recovery,
 	}
 }
diff --git a/test/longhaul/operations/kill_primary_test.go b/test/longhaul/operations/kill_primary_test.go
index aab917591..f35d49b39 100644
--- a/test/longhaul/operations/kill_primary_test.go
+++ b/test/longhaul/operations/kill_primary_test.go
@@ -22,10 +22,10 @@ var _ = Describe("KillPrimaryPod", func() {
 		Expect(k.Weight()).To(Equal(2))
 	})
 
-	It("OutagePolicy allows a moderate failover write-failure budget", func() {
+	It("OutagePolicy bounds the failover write outage in wall-clock time", func() {
 		k := NewKillPrimaryPod(&fakeClient{}, nil, 3*time.Minute)
 		p := k.OutagePolicy()
-		Expect(p.AllowedWriteFailures).To(Equal(int64(50)))
+		Expect(p.MaxWriteOutage).To(Equal(30 * time.Second))
 		Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute))
 	})
 
diff --git a/test/longhaul/operations/scale_test.go b/test/longhaul/operations/scale_test.go
index 222d215a5..8c9fc953b 100644
--- a/test/longhaul/operations/scale_test.go
+++ b/test/longhaul/operations/scale_test.go
@@ -109,7 +109,7 @@ var _ = Describe("ScaleUp", func() {
 	It("OutagePolicy uses the near-zero NoOutagePolicy budget and echoes MustRecoverWithin", func() {
 		s := NewScaleUp(&fakeClient{}, nil, 3, 5*time.Minute)
 		p := s.OutagePolicy()
-		Expect(p.AllowedWriteFailures).To(Equal(journal.NoOutageWriteFailureCushion))
+		Expect(p.MaxWriteOutage).To(Equal(journal.NoOutageWriteOutageCushion))
 		Expect(p.MustRecoverWithin).To(Equal(5 * time.Minute))
 	})
 })
@@ -152,6 +152,6 @@ var _ = Describe("ScaleDown", func() {
 	It("OutagePolicy shares the near-zero NoOutagePolicy budget with scale-up", func() {
 		s := NewScaleDown(&fakeClient{}, nil, 1, 5*time.Minute)
 		p := s.OutagePolicy()
-		Expect(p.AllowedWriteFailures).To(Equal(journal.NoOutageWriteFailureCushion))
+		Expect(p.MaxWriteOutage).To(Equal(journal.NoOutageWriteOutageCushion))
 	})
 })
diff --git a/test/longhaul/operations/upgrade.go b/test/longhaul/operations/upgrade.go
index f9a586b6e..6f823c47b 100644
--- a/test/longhaul/operations/upgrade.go
+++ b/test/longhaul/operations/upgrade.go
@@ -170,11 +170,14 @@ func (u *UpgradeDocumentDB) readDesiredVersion(ctx context.Context) (string, err
 	return cm.Data[VersionConfigMapKey], nil
 }
 
-// OutagePolicy allows for a longer disruption window during an upgrade
-// because rolling restarts touch every pod sequentially.
+// OutagePolicy bounds the write outage of a rolling upgrade. Standby restarts
+// do not block writes; the write path is only interrupted during the primary
+// switchover, so the outage is comparable to a failover with some extra slack
+// for the graceful switchover coordination. Expressed as wall-clock outage
+// time, independent of the configured writer count.
 func (u *UpgradeDocumentDB) OutagePolicy() journal.OutagePolicy {
 	return journal.OutagePolicy{
-		AllowedWriteFailures: 200,
-		MustRecoverWithin:    u.recovery,
+		MaxWriteOutage:    45 * time.Second,
+		MustRecoverWithin: u.recovery,
 	}
 }
diff --git a/test/longhaul/operations/upgrade_test.go b/test/longhaul/operations/upgrade_test.go
index f442e5b58..9b9a7a59c 100644
--- a/test/longhaul/operations/upgrade_test.go
+++ b/test/longhaul/operations/upgrade_test.go
@@ -22,10 +22,10 @@ var _ = Describe("UpgradeDocumentDB", func() {
 		Expect(u.Weight()).To(Equal(1))
 	})
 
-	It("OutagePolicy gives upgrades a more lenient failure budget", func() {
+	It("OutagePolicy gives upgrades a more lenient write-outage budget", func() {
 		u := NewUpgradeDocumentDB(&fakeClient{}, fake.NewSimpleClientset(), nil, nil, "ns", 10*time.Minute)
 		p := u.OutagePolicy()
-		Expect(p.AllowedWriteFailures).To(Equal(int64(200)))
+		Expect(p.MaxWriteOutage).To(Equal(45 * time.Second))
 		Expect(p.MustRecoverWithin).To(Equal(10 * time.Minute))
 	})
 
diff --git a/test/longhaul/report/report.go b/test/longhaul/report/report.go
index e7316a7f7..07576ef58 100644
--- a/test/longhaul/report/report.go
+++ b/test/longhaul/report/report.go
@@ -93,15 +93,16 @@ func GenerateMarkdown(s Summary) string {
 	// Disruption Windows
 	if len(s.Windows) > 0 {
 		b.WriteString("## Disruption Windows\n\n")
-		b.WriteString("| Operation | Duration | Write Failures | Policy Exceeded |\n")
-		b.WriteString("|-----------|----------|----------------|------------------|\n")
+		b.WriteString("| Operation | Duration | Write Failures | Est. Write Outage | Policy Exceeded |\n")
+		b.WriteString("|-----------|----------|----------------|-------------------|------------------|\n")
 		for _, w := range s.Windows {
 			exceeded := "No"
 			if w.ExceededPolicy() {
 				exceeded = "**YES**"
 			}
-			fmt.Fprintf(&b, "| %s | %s | %d | %s |\n",
-				w.OperationName, w.Duration().Round(time.Second), w.WriteFailures, exceeded)
+			fmt.Fprintf(&b, "| %s | %s | %d | %s | %s |\n",
+				w.OperationName, w.Duration().Round(time.Second), w.WriteFailures,
+				w.EstimatedWriteOutage().Round(time.Millisecond), exceeded)
 		}
 		b.WriteString("\n")
 	}
diff --git a/test/longhaul/report/report_test.go b/test/longhaul/report/report_test.go
index f8b77297e..33fc844b5 100644
--- a/test/longhaul/report/report_test.go
+++ b/test/longhaul/report/report_test.go
@@ -74,11 +74,12 @@ var _ = Describe("GenerateMarkdown", func() {
 			md := GenerateMarkdown(Summary{
 				Result: ResultPass,
 				Windows: []journal.DisruptionWindow{{
-					OperationName: "scale-up",
-					StartTime:     now.Add(-30 * time.Second),
-					EndTime:       now,
-					WriteFailures: 3,
-					Policy:        journal.OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50},
+					OperationName:   "scale-up",
+					StartTime:       now.Add(-30 * time.Second),
+					EndTime:         now,
+					WriteFailures:   3,
+					WritesPerSecond: 50,
+					Policy:          journal.OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second},
 				}},
 			})
 			Expect(md).To(ContainSubstring("Disruption Windows"))
diff --git a/test/longhaul/workload/writer.go b/test/longhaul/workload/writer.go
index c2fdcc3e2..dddfd0e54 100644
--- a/test/longhaul/workload/writer.go
+++ b/test/longhaul/workload/writer.go
@@ -28,6 +28,18 @@ const (
 	writeInterval = 100 * time.Millisecond
 )
 
+// AggregateWriteRate returns the workload's aggregate write rate in writes per
+// second across all writer goroutines, given the configured writer count. It is
+// the reciprocal of the per-writer writeInterval scaled by numWriters, and is
+// used to convert observed write-failure counts into an estimated outage
+// duration (see journal.DisruptionWindow). Returns 0 for a non-positive count.
+func AggregateWriteRate(numWriters int) float64 {
+	if numWriters <= 0 {
+		return 0
+	}
+	return float64(numWriters) / writeInterval.Seconds()
+}
+
 // WriteDocument is the schema for data-plane durability tracking.
 type WriteDocument struct {
 	WriterID  string    `bson:"writer_id"`

From ae8a122cf2334f7538972a9390e6376ecb1b33c2 Mon Sep 17 00:00:00 2001
From: Wenting Wu <wentingwu@microsoft.com>
Date: Fri, 10 Jul 2026 13:29:21 -0400
Subject: [PATCH 4/6] docs(longhaul): clarify OutagePolicy field roles

Document that MaxWriteOutage (data plane) and MustRecoverWithin
(control plane) assert on orthogonal subsystems and fail independently,
and that MustRecoverWithin is the only path that fails the run when the
cluster never converges (op errors are logged, not scored).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Signed-off-by: Wenting Wu <wentingwu@microsoft.com>
---
 test/longhaul/journal/policy.go | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/test/longhaul/journal/policy.go b/test/longhaul/journal/policy.go
index 738e86438..94c63a5b0 100644
--- a/test/longhaul/journal/policy.go
+++ b/test/longhaul/journal/policy.go
@@ -5,7 +5,13 @@ package journal
 
 import "time"
 
-// OutagePolicy defines acceptable disruption bounds for an operation.
+// OutagePolicy defines acceptable disruption bounds for an operation. Its two
+// fields assert on different subsystems and fail independently (ExceededPolicy
+// trips if either is exceeded): MaxWriteOutage bounds the client-visible write
+// interruption (data plane), while MustRecoverWithin bounds full cluster
+// recovery (control plane). Each can be violated while the other is fine — e.g.
+// a promoted standby that never rejoins keeps writes flowing yet leaves the
+// cluster degraded, and only MustRecoverWithin catches it.
 type OutagePolicy struct {
 	// MaxWriteOutage bounds how long the write path (client -> gateway ->
 	// primary) may be unavailable during the window. It is evaluated from the
@@ -15,7 +21,10 @@ type OutagePolicy struct {
 	// goroutines (LONGHAUL_NUM_WRITERS) are configured.
 	MaxWriteOutage time.Duration
 
-	// MustRecoverWithin is the maximum time from operation start to full recovery.
+	// MustRecoverWithin is the maximum time from operation start to full cluster
+	// recovery (steady state). Because a failed op is only logged, not counted
+	// toward the run verdict, this is the sole mechanism that turns a cluster
+	// that never converges back into a FAIL.
 	MustRecoverWithin time.Duration
 }
 

From 7e3205710182d47b4ce5dd01a802ca33a5828f2a Mon Sep 17 00:00:00 2001
From: Wenting Wu <wentingwu@microsoft.com>
Date: Fri, 10 Jul 2026 13:39:42 -0400
Subject: [PATCH 5/6] docs(longhaul): correct OutagePolicy field framing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

MustRecoverWithin bounds the managed database cluster's return to full
topology (all pods Ready, CR Ready) — not the operator/control plane.
Reframe as write-availability vs. full-topology recovery to avoid
implying scale/kill-primary recovery involves the operator.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Signed-off-by: Wenting Wu <wentingwu@microsoft.com>
---
 test/longhaul/journal/policy.go | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/test/longhaul/journal/policy.go b/test/longhaul/journal/policy.go
index 94c63a5b0..1d41ac581 100644
--- a/test/longhaul/journal/policy.go
+++ b/test/longhaul/journal/policy.go
@@ -6,12 +6,13 @@ package journal
 import "time"
 
 // OutagePolicy defines acceptable disruption bounds for an operation. Its two
-// fields assert on different subsystems and fail independently (ExceededPolicy
-// trips if either is exceeded): MaxWriteOutage bounds the client-visible write
-// interruption (data plane), while MustRecoverWithin bounds full cluster
-// recovery (control plane). Each can be violated while the other is fine — e.g.
-// a promoted standby that never rejoins keeps writes flowing yet leaves the
-// cluster degraded, and only MustRecoverWithin catches it.
+// fields assert on different properties of the managed cluster and fail
+// independently (ExceededPolicy trips if either is exceeded): MaxWriteOutage
+// bounds client-visible write availability, while MustRecoverWithin bounds the
+// cluster's return to its full declared topology (all pods Ready, CR Ready).
+// Each can be violated while the other is fine — e.g. after a failover writes
+// resume quickly (MaxWriteOutage happy) yet the cluster stays degraded until a
+// replacement standby rejoins, which only MustRecoverWithin catches.
 type OutagePolicy struct {
 	// MaxWriteOutage bounds how long the write path (client -> gateway ->
 	// primary) may be unavailable during the window. It is evaluated from the

From ace474b26d2b000594c346c228c6e49acf4f910b Mon Sep 17 00:00:00 2001
From: Wenting Wu <wentingwu@microsoft.com>
Date: Fri, 10 Jul 2026 13:45:38 -0400
Subject: [PATCH 6/6] refactor(longhaul): share one primary-handover
 write-outage budget

kill-primary-pod and upgrade-documentdb both interrupt writes for a
single primary handover, so they now share journal.PrimaryHandoverPolicy
(30s) instead of diverging (was 30s vs 45s). The 45s was inflated by
conflating the rolling upgrade's whole-topology restart with its
write-outage window; that longer restart is bounded by MustRecoverWithin
instead. A graceful switchover (upgrade) is no worse than an ungraceful
failover (kill-primary), which also pays a detection delay.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Signed-off-by: Wenting Wu <wentingwu@microsoft.com>
---
 test/longhaul/README.md                       |  7 +++--
 test/longhaul/journal/policy.go               | 28 +++++++++++++++++++
 test/longhaul/operations/kill_primary.go      | 12 +++-----
 test/longhaul/operations/kill_primary_test.go |  4 +--
 test/longhaul/operations/upgrade.go           | 13 ++++-----
 test/longhaul/operations/upgrade_test.go      |  5 ++--
 6 files changed, 47 insertions(+), 22 deletions(-)

diff --git a/test/longhaul/README.md b/test/longhaul/README.md
index 452fe3785..38ea91d0a 100644
--- a/test/longhaul/README.md
+++ b/test/longhaul/README.md
@@ -157,8 +157,11 @@ Outage budgets are expressed as **wall-clock write-outage durations**
 converts the observed failure count into an estimated outage using the workload's
 aggregate write rate (`workload.AggregateWriteRate(NumWriters)`), so the budgets
 are independent of `LONGHAUL_NUM_WRITERS`: `NoOutagePolicy` ≈ 300ms (noise
-cushion), `kill-primary-pod` = 30s (single automatic failover), and
-`upgrade-documentdb` = 45s (primary switchover during the rolling upgrade).
+cushion), while `kill-primary-pod` and `upgrade-documentdb` share the
+`journal.PrimaryHandoverPolicy` budget of 30s — both interrupt writes for a
+single primary handover (an ungraceful failover vs. a graceful switchover), and
+an upgrade's longer whole-topology restart is bounded by `MustRecoverWithin`,
+not the write-outage budget.
 
 ### RBAC for chaos operations
 
diff --git a/test/longhaul/journal/policy.go b/test/longhaul/journal/policy.go
index 1d41ac581..c6226877d 100644
--- a/test/longhaul/journal/policy.go
+++ b/test/longhaul/journal/policy.go
@@ -62,6 +62,34 @@ func NoOutagePolicy(recovery time.Duration) OutagePolicy {
 	}
 }
 
+// PrimaryHandoverWriteOutage is the write-outage budget for operations that
+// interrupt writes for exactly one primary handover. It is shared so the two
+// such operations cannot drift apart:
+//   - kill-primary-pod — an *ungraceful* failover (detect the lost pod, then
+//     promote a standby), and
+//   - upgrade-documentdb — a *graceful* switchover of the primary; the standby
+//     pod restarts during the rolling upgrade do NOT interrupt writes, so the
+//     write outage is just the one switchover (and a graceful switchover is
+//     typically no worse than an ungraceful failover, which pays a detection
+//     delay). The upgrade's longer, whole-topology restart is bounded by
+//     MustRecoverWithin, not here.
+//
+// Sized to comfortably cover a healthy single CNPG failover; heuristic pending
+// calibration against real long-haul runs.
+const PrimaryHandoverWriteOutage = 30 * time.Second
+
+// PrimaryHandoverPolicy is the outage budget for operations whose write path is
+// interrupted for a single primary handover (see PrimaryHandoverWriteOutage).
+// recovery bounds how long the cluster may take to return to full topology,
+// which can legitimately differ per operation (a rolling upgrade restarts every
+// pod and takes longer than a single failover).
+func PrimaryHandoverPolicy(recovery time.Duration) OutagePolicy {
+	return OutagePolicy{
+		MaxWriteOutage:    PrimaryHandoverWriteOutage,
+		MustRecoverWithin: recovery,
+	}
+}
+
 // DisruptionWindow represents an active or closed disruption period.
 type DisruptionWindow struct {
 	// OperationName identifies which operation opened this window.
diff --git a/test/longhaul/operations/kill_primary.go b/test/longhaul/operations/kill_primary.go
index 15084085d..dbd960571 100644
--- a/test/longhaul/operations/kill_primary.go
+++ b/test/longhaul/operations/kill_primary.go
@@ -67,13 +67,9 @@ func (k *KillPrimaryPod) Execute(ctx context.Context) error {
 }
 
 // OutagePolicy bounds the write outage of an automatic failover. Killing the
-// primary interrupts writes until CNPG detects the loss and promotes a standby;
-// a healthy single failover should restore the write path well within this
-// budget. Expressed as wall-clock outage time, so it is independent of the
-// configured writer count.
+// primary interrupts writes until CNPG detects the loss and promotes a standby.
+// It shares the single-primary-handover budget with upgrade-documentdb (see
+// journal.PrimaryHandoverPolicy).
 func (k *KillPrimaryPod) OutagePolicy() journal.OutagePolicy {
-	return journal.OutagePolicy{
-		MaxWriteOutage:    30 * time.Second,
-		MustRecoverWithin: k.recovery,
-	}
+	return journal.PrimaryHandoverPolicy(k.recovery)
 }
diff --git a/test/longhaul/operations/kill_primary_test.go b/test/longhaul/operations/kill_primary_test.go
index f35d49b39..36c723006 100644
--- a/test/longhaul/operations/kill_primary_test.go
+++ b/test/longhaul/operations/kill_primary_test.go
@@ -22,10 +22,10 @@ var _ = Describe("KillPrimaryPod", func() {
 		Expect(k.Weight()).To(Equal(2))
 	})
 
-	It("OutagePolicy bounds the failover write outage in wall-clock time", func() {
+	It("OutagePolicy shares the single-primary-handover budget with upgrade", func() {
 		k := NewKillPrimaryPod(&fakeClient{}, nil, 3*time.Minute)
 		p := k.OutagePolicy()
-		Expect(p.MaxWriteOutage).To(Equal(30 * time.Second))
+		Expect(p.MaxWriteOutage).To(Equal(journal.PrimaryHandoverWriteOutage))
 		Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute))
 	})
 
diff --git a/test/longhaul/operations/upgrade.go b/test/longhaul/operations/upgrade.go
index 6f823c47b..554356997 100644
--- a/test/longhaul/operations/upgrade.go
+++ b/test/longhaul/operations/upgrade.go
@@ -171,13 +171,10 @@ func (u *UpgradeDocumentDB) readDesiredVersion(ctx context.Context) (string, err
 }
 
 // OutagePolicy bounds the write outage of a rolling upgrade. Standby restarts
-// do not block writes; the write path is only interrupted during the primary
-// switchover, so the outage is comparable to a failover with some extra slack
-// for the graceful switchover coordination. Expressed as wall-clock outage
-// time, independent of the configured writer count.
+// do not block writes; the write path is only interrupted during the single
+// graceful primary switchover, so it shares the primary-handover budget with
+// kill-primary-pod (see journal.PrimaryHandoverPolicy). The upgrade's longer
+// whole-topology restart is bounded separately by MustRecoverWithin.
 func (u *UpgradeDocumentDB) OutagePolicy() journal.OutagePolicy {
-	return journal.OutagePolicy{
-		MaxWriteOutage:    45 * time.Second,
-		MustRecoverWithin: u.recovery,
-	}
+	return journal.PrimaryHandoverPolicy(u.recovery)
 }
diff --git a/test/longhaul/operations/upgrade_test.go b/test/longhaul/operations/upgrade_test.go
index 9b9a7a59c..ae5b7450e 100644
--- a/test/longhaul/operations/upgrade_test.go
+++ b/test/longhaul/operations/upgrade_test.go
@@ -10,6 +10,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
+	"github.com/documentdb/documentdb-operator/test/longhaul/journal"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/fake"
@@ -22,10 +23,10 @@ var _ = Describe("UpgradeDocumentDB", func() {
 		Expect(u.Weight()).To(Equal(1))
 	})
 
-	It("OutagePolicy gives upgrades a more lenient write-outage budget", func() {
+	It("OutagePolicy shares the single-primary-handover budget with kill-primary", func() {
 		u := NewUpgradeDocumentDB(&fakeClient{}, fake.NewSimpleClientset(), nil, nil, "ns", 10*time.Minute)
 		p := u.OutagePolicy()
-		Expect(p.MaxWriteOutage).To(Equal(45 * time.Second))
+		Expect(p.MaxWriteOutage).To(Equal(journal.PrimaryHandoverWriteOutage))
 		Expect(p.MustRecoverWithin).To(Equal(10 * time.Minute))
 	})