diff --git a/test/longhaul/README.md b/test/longhaul/README.md index a7d311e9..38ea91d0 100644 --- a/test/longhaul/README.md +++ b/test/longhaul/README.md @@ -125,6 +125,7 @@ All configuration is via environment variables. | `LONGHAUL_DOCUMENTDB_URI` | Yes | — | Connection string to the DocumentDB gateway. | | `LONGHAUL_CLUSTER_NAME` | Yes | — | Name of the target DocumentDB cluster CR. | | `LONGHAUL_NAMESPACE` | No | `default` | Kubernetes namespace of the target cluster. | +| `LONGHAUL_OPERATOR_NAMESPACE` | No | `documentdb-operator` | Namespace of the DocumentDB operator Deployment (target of the `kill-operator-pod` chaos op). | | `LONGHAUL_MAX_DURATION` | No | `30m` | Max test duration. Use `0s` for run-until-failure. | | `LONGHAUL_NUM_WRITERS` | No | `5` | Number of concurrent writers. | | `LONGHAUL_OP_COOLDOWN` | No | `5m` | Cooldown between management operations. | @@ -134,6 +135,44 @@ All configuration is via environment variables. | `LONGHAUL_REPORT_INTERVAL` | No | `1h` | How often to write checkpoint reports to ConfigMap. | | `LONGHAUL_RESET_DATA` | No | `false` | If `true`, drop the workload collection on startup. Off by default so a Deployment pod restart preserves durability history. | +## Operations + +The scheduler runs one disruptive operation at a time, gated by steady state and +a global cooldown. Current operations: + +| Operation | Kind | Notes | +|-----------|------|-------| +| `scale-up` / `scale-down` | Topology | Adjusts `spec.instancesPerNode` within `[MIN, MAX]`. Only adds/removes a standby, so the primary write path is untouched (near-zero outage budget). | +| `upgrade-documentdb` | Topology | In-place version upgrade; requires HA (`instancesPerNode>=2`). | +| `kill-operator-pod` | Chaos | Deletes the operator pod; asserts the data plane keeps serving (near-zero outage budget). | +| `kill-primary-pod` | Chaos | Deletes the CNPG primary pod to exercise automatic failover; requires HA (`instancesPerNode>=2`). | + +Operations that keep the write path up throughout — the scale ops and +`kill-operator-pod` — share the near-zero `journal.NoOutagePolicy` budget instead +of ad-hoc per-op numbers, so a regression that unexpectedly disrupts writes +during a "safe" operation trips the policy. + +Outage budgets are expressed as **wall-clock write-outage durations** +(`OutagePolicy.MaxWriteOutage`), not raw write-failure counts. The journal +converts the observed failure count into an estimated outage using the workload's +aggregate write rate (`workload.AggregateWriteRate(NumWriters)`), so the budgets +are independent of `LONGHAUL_NUM_WRITERS`: `NoOutagePolicy` ≈ 300ms (noise +cushion), while `kill-primary-pod` and `upgrade-documentdb` share the +`journal.PrimaryHandoverPolicy` budget of 30s — both interrupt writes for a +single primary handover (an ungraceful failover vs. a graceful switchover), and +an upgrade's longer whole-topology restart is bounded by `MustRecoverWithin`, +not the write-outage budget. + +### RBAC for chaos operations + +Beyond the base RBAC the driver already needs, the chaos operations require the +driver ServiceAccount to be granted (in `deploy/rbac.yaml`): + +- **`kill-primary-pod`** — `get`/`list` on `clusters.postgresql.cnpg.io` (to read + `status.currentPrimary`) and `delete` on `pods` in the cluster namespace. +- **`kill-operator-pod`** — `get` on `deployments` and `get`/`list`/`delete` on + `pods` in the operator namespace (`LONGHAUL_OPERATOR_NAMESPACE`). + ## CI Safety The long haul test binary is deployed as a Kubernetes Deployment on a dedicated AKS diff --git a/test/longhaul/cmd/longhaul/main.go b/test/longhaul/cmd/longhaul/main.go index c7f375bd..a93870db 100644 --- a/test/longhaul/cmd/longhaul/main.go +++ b/test/longhaul/cmd/longhaul/main.go @@ -57,6 +57,7 @@ func run(cfg config.Config) int { // Initialize components. j := journal.New() + j.SetWriteRate(workload.AggregateWriteRate(cfg.NumWriters)) metrics := workload.NewMetrics() // Connect to DocumentDB. @@ -140,6 +141,8 @@ func run(cfg config.Config) int { operations.NewScaleUp(clusterClient, healthMon, cfg.MaxInstances, cfg.RecoveryTimeout), operations.NewScaleDown(clusterClient, healthMon, cfg.MinInstances, cfg.RecoveryTimeout), operations.NewUpgradeDocumentDB(clusterClient, k8sClientset, healthMon, j, cfg.Namespace, cfg.RecoveryTimeout), + operations.NewKillOperatorPod(k8sClientset, cfg.OperatorNamespace, cfg.RecoveryTimeout), + operations.NewKillPrimaryPod(clusterClient, healthMon, cfg.RecoveryTimeout), } // Start operation scheduler. diff --git a/test/longhaul/config/config.go b/test/longhaul/config/config.go index 5665bcd7..918431c3 100644 --- a/test/longhaul/config/config.go +++ b/test/longhaul/config/config.go @@ -18,6 +18,10 @@ const ( EnvNamespace = "LONGHAUL_NAMESPACE" EnvClusterName = "LONGHAUL_CLUSTER_NAME" + // EnvOperatorNamespace is the namespace where the DocumentDB operator + // Deployment runs (target of the kill-operator-pod chaos op). + EnvOperatorNamespace = "LONGHAUL_OPERATOR_NAMESPACE" + // Workload and operation tuning. EnvDocumentDBURI = "LONGHAUL_DOCUMENTDB_URI" EnvNumWriters = "LONGHAUL_NUM_WRITERS" @@ -47,6 +51,10 @@ type Config struct { // ClusterName is the name of the target DocumentDB cluster CR. ClusterName string + // OperatorNamespace is the namespace of the DocumentDB operator Deployment, + // targeted by the kill-operator-pod chaos operation. + OperatorNamespace string + // DocumentDBURI is the DocumentDB connection string for data-plane workload. DocumentDBURI string @@ -82,17 +90,18 @@ type Config struct { // DefaultConfig returns a Config with safe defaults for local development. func DefaultConfig() Config { return Config{ - MaxDuration: 30 * time.Minute, - Namespace: "default", - ClusterName: "", - DocumentDBURI: "", - NumWriters: 5, - OpCooldown: 5 * time.Minute, - RecoveryTimeout: 5 * time.Minute, - SteadyStateWait: 60 * time.Second, - MinInstances: 1, - MaxInstances: 3, - ReportInterval: 1 * time.Hour, + MaxDuration: 30 * time.Minute, + Namespace: "default", + ClusterName: "", + OperatorNamespace: "documentdb-operator", + DocumentDBURI: "", + NumWriters: 5, + OpCooldown: 5 * time.Minute, + RecoveryTimeout: 5 * time.Minute, + SteadyStateWait: 60 * time.Second, + MinInstances: 1, + MaxInstances: 3, + ReportInterval: 1 * time.Hour, } } @@ -117,6 +126,10 @@ func LoadFromEnv() (Config, error) { cfg.ClusterName = v } + if v := os.Getenv(EnvOperatorNamespace); v != "" { + cfg.OperatorNamespace = v + } + if v := os.Getenv(EnvDocumentDBURI); v != "" { cfg.DocumentDBURI = v } @@ -195,6 +208,9 @@ func (c *Config) Validate() error { if c.ClusterName == "" { return fmt.Errorf("cluster name must not be empty") } + if c.OperatorNamespace == "" { + return fmt.Errorf("operator namespace must not be empty") + } if c.NumWriters < 1 { return fmt.Errorf("num writers must be at least 1, got %d", c.NumWriters) } diff --git a/test/longhaul/config/config_test.go b/test/longhaul/config/config_test.go index 506803f5..c02196b8 100644 --- a/test/longhaul/config/config_test.go +++ b/test/longhaul/config/config_test.go @@ -17,6 +17,7 @@ var _ = Describe("Config", func() { Expect(cfg.MaxDuration).To(Equal(30 * time.Minute)) Expect(cfg.Namespace).To(Equal("default")) Expect(cfg.ClusterName).To(BeEmpty()) + Expect(cfg.OperatorNamespace).To(Equal("documentdb-operator")) Expect(cfg.NumWriters).To(Equal(5)) Expect(cfg.OpCooldown).To(Equal(5 * time.Minute)) Expect(cfg.RecoveryTimeout).To(Equal(5 * time.Minute)) @@ -32,6 +33,7 @@ var _ = Describe("Config", func() { BeforeEach(func() { for _, k := range []string{ EnvEnabled, EnvMaxDuration, EnvNamespace, EnvClusterName, + EnvOperatorNamespace, EnvDocumentDBURI, EnvNumWriters, EnvOpCooldown, EnvRecoveryTimeout, EnvSteadyStateWait, EnvMinInstances, EnvMaxInstances, EnvReportInterval, @@ -69,6 +71,13 @@ var _ = Describe("Config", func() { Expect(cfg.ClusterName).To(Equal("my-cluster")) }) + It("parses OperatorNamespace from env", func() { + GinkgoT().Setenv(EnvOperatorNamespace, "custom-operator-ns") + cfg, err := LoadFromEnv() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.OperatorNamespace).To(Equal("custom-operator-ns")) + }) + It("returns error for invalid MaxDuration", func() { GinkgoT().Setenv(EnvMaxDuration, "not-a-duration") _, err := LoadFromEnv() @@ -124,6 +133,13 @@ var _ = Describe("Config", func() { Expect(cfg.Validate()).To(MatchError(ContainSubstring("cluster name"))) }) + It("fails when OperatorNamespace is empty", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperatorNamespace = "" + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operator namespace"))) + }) + It("fails when MaxDuration is negative", func() { cfg := DefaultConfig() cfg.ClusterName = "test" diff --git a/test/longhaul/go.mod b/test/longhaul/go.mod index 06157717..dd41c099 100644 --- a/test/longhaul/go.mod +++ b/test/longhaul/go.mod @@ -3,6 +3,7 @@ module github.com/documentdb/documentdb-operator/test/longhaul go 1.25.11 require ( + github.com/cloudnative-pg/cloudnative-pg v1.28.1 github.com/documentdb/documentdb-operator v0.0.0-00010101000000-000000000000 github.com/documentdb/documentdb-operator/test/shared v0.0.0-00010101000000-000000000000 github.com/onsi/ginkgo/v2 v2.28.1 @@ -24,7 +25,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudnative-pg/barman-cloud v0.4.1-0.20260108104508-ced266c145f5 // indirect - github.com/cloudnative-pg/cloudnative-pg v1.28.1 // indirect github.com/cloudnative-pg/cnpg-i v0.3.1 // indirect github.com/cloudnative-pg/machinery v0.3.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/test/longhaul/journal/journal.go b/test/longhaul/journal/journal.go index bfddebc7..2ad07168 100644 --- a/test/longhaul/journal/journal.go +++ b/test/longhaul/journal/journal.go @@ -52,13 +52,39 @@ type Journal struct { // All closed disruption windows. closedWindows []DisruptionWindow + + // writesPerSecond is the workload's aggregate write rate, stamped onto each + // disruption window so ExceededPolicy can convert write-failure counts into + // an estimated outage duration. Defaults to DefaultWritesPerSecond; override + // with SetWriteRate once the real writer count is known. + writesPerSecond float64 } +// DefaultWritesPerSecond is the assumed aggregate write rate used until +// SetWriteRate is called. It matches the default workload (5 writers at one +// write per 100ms = 50 writes/s) so tests and un-configured journals still +// evaluate write-outage budgets sensibly. +const DefaultWritesPerSecond = 50.0 + // New creates a new empty Journal. func New() *Journal { return &Journal{ - events: make([]Event, 0, 256), + events: make([]Event, 0, 256), + writesPerSecond: DefaultWritesPerSecond, + } +} + +// SetWriteRate records the workload's aggregate write rate (writes/second across +// all writers) so disruption windows can translate write-failure counts into an +// estimated outage duration. Non-positive values are ignored, preserving the +// default. Safe for concurrent use. +func (j *Journal) SetWriteRate(writesPerSecond float64) { + if writesPerSecond <= 0 { + return } + j.mu.Lock() + defer j.mu.Unlock() + j.writesPerSecond = writesPerSecond } // Record appends a new event to the journal. Safe for concurrent use. @@ -112,9 +138,10 @@ func (j *Journal) OpenDisruptionWindow(operationName string, policy OutagePolicy } j.activeWindow = &DisruptionWindow{ - OperationName: operationName, - StartTime: time.Now(), - Policy: policy, + OperationName: operationName, + StartTime: time.Now(), + Policy: policy, + WritesPerSecond: j.writesPerSecond, } j.events = append(j.events, Event{ diff --git a/test/longhaul/journal/journal_test.go b/test/longhaul/journal/journal_test.go index 02e2b3c6..43c159db 100644 --- a/test/longhaul/journal/journal_test.go +++ b/test/longhaul/journal/journal_test.go @@ -43,7 +43,7 @@ var _ = Describe("Journal", func() { Describe("DisruptionWindow lifecycle", func() { It("opens, records failures, and closes correctly", func() { j := New() - policy := OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 10} + policy := OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second} Expect(j.ActiveWindow()).To(BeNil()) @@ -89,14 +89,14 @@ var _ = Describe("Journal", func() { It("returns false on a closed window within budget", func() { j := New() - j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 10}) + j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}) j.CloseDisruptionWindow() Expect(j.HasPolicyViolation()).To(BeFalse()) }) - It("returns true on a closed window over write-failure budget", func() { + It("returns true on a closed window over write-outage budget", func() { j := New() - j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 1}) + j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: 10 * time.Millisecond}) j.RecordWriteFailure() j.RecordWriteFailure() j.CloseDisruptionWindow() @@ -105,7 +105,7 @@ var _ = Describe("Journal", func() { It("returns true on an active window over time budget", func() { j := New() - j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Nanosecond, AllowedWriteFailures: 10}) + j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Nanosecond, MaxWriteOutage: time.Second}) time.Sleep(1 * time.Millisecond) Expect(j.HasPolicyViolation()).To(BeTrue()) }) diff --git a/test/longhaul/journal/policy.go b/test/longhaul/journal/policy.go index 25d28e82..c6226877 100644 --- a/test/longhaul/journal/policy.go +++ b/test/longhaul/journal/policy.go @@ -5,20 +5,88 @@ package journal import "time" -// OutagePolicy defines acceptable disruption bounds for an operation. +// OutagePolicy defines acceptable disruption bounds for an operation. Its two +// fields assert on different properties of the managed cluster and fail +// independently (ExceededPolicy trips if either is exceeded): MaxWriteOutage +// bounds client-visible write availability, while MustRecoverWithin bounds the +// cluster's return to its full declared topology (all pods Ready, CR Ready). +// Each can be violated while the other is fine — e.g. after a failover writes +// resume quickly (MaxWriteOutage happy) yet the cluster stays degraded until a +// replacement standby rejoins, which only MustRecoverWithin catches. type OutagePolicy struct { - // AllowedWriteFailures is the maximum number of write failures during the window. - AllowedWriteFailures int64 + // MaxWriteOutage bounds how long the write path (client -> gateway -> + // primary) may be unavailable during the window. It is evaluated from the + // observed write-failure count normalized by the workload's aggregate write + // rate (see DisruptionWindow.EstimatedWriteOutage), so the budget is + // expressed in wall-clock outage time and is independent of how many writer + // goroutines (LONGHAUL_NUM_WRITERS) are configured. + MaxWriteOutage time.Duration - // MustRecoverWithin is the maximum time from operation start to full recovery. + // MustRecoverWithin is the maximum time from operation start to full cluster + // recovery (steady state). Because a failed op is only logged, not counted + // toward the run verdict, this is the sole mechanism that turns a cluster + // that never converges back into a FAIL. MustRecoverWithin time.Duration } // DefaultOutagePolicy returns a conservative policy suitable for most operations. func DefaultOutagePolicy() OutagePolicy { return OutagePolicy{ - AllowedWriteFailures: 50, - MustRecoverWithin: 5 * time.Minute, + MaxWriteOutage: 5 * time.Second, + MustRecoverWithin: 5 * time.Minute, + } +} + +// NoOutageWriteOutageCushion is the tiny write-outage budget granted to +// operations that are expected NOT to disrupt the data plane. It is not a +// tolerance for real outages: one fully-failed write tick (every configured +// writer failing once) maps to exactly one writeInterval of estimated outage +// (~100ms) regardless of writer count, so this ~3-tick cushion absorbs unrelated +// background noise (a client reconnect, service-endpoint churn) without +// tolerating a genuine primary outage. Centralized so it can be recalibrated +// against real long-haul runs in one place. +const NoOutageWriteOutageCushion = 300 * time.Millisecond + +// NoOutagePolicy is the outage budget for operations that keep the write path +// up throughout and therefore must not cause a write outage. It is shared by +// every "no data-plane impact" operation: +// - control-plane faults, e.g. an operator pod restart, and +// - scaling that only adds or removes a standby replica (the primary, and +// thus the write path, is never touched). +// +// recovery bounds how long the cluster may take to return to steady state. +func NoOutagePolicy(recovery time.Duration) OutagePolicy { + return OutagePolicy{ + MaxWriteOutage: NoOutageWriteOutageCushion, + MustRecoverWithin: recovery, + } +} + +// PrimaryHandoverWriteOutage is the write-outage budget for operations that +// interrupt writes for exactly one primary handover. It is shared so the two +// such operations cannot drift apart: +// - kill-primary-pod — an *ungraceful* failover (detect the lost pod, then +// promote a standby), and +// - upgrade-documentdb — a *graceful* switchover of the primary; the standby +// pod restarts during the rolling upgrade do NOT interrupt writes, so the +// write outage is just the one switchover (and a graceful switchover is +// typically no worse than an ungraceful failover, which pays a detection +// delay). The upgrade's longer, whole-topology restart is bounded by +// MustRecoverWithin, not here. +// +// Sized to comfortably cover a healthy single CNPG failover; heuristic pending +// calibration against real long-haul runs. +const PrimaryHandoverWriteOutage = 30 * time.Second + +// PrimaryHandoverPolicy is the outage budget for operations whose write path is +// interrupted for a single primary handover (see PrimaryHandoverWriteOutage). +// recovery bounds how long the cluster may take to return to full topology, +// which can legitimately differ per operation (a rolling upgrade restarts every +// pod and takes longer than a single failover). +func PrimaryHandoverPolicy(recovery time.Duration) OutagePolicy { + return OutagePolicy{ + MaxWriteOutage: PrimaryHandoverWriteOutage, + MustRecoverWithin: recovery, } } @@ -38,6 +106,25 @@ type DisruptionWindow struct { // WriteFailures counts failures observed during this window. WriteFailures int64 + + // WritesPerSecond is the workload's aggregate write rate at the time the + // window opened. It is used to convert the raw WriteFailures count into an + // estimated write-outage duration (see EstimatedWriteOutage). A real outage + // makes every writer fail on every tick, so failures accrue at the full + // aggregate rate and count/rate recovers the wall-clock outage duration + // regardless of writer count. Zero disables the write-outage check. + WritesPerSecond float64 +} + +// EstimatedWriteOutage converts the observed write-failure count into an +// approximate duration for which the write path was unavailable, using the +// aggregate write rate captured when the window opened. Returns 0 when the rate +// is unknown (<= 0), which disables the write-outage portion of the policy. +func (w *DisruptionWindow) EstimatedWriteOutage() time.Duration { + if w.WritesPerSecond <= 0 { + return 0 + } + return time.Duration(float64(w.WriteFailures) / w.WritesPerSecond * float64(time.Second)) } // IsActive returns true if the disruption window has not been closed. @@ -59,7 +146,7 @@ func (w *DisruptionWindow) ExceededPolicy() bool { if w.Duration() > w.Policy.MustRecoverWithin { return true } - if w.WriteFailures > w.Policy.AllowedWriteFailures { + if w.EstimatedWriteOutage() > w.Policy.MaxWriteOutage { return true } return false diff --git a/test/longhaul/journal/policy_test.go b/test/longhaul/journal/policy_test.go index 4fe26b1a..dec57733 100644 --- a/test/longhaul/journal/policy_test.go +++ b/test/longhaul/journal/policy_test.go @@ -43,42 +43,65 @@ var _ = Describe("DisruptionWindow", func() { }, Entry("within all budgets", DisruptionWindow{ - StartTime: time.Now().Add(-10 * time.Second), - EndTime: time.Now(), - WriteFailures: 5, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + WriteFailures: 5, // 5/50 = 0.1s < 1s + WritesPerSecond: 50, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, false), Entry("exceeds MustRecoverWithin", DisruptionWindow{ - StartTime: time.Now().Add(-2 * time.Minute), - EndTime: time.Now(), - WriteFailures: 1, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-2 * time.Minute), + EndTime: time.Now(), + WriteFailures: 1, + WritesPerSecond: 50, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, true), - Entry("exceeds AllowedWriteFailures", + Entry("exceeds MaxWriteOutage", DisruptionWindow{ - StartTime: time.Now().Add(-10 * time.Second), - EndTime: time.Now(), - WriteFailures: 100, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + WriteFailures: 100, // 100/50 = 2s > 1s + WritesPerSecond: 50, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, true), - Entry("boundary: equal to write-failure budget is allowed", + Entry("boundary: estimated outage equal to budget is allowed", DisruptionWindow{ - StartTime: time.Now().Add(-10 * time.Second), - EndTime: time.Now(), - WriteFailures: 50, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + WriteFailures: 50, // 50/50 = exactly 1s + WritesPerSecond: 50, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, + }, false), + Entry("unknown write rate disables the write-outage check", + DisruptionWindow{ + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + WriteFailures: 100000, + WritesPerSecond: 0, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, false), Entry("active window also evaluated against MustRecoverWithin", DisruptionWindow{ - StartTime: time.Now().Add(-2 * time.Minute), - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-2 * time.Minute), + WritesPerSecond: 50, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, true), ) It("DefaultOutagePolicy returns no zero-valued field", func() { p := DefaultOutagePolicy() Expect(p.MustRecoverWithin).NotTo(BeZero()) - Expect(p.AllowedWriteFailures).NotTo(BeZero()) + Expect(p.MaxWriteOutage).NotTo(BeZero()) + }) + + It("NoOutagePolicy grants the near-zero cushion and echoes recovery", func() { + p := NoOutagePolicy(3 * time.Minute) + Expect(p.MaxWriteOutage).To(Equal(NoOutageWriteOutageCushion)) + Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute)) + }) + + It("NoOutagePolicy is far tighter than DefaultOutagePolicy", func() { + Expect(NoOutageWriteOutageCushion).To(BeNumerically("<", DefaultOutagePolicy().MaxWriteOutage)) }) }) diff --git a/test/longhaul/monitor/health.go b/test/longhaul/monitor/health.go index 0c5e9516..939f4805 100644 --- a/test/longhaul/monitor/health.go +++ b/test/longhaul/monitor/health.go @@ -49,6 +49,15 @@ type ClusterClient interface { // UpgradeDocumentDB patches spec.documentDBVersion and spec.schemaVersion="auto". UpgradeDocumentDB(ctx context.Context, version string) error + + // GetPrimaryInstance returns the name of the pod currently serving as the + // CNPG primary (from Cluster.status.currentPrimary). The pod name equals + // the CNPG instance name. Returns an error if no primary is known yet. + GetPrimaryInstance(ctx context.Context) (string, error) + + // DeletePod deletes the named pod in the cluster namespace. Used by chaos + // operations to inject pod-loss faults. + DeletePod(ctx context.Context, name string) error } // HealthMonitor continuously monitors cluster health and tracks steady-state. diff --git a/test/longhaul/monitor/health_test.go b/test/longhaul/monitor/health_test.go index 1ad903fd..4d5385e1 100644 --- a/test/longhaul/monitor/health_test.go +++ b/test/longhaul/monitor/health_test.go @@ -41,9 +41,11 @@ func (f *fakeClusterClient) GetClusterHealth(_ context.Context) (ClusterHealth, func (f *fakeClusterClient) GetCurrentDocumentDBImageTag(_ context.Context) (string, error) { return "", nil } -func (f *fakeClusterClient) GetInstancesPerNode(_ context.Context) (int, error) { return 1, nil } -func (f *fakeClusterClient) ScaleCluster(_ context.Context, _ int) error { return nil } -func (f *fakeClusterClient) UpgradeDocumentDB(_ context.Context, _ string) error { return nil } +func (f *fakeClusterClient) GetInstancesPerNode(_ context.Context) (int, error) { return 1, nil } +func (f *fakeClusterClient) ScaleCluster(_ context.Context, _ int) error { return nil } +func (f *fakeClusterClient) UpgradeDocumentDB(_ context.Context, _ string) error { return nil } +func (f *fakeClusterClient) GetPrimaryInstance(_ context.Context) (string, error) { return "", nil } +func (f *fakeClusterClient) DeletePod(_ context.Context, _ string) error { return nil } var _ = Describe("HealthMonitor", func() { Describe("IsSteadyState", func() { diff --git a/test/longhaul/monitor/k8sclient.go b/test/longhaul/monitor/k8sclient.go index b51ffdd8..f5b9807d 100644 --- a/test/longhaul/monitor/k8sclient.go +++ b/test/longhaul/monitor/k8sclient.go @@ -19,6 +19,8 @@ import ( metricsv "k8s.io/metrics/pkg/client/clientset/versioned" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + previewv1 "github.com/documentdb/documentdb-operator/api/preview" shareddb "github.com/documentdb/documentdb-operator/test/shared/documentdb" sharedk8s "github.com/documentdb/documentdb-operator/test/shared/k8s" @@ -61,7 +63,7 @@ func NewK8sClusterClient(cfg K8sClientConfig) (*K8sClusterClient, error) { return nil, fmt.Errorf("failed to create clientset: %w", err) } - scheme, err := shareddb.NewScheme() + scheme, err := shareddb.NewScheme(cnpgv1.AddToScheme) if err != nil { return nil, fmt.Errorf("failed to build scheme: %w", err) } @@ -227,6 +229,29 @@ func (k *K8sClusterClient) UpgradeDocumentDB(ctx context.Context, version string return nil } +// GetPrimaryInstance reads status.currentPrimary from the CNPG Cluster that +// backs this DocumentDB. The CNPG Cluster name equals the DocumentDB CR name, +// and the returned instance name equals the primary pod name. +func (k *K8sClusterClient) GetPrimaryInstance(ctx context.Context) (string, error) { + var cluster cnpgv1.Cluster + key := types.NamespacedName{Namespace: k.namespace, Name: k.clusterName} + if err := k.crClient.Get(ctx, key, &cluster); err != nil { + return "", fmt.Errorf("failed to get CNPG Cluster: %w", err) + } + if cluster.Status.CurrentPrimary == "" { + return "", fmt.Errorf("CNPG Cluster %s has no current primary yet", k.clusterName) + } + return cluster.Status.CurrentPrimary, nil +} + +// DeletePod deletes the named pod in the cluster namespace. +func (k *K8sClusterClient) DeletePod(ctx context.Context, name string) error { + if err := k.clientset.CoreV1().Pods(k.namespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil { + return fmt.Errorf("failed to delete pod %s/%s: %w", k.namespace, name, err) + } + return nil +} + // GetPodMetrics queries metrics-server for pod resource usage. // Returns nil, nil if metrics-server is not available. func (k *K8sClusterClient) GetPodMetrics(ctx context.Context) ([]PodMetrics, error) { diff --git a/test/longhaul/operations/kill_operator.go b/test/longhaul/operations/kill_operator.go new file mode 100644 index 00000000..6018aab9 --- /dev/null +++ b/test/longhaul/operations/kill_operator.go @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "fmt" + "time" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" +) + +// OperatorDeploymentName is the fixed name of the operator Deployment. The +// operator is a cluster singleton, so this name is stable across installs. +const OperatorDeploymentName = "documentdb-operator" + +// KillOperatorPod deletes the running operator pod to verify that an operator +// restart does not disrupt the data plane. The CNPG-managed database keeps +// serving reads and writes while the Deployment reschedules the control plane, +// so the workload verifier should observe (near) zero write failures. Recovery +// is asserted by the Deployment returning to Available. +type KillOperatorPod struct { + clientset kubernetes.Interface + namespace string + deployment string + recovery time.Duration +} + +// NewKillOperatorPod creates a KillOperatorPod operation targeting the operator +// Deployment in the given namespace. +func NewKillOperatorPod(clientset kubernetes.Interface, namespace string, recovery time.Duration) *KillOperatorPod { + return &KillOperatorPod{ + clientset: clientset, + namespace: namespace, + deployment: OperatorDeploymentName, + recovery: recovery, + } +} + +func (k *KillOperatorPod) Name() string { return "kill-operator-pod" } + +func (k *KillOperatorPod) Weight() int { return 2 } + +// Precondition requires the operator Deployment to exist and currently be +// Available, so the fault isn't stacked on an already-restarting operator. +func (k *KillOperatorPod) Precondition(ctx context.Context) (bool, string) { + dep, err := k.getDeployment(ctx) + if err != nil { + return false, fmt.Sprintf("cannot get operator deployment: %v", err) + } + if !isDeploymentAvailable(dep) { + return false, "operator deployment not currently available" + } + return true, "" +} + +func (k *KillOperatorPod) Execute(ctx context.Context) error { + dep, err := k.getDeployment(ctx) + if err != nil { + return fmt.Errorf("get operator deployment: %w", err) + } + + // Resolve the pod set from the Deployment's own selector so we don't + // depend on the release-name-derived "app" label value. + selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels).String() + pods, err := k.clientset.CoreV1().Pods(k.namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return fmt.Errorf("list operator pods: %w", err) + } + + target := oldestRunningPod(pods.Items) + if target == "" { + return fmt.Errorf("no running operator pod found for selector %q", selector) + } + if err := k.clientset.CoreV1().Pods(k.namespace).Delete(ctx, target, metav1.DeleteOptions{}); err != nil { + return fmt.Errorf("delete operator pod %s: %w", target, err) + } + + // Wait for the Deployment to reschedule and become Available again. + recoveryCtx, cancel := context.WithTimeout(ctx, k.recovery) + defer cancel() + return k.waitForDeploymentAvailable(recoveryCtx) +} + +// OutagePolicy: an operator restart is a control-plane fault that must not take +// down the data plane, so it shares the near-zero NoOutagePolicy budget. +func (k *KillOperatorPod) OutagePolicy() journal.OutagePolicy { + return journal.NoOutagePolicy(k.recovery) +} + +func (k *KillOperatorPod) getDeployment(ctx context.Context) (*appsv1.Deployment, error) { + return k.clientset.AppsV1().Deployments(k.namespace).Get(ctx, k.deployment, metav1.GetOptions{}) +} + +func (k *KillOperatorPod) waitForDeploymentAvailable(ctx context.Context) error { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + if dep, err := k.getDeployment(ctx); err == nil && isDeploymentAvailable(dep) { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("timed out waiting for operator deployment to become available: %w", ctx.Err()) + case <-ticker.C: + } + } +} + +// isDeploymentAvailable reports whether the Deployment has its full desired +// replica count ready with none unavailable and the observed generation caught +// up to the latest spec. +func isDeploymentAvailable(dep *appsv1.Deployment) bool { + if dep == nil { + return false + } + desired := int32(1) + if dep.Spec.Replicas != nil { + desired = *dep.Spec.Replicas + } + if dep.Status.ObservedGeneration < dep.Generation { + return false + } + return dep.Status.ReadyReplicas >= desired && dep.Status.UnavailableReplicas == 0 +} + +// oldestRunningPod returns the name of the oldest pod in the Running phase, or +// "" if none are running. Targeting the oldest makes the choice deterministic. +func oldestRunningPod(pods []corev1.Pod) string { + name := "" + var oldest time.Time + for i := range pods { + p := &pods[i] + if p.Status.Phase != corev1.PodRunning || p.DeletionTimestamp != nil { + continue + } + ts := p.CreationTimestamp.Time + if name == "" || ts.Before(oldest) { + name = p.Name + oldest = ts + } + } + return name +} diff --git a/test/longhaul/operations/kill_operator_test.go b/test/longhaul/operations/kill_operator_test.go new file mode 100644 index 00000000..d409b5ab --- /dev/null +++ b/test/longhaul/operations/kill_operator_test.go @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +const opNS = "documentdb-operator" + +func operatorDeployment(desired, ready, unavailable int32, gen, observed int64) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: OperatorDeploymentName, + Namespace: opNS, + Generation: gen, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &desired, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "documentdb-operator"}}, + }, + Status: appsv1.DeploymentStatus{ + ReadyReplicas: ready, + UnavailableReplicas: unavailable, + ObservedGeneration: observed, + }, + } +} + +func operatorPod(name string, phase corev1.PodPhase, ageSeconds int) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: opNS, + Labels: map[string]string{"app": "documentdb-operator"}, + CreationTimestamp: metav1.NewTime(time.Now().Add(-time.Duration(ageSeconds) * time.Second)), + }, + Status: corev1.PodStatus{Phase: phase}, + } +} + +var _ = Describe("KillOperatorPod", func() { + It("Name is kill-operator-pod and Weight is 2", func() { + k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, time.Minute) + Expect(k.Name()).To(Equal("kill-operator-pod")) + Expect(k.Weight()).To(Equal(2)) + }) + + It("OutagePolicy uses the near-zero NoOutagePolicy budget", func() { + k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, 2*time.Minute) + p := k.OutagePolicy() + Expect(p.MaxWriteOutage).To(Equal(journal.NoOutageWriteOutageCushion)) + Expect(p.MustRecoverWithin).To(Equal(2 * time.Minute)) + }) + + Describe("Precondition", func() { + It("skips when the deployment is missing", func() { + k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, time.Minute) + ok, reason := k.Precondition(context.Background()) + Expect(ok).To(BeFalse()) + Expect(reason).To(ContainSubstring("cannot get operator deployment")) + }) + + It("skips when the deployment is not available", func() { + dep := operatorDeployment(1, 0, 1, 1, 1) + k := NewKillOperatorPod(fake.NewSimpleClientset(dep), opNS, time.Minute) + ok, reason := k.Precondition(context.Background()) + Expect(ok).To(BeFalse()) + Expect(reason).To(ContainSubstring("not currently available")) + }) + + It("is eligible when the deployment is available", func() { + dep := operatorDeployment(1, 1, 0, 1, 1) + k := NewKillOperatorPod(fake.NewSimpleClientset(dep), opNS, time.Minute) + ok, _ := k.Precondition(context.Background()) + Expect(ok).To(BeTrue()) + }) + }) + + Describe("Execute", func() { + It("deletes the oldest running operator pod and returns once available", func() { + dep := operatorDeployment(1, 1, 0, 1, 1) + newer := operatorPod("op-new", corev1.PodRunning, 10) + older := operatorPod("op-old", corev1.PodRunning, 100) + cs := fake.NewSimpleClientset(dep, newer, older) + k := NewKillOperatorPod(cs, opNS, time.Minute) + + err := k.Execute(context.Background()) + Expect(err).NotTo(HaveOccurred()) + + _, getErr := cs.CoreV1().Pods(opNS).Get(context.Background(), "op-old", metav1.GetOptions{}) + Expect(getErr).To(HaveOccurred(), "oldest pod should have been deleted") + _, getErr = cs.CoreV1().Pods(opNS).Get(context.Background(), "op-new", metav1.GetOptions{}) + Expect(getErr).NotTo(HaveOccurred(), "newer pod should be untouched") + }) + + It("fails when no running pod matches the selector", func() { + dep := operatorDeployment(1, 1, 0, 1, 1) + pending := operatorPod("op-pending", corev1.PodPending, 10) + cs := fake.NewSimpleClientset(dep, pending) + k := NewKillOperatorPod(cs, opNS, time.Minute) + + err := k.Execute(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no running operator pod")) + }) + }) +}) diff --git a/test/longhaul/operations/kill_primary.go b/test/longhaul/operations/kill_primary.go new file mode 100644 index 00000000..dbd96057 --- /dev/null +++ b/test/longhaul/operations/kill_primary.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "fmt" + "time" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + "github.com/documentdb/documentdb-operator/test/longhaul/monitor" +) + +// KillPrimaryPod deletes the CNPG primary pod to exercise the automatic +// failover path: CNPG must promote a standby, and the cluster must return to +// steady state within the recovery budget. The continuous workload verifier +// independently catches any data loss caused by the failover. +type KillPrimaryPod struct { + client monitor.ClusterClient + healthMon *monitor.HealthMonitor + recovery time.Duration +} + +// NewKillPrimaryPod creates a KillPrimaryPod operation. +func NewKillPrimaryPod(client monitor.ClusterClient, health *monitor.HealthMonitor, recovery time.Duration) *KillPrimaryPod { + return &KillPrimaryPod{ + client: client, + healthMon: health, + recovery: recovery, + } +} + +func (k *KillPrimaryPod) Name() string { return "kill-primary-pod" } + +func (k *KillPrimaryPod) Weight() int { return 2 } + +// Precondition requires at least one standby (instancesPerNode>=2). Killing the +// sole instance of a single-instance cluster would cause guaranteed downtime +// with no failover target — a true-but-useless policy violation. The same guard +// (and rationale) is used by UpgradeDocumentDB; skips don't consume the +// scheduler cooldown, so this is free to re-evaluate on the next tick. +func (k *KillPrimaryPod) Precondition(ctx context.Context) (bool, string) { + ipn, err := k.client.GetInstancesPerNode(ctx) + if err != nil { + return false, fmt.Sprintf("cannot read instancesPerNode: %v", err) + } + if ipn < 2 { + return false, fmt.Sprintf("instancesPerNode=%d (no HA standby) — killing primary would cause real downtime; skipping", ipn) + } + return true, "" +} + +func (k *KillPrimaryPod) Execute(ctx context.Context) error { + primary, err := k.client.GetPrimaryInstance(ctx) + if err != nil { + return fmt.Errorf("get primary instance: %w", err) + } + if err := k.client.DeletePod(ctx, primary); err != nil { + return fmt.Errorf("delete primary pod %s: %w", primary, err) + } + + // Wait for CNPG to elect a new primary and the cluster to settle. + recoveryCtx, cancel := context.WithTimeout(ctx, k.recovery) + defer cancel() + return k.healthMon.WaitForSteadyState(recoveryCtx) +} + +// OutagePolicy bounds the write outage of an automatic failover. Killing the +// primary interrupts writes until CNPG detects the loss and promotes a standby. +// It shares the single-primary-handover budget with upgrade-documentdb (see +// journal.PrimaryHandoverPolicy). +func (k *KillPrimaryPod) OutagePolicy() journal.OutagePolicy { + return journal.PrimaryHandoverPolicy(k.recovery) +} diff --git a/test/longhaul/operations/kill_primary_test.go b/test/longhaul/operations/kill_primary_test.go new file mode 100644 index 00000000..36c72300 --- /dev/null +++ b/test/longhaul/operations/kill_primary_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "errors" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + "github.com/documentdb/documentdb-operator/test/longhaul/monitor" +) + +var _ = Describe("KillPrimaryPod", func() { + It("Name is kill-primary-pod and Weight is 2", func() { + k := NewKillPrimaryPod(&fakeClient{}, nil, time.Minute) + Expect(k.Name()).To(Equal("kill-primary-pod")) + Expect(k.Weight()).To(Equal(2)) + }) + + It("OutagePolicy shares the single-primary-handover budget with upgrade", func() { + k := NewKillPrimaryPod(&fakeClient{}, nil, 3*time.Minute) + p := k.OutagePolicy() + Expect(p.MaxWriteOutage).To(Equal(journal.PrimaryHandoverWriteOutage)) + Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute)) + }) + + DescribeTable("Precondition", + func(ipn int, ipnErr error, wantOK bool, wantReasonHas string) { + c := &fakeClient{instancesPerNode: ipn, ipnErr: ipnErr} + k := NewKillPrimaryPod(c, nil, time.Minute) + + ok, reason := k.Precondition(context.Background()) + Expect(ok).To(Equal(wantOK), "reason=%q", reason) + if wantReasonHas != "" { + Expect(reason).To(ContainSubstring(wantReasonHas)) + } + }, + Entry("single-instance: ipn=1 -> skip", 1, nil, false, "no HA standby"), + Entry("read error -> skip", 0, errors.New("boom"), false, "cannot read instancesPerNode"), + Entry("HA: ipn=2 -> eligible", 2, nil, true, ""), + Entry("HA: ipn=3 -> eligible", 3, nil, true, ""), + ) + + It("Execute deletes the reported primary pod", func() { + c := &fakeClient{instancesPerNode: 2, primary: "cluster-1"} + // The health monitor never reaches steady state here (its Run loop + // isn't started), so Execute times out on WaitForSteadyState — but the + // primary delete side-effect has already happened, which is what we + // assert. A short recovery keeps the test fast. + hm := monitor.NewHealthMonitor(c, journal.New(), time.Hour) + k := NewKillPrimaryPod(c, hm, 500*time.Millisecond) + + _ = k.Execute(context.Background()) + c.mu.Lock() + defer c.mu.Unlock() + Expect(c.deletedPods).To(ConsistOf("cluster-1")) + }) + + It("Execute fails without deleting when the primary is unknown", func() { + c := &fakeClient{instancesPerNode: 2, primaryErr: errors.New("no primary")} + k := NewKillPrimaryPod(c, nil, time.Second) + + err := k.Execute(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("get primary instance")) + c.mu.Lock() + defer c.mu.Unlock() + Expect(c.deletedPods).To(BeEmpty()) + }) +}) diff --git a/test/longhaul/operations/scale.go b/test/longhaul/operations/scale.go index 9a167295..f002fdc6 100644 --- a/test/longhaul/operations/scale.go +++ b/test/longhaul/operations/scale.go @@ -77,6 +77,9 @@ type ScaleUp struct{ scaleOp } // NewScaleUp creates a ScaleUp operation. maxInstances is clamped to the // CRD upper bound (3) to avoid admission rejections. +// +// Scaling up only adds a standby replica (the primary and thus the write path +// is untouched), so it uses the near-zero NoOutagePolicy budget. func NewScaleUp(client monitor.ClusterClient, health *monitor.HealthMonitor, maxInstances int, recovery time.Duration) *ScaleUp { if maxInstances > 3 { maxInstances = 3 @@ -90,10 +93,7 @@ func NewScaleUp(client monitor.ClusterClient, health *monitor.HealthMonitor, max bound: maxInstances, boundKind: "max", recovery: recovery, - policy: journal.OutagePolicy{ - AllowedWriteFailures: 20, - MustRecoverWithin: recovery, - }, + policy: journal.NoOutagePolicy(recovery), }} } @@ -105,6 +105,10 @@ type ScaleDown struct{ scaleOp } // NewScaleDown creates a ScaleDown operation. minInstances is clamped to the // CRD lower bound (1) to avoid admission rejections. +// +// Scaling down removes the highest-ordinal standby (CNPG never removes the +// primary), so the write path stays up and it uses the same near-zero +// NoOutagePolicy budget as scale-up. func NewScaleDown(client monitor.ClusterClient, health *monitor.HealthMonitor, minInstances int, recovery time.Duration) *ScaleDown { if minInstances < 1 { minInstances = 1 @@ -118,10 +122,7 @@ func NewScaleDown(client monitor.ClusterClient, health *monitor.HealthMonitor, m bound: minInstances, boundKind: "min", recovery: recovery, - policy: journal.OutagePolicy{ - AllowedWriteFailures: 50, - MustRecoverWithin: recovery, - }, + policy: journal.NoOutagePolicy(recovery), }} } diff --git a/test/longhaul/operations/scale_test.go b/test/longhaul/operations/scale_test.go index 3df032c0..8c9fc953 100644 --- a/test/longhaul/operations/scale_test.go +++ b/test/longhaul/operations/scale_test.go @@ -12,6 +12,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" "github.com/documentdb/documentdb-operator/test/longhaul/monitor" ) @@ -23,6 +24,10 @@ type fakeClient struct { imageTag string scaleCalls []int upgradeCalls []string + primary string + primaryErr error + deleteErr error + deletedPods []string } func (f *fakeClient) GetClusterHealth(_ context.Context) (monitor.ClusterHealth, error) { @@ -51,6 +56,20 @@ func (f *fakeClient) UpgradeDocumentDB(_ context.Context, v string) error { f.upgradeCalls = append(f.upgradeCalls, v) return nil } +func (f *fakeClient) GetPrimaryInstance(_ context.Context) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.primary, f.primaryErr +} +func (f *fakeClient) DeletePod(_ context.Context, name string) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.deleteErr != nil { + return f.deleteErr + } + f.deletedPods = append(f.deletedPods, name) + return nil +} var _ = Describe("ScaleUp", func() { DescribeTable("clamps maxInstances to the CRD upper bound", @@ -87,10 +106,10 @@ var _ = Describe("ScaleUp", func() { Entry("blocked: ipn read error", 0, errors.New("apiserver down"), 3, false, "cannot get instancesPerNode"), ) - It("OutagePolicy uses tighter budgets and echoes MustRecoverWithin", func() { + It("OutagePolicy uses the near-zero NoOutagePolicy budget and echoes MustRecoverWithin", func() { s := NewScaleUp(&fakeClient{}, nil, 3, 5*time.Minute) p := s.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(int64(20))) + Expect(p.MaxWriteOutage).To(Equal(journal.NoOutageWriteOutageCushion)) Expect(p.MustRecoverWithin).To(Equal(5 * time.Minute)) }) }) @@ -130,9 +149,9 @@ var _ = Describe("ScaleDown", func() { Entry("blocked: ipn read error", 0, errors.New("apiserver down"), 1, false, "cannot get instancesPerNode"), ) - It("OutagePolicy is more lenient than scale-up", func() { + It("OutagePolicy shares the near-zero NoOutagePolicy budget with scale-up", func() { s := NewScaleDown(&fakeClient{}, nil, 1, 5*time.Minute) p := s.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(int64(50))) + Expect(p.MaxWriteOutage).To(Equal(journal.NoOutageWriteOutageCushion)) }) }) diff --git a/test/longhaul/operations/upgrade.go b/test/longhaul/operations/upgrade.go index f9a586b6..55435699 100644 --- a/test/longhaul/operations/upgrade.go +++ b/test/longhaul/operations/upgrade.go @@ -170,11 +170,11 @@ func (u *UpgradeDocumentDB) readDesiredVersion(ctx context.Context) (string, err return cm.Data[VersionConfigMapKey], nil } -// OutagePolicy allows for a longer disruption window during an upgrade -// because rolling restarts touch every pod sequentially. +// OutagePolicy bounds the write outage of a rolling upgrade. Standby restarts +// do not block writes; the write path is only interrupted during the single +// graceful primary switchover, so it shares the primary-handover budget with +// kill-primary-pod (see journal.PrimaryHandoverPolicy). The upgrade's longer +// whole-topology restart is bounded separately by MustRecoverWithin. func (u *UpgradeDocumentDB) OutagePolicy() journal.OutagePolicy { - return journal.OutagePolicy{ - AllowedWriteFailures: 200, - MustRecoverWithin: u.recovery, - } + return journal.PrimaryHandoverPolicy(u.recovery) } diff --git a/test/longhaul/operations/upgrade_test.go b/test/longhaul/operations/upgrade_test.go index f442e5b5..ae5b7450 100644 --- a/test/longhaul/operations/upgrade_test.go +++ b/test/longhaul/operations/upgrade_test.go @@ -10,6 +10,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -22,10 +23,10 @@ var _ = Describe("UpgradeDocumentDB", func() { Expect(u.Weight()).To(Equal(1)) }) - It("OutagePolicy gives upgrades a more lenient failure budget", func() { + It("OutagePolicy shares the single-primary-handover budget with kill-primary", func() { u := NewUpgradeDocumentDB(&fakeClient{}, fake.NewSimpleClientset(), nil, nil, "ns", 10*time.Minute) p := u.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(int64(200))) + Expect(p.MaxWriteOutage).To(Equal(journal.PrimaryHandoverWriteOutage)) Expect(p.MustRecoverWithin).To(Equal(10 * time.Minute)) }) diff --git a/test/longhaul/report/report.go b/test/longhaul/report/report.go index e7316a7f..07576ef5 100644 --- a/test/longhaul/report/report.go +++ b/test/longhaul/report/report.go @@ -93,15 +93,16 @@ func GenerateMarkdown(s Summary) string { // Disruption Windows if len(s.Windows) > 0 { b.WriteString("## Disruption Windows\n\n") - b.WriteString("| Operation | Duration | Write Failures | Policy Exceeded |\n") - b.WriteString("|-----------|----------|----------------|------------------|\n") + b.WriteString("| Operation | Duration | Write Failures | Est. Write Outage | Policy Exceeded |\n") + b.WriteString("|-----------|----------|----------------|-------------------|------------------|\n") for _, w := range s.Windows { exceeded := "No" if w.ExceededPolicy() { exceeded = "**YES**" } - fmt.Fprintf(&b, "| %s | %s | %d | %s |\n", - w.OperationName, w.Duration().Round(time.Second), w.WriteFailures, exceeded) + fmt.Fprintf(&b, "| %s | %s | %d | %s | %s |\n", + w.OperationName, w.Duration().Round(time.Second), w.WriteFailures, + w.EstimatedWriteOutage().Round(time.Millisecond), exceeded) } b.WriteString("\n") } diff --git a/test/longhaul/report/report_test.go b/test/longhaul/report/report_test.go index f8b77297..33fc844b 100644 --- a/test/longhaul/report/report_test.go +++ b/test/longhaul/report/report_test.go @@ -74,11 +74,12 @@ var _ = Describe("GenerateMarkdown", func() { md := GenerateMarkdown(Summary{ Result: ResultPass, Windows: []journal.DisruptionWindow{{ - OperationName: "scale-up", - StartTime: now.Add(-30 * time.Second), - EndTime: now, - WriteFailures: 3, - Policy: journal.OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + OperationName: "scale-up", + StartTime: now.Add(-30 * time.Second), + EndTime: now, + WriteFailures: 3, + WritesPerSecond: 50, + Policy: journal.OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }}, }) Expect(md).To(ContainSubstring("Disruption Windows")) diff --git a/test/longhaul/workload/writer.go b/test/longhaul/workload/writer.go index c2fdcc3e..dddfd0e5 100644 --- a/test/longhaul/workload/writer.go +++ b/test/longhaul/workload/writer.go @@ -28,6 +28,18 @@ const ( writeInterval = 100 * time.Millisecond ) +// AggregateWriteRate returns the workload's aggregate write rate in writes per +// second across all writer goroutines, given the configured writer count. It is +// the reciprocal of the per-writer writeInterval scaled by numWriters, and is +// used to convert observed write-failure counts into an estimated outage +// duration (see journal.DisruptionWindow). Returns 0 for a non-positive count. +func AggregateWriteRate(numWriters int) float64 { + if numWriters <= 0 { + return 0 + } + return float64(numWriters) / writeInterval.Seconds() +} + // WriteDocument is the schema for data-plane durability tracking. type WriteDocument struct { WriterID string `bson:"writer_id"`