test(longhaul): add data-protection verifier (ScheduledBackup + retention)

.gitignore

@@ -278,6 +278,9 @@ Backup*/
 !test/e2e/manifests/backup/
 !test/e2e/tests/backup/
 !test/e2e/pkg/e2eutils/backup/
+# The long-haul driver's data-protection component lives here and is
+# likewise swallowed by the generic Backup*/ rule above.
+!test/longhaul/backup/
 UpgradeLog*.XML
 UpgradeLog*.htm
 ServiceFabricBackup/

test/longhaul/README.md

@@ -132,8 +132,47 @@ All configuration is via environment variables.
 | `LONGHAUL_MIN_INSTANCES` | No | `1` | Minimum `spec.instancesPerNode` for scale-down operations (CRD lower bound: 1). |
 | `LONGHAUL_MAX_INSTANCES` | No | `3` | Maximum `spec.instancesPerNode` for scale-up operations (CRD upper bound: 3). |
 | `LONGHAUL_REPORT_INTERVAL` | No | `1h` | How often to write checkpoint reports to ConfigMap. |
+| `LONGHAUL_BACKUP_ENABLED` | No | `true` | Enable the ScheduledBackup + retention verifier. |
+| `LONGHAUL_BACKUP_SCHEDULE` | No | `0 */6 * * *` | Cron schedule for the canary `ScheduledBackup`. |
+| `LONGHAUL_BACKUP_RETENTION_DAYS` | No | `1` | Retention window applied to child backups; also used to derive the retention-leak deadline. |
 | `LONGHAUL_RESET_DATA` | No | `false` | If `true`, drop the workload collection on startup. Off by default so a Deployment pod restart preserves durability history. |
 
+### Data Protection (ScheduledBackup + retention)
+
+When `LONGHAUL_BACKUP_ENABLED` is true, the driver ensures a `ScheduledBackup`
+named `<cluster>-longhaul` exists and matches the run's schedule/retention
+(an existing CR is reconciled in place, never recreated, so backup history is
+preserved across restarts and parameter changes) and runs a verifier
+concurrently with the operation scheduler (backup is deliberately **not**
+isolated from topology/chaos, per the design).
+
+The verifier only checks the properties a **multi-day** run can establish —
+things unit and e2e tests cannot:
+
+- **Scheduling liveness** — `status.lastScheduledTime` keeps advancing; a stalled
+  scheduler (past `status.nextScheduledTime` + grace) raises a warning.
+- **Completion** — child `Backup` CRs keep reaching `completed`; terminal
+  failures (`failed` / `skipped`) are counted.
+- **Retention leak** — no completed backup outlives its retention window
+  (`stoppedAt + spec.retentionDays*24h` + grace). The window is taken from each
+  backup's **own** `spec.retentionDays` (stamped at creation), so the check
+  stays correct even if a later run uses a different retention. A lingering
+  backup is a **FAIL**: expired backups aren't garbage-collected and the
+  population (and its PVCs / VolumeSnapshots) grows unbounded.
+
+It deliberately does **not** re-verify the operator's retention *arithmetic*
+(`expiredAt == stoppedAt + retentionDays*24h`) — that is a pure function already
+covered by the operator's unit tests and needs no accumulation. The oracle here
+is black-box: expired backups disappear. Because the minimum meaningful
+retention is 1 day, the leak check only fires on multi-day runs — exactly the
+accumulation window long-haul exists to cover.
+
+> **RBAC.** The driver ServiceAccount needs `create`/`get`/`list`/`update` on
+> `scheduledbackups.documentdb.io` and `list` on `backups.documentdb.io`. These
+> verbs must be present in `deploy/rbac.yaml` (added in the CI/CD PR) for the
+> backup verifier to function; without them it logs an error and the rest of the
+> run continues.
+
 ## CI Safety
 
 The long haul test binary is deployed as a Kubernetes Deployment on a dedicated AKS

test/longhaul/backup/metrics.go

@@ -0,0 +1,96 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+// Package backup implements the data-protection component of the long haul
+// driver. It provisions a ScheduledBackup against the canary cluster and
+// continuously verifies the properties that only a multi-day run can
+// establish: that backups keep being produced on schedule and completing,
+// and that expired backups are actually garbage-collected so the backup
+// population stays bounded over time (no PVC / VolumeSnapshot accumulation).
+//
+// It deliberately does NOT re-verify the operator's retention *arithmetic*
+// (expiredAt == stoppedAt + retentionDays*24h) — that is a pure function
+// already covered by the operator's unit tests and needs no accumulation.
+// The oracle here is black-box: expired backups disappear.
+//
+// The component runs concurrently with the operation scheduler — per the
+// long-haul design, backup is deliberately NOT isolated so that
+// backup-vs-topology serialization bugs surface here rather than in
+// production.
+package backup
+
+import (
+	"sync/atomic"
+	"time"
+)
+
+// Metrics tracks aggregate backup-verification counters using atomic
+// operations so the reporter goroutine can snapshot them without locking.
+//
+// RetentionLeaks is the retention oracle: any non-zero value means a
+// completed backup outlived its retention window (the operator failed to
+// garbage-collect it) and flips the run verdict to FAIL. The remaining
+// counters are observational and feed the report.
+type Metrics struct {
+	// Scheduled counts backups observed to have been scheduled by the
+	// ScheduledBackup (advances of status.lastScheduledTime).
+	Scheduled atomic.Int64
+
+	// Completed is the number of child backups observed in the "completed"
+	// phase (deduplicated by name across verification cycles).
+	Completed atomic.Int64
+
+	// Failed is the number of child backups observed in a terminal failure
+	// phase (deduplicated by name).
+	Failed atomic.Int64
+
+	// RetentionLeaks counts completed backups still present past their
+	// retention window (stoppedAt + retentionDays*24h). Non-zero => FAIL.
+	RetentionLeaks atomic.Int64
+
+	// LastChildCount is the number of child backups observed on the most
+	// recent verification cycle (the live backup population — expected to
+	// stabilize near retentionWindow/scheduleInterval at steady state).
+	LastChildCount atomic.Int64
+
+	// LastScheduledUnix is the Unix timestamp of the most recently observed
+	// status.lastScheduledTime; 0 until the first backup is scheduled.
+	LastScheduledUnix atomic.Int64
+}
+
+// NewMetrics creates an empty Metrics.
+func NewMetrics() *Metrics {
+	return &Metrics{}
+}
+
+// MetricsSnapshot is a point-in-time copy of Metrics.
+type MetricsSnapshot struct {
+	Scheduled      int64
+	Completed      int64
+	Failed         int64
+	RetentionLeaks int64
+	LastChildCount int64
+	LastScheduled  time.Time
+}
+
+// Snapshot captures the current metric values atomically.
+func (m *Metrics) Snapshot() MetricsSnapshot {
+	var lastScheduled time.Time
+	if unix := m.LastScheduledUnix.Load(); unix > 0 {
+		lastScheduled = time.Unix(unix, 0)
+	}
+	return MetricsSnapshot{
+		Scheduled:      m.Scheduled.Load(),
+		Completed:      m.Completed.Load(),
+		Failed:         m.Failed.Load(),
+		RetentionLeaks: m.RetentionLeaks.Load(),
+		LastChildCount: m.LastChildCount.Load(),
+		LastScheduled:  lastScheduled,
+	}
+}
+
+// HasRetentionLeak returns true if any completed backup has outlived its
+// retention window. A true result flips the overall run verdict to FAIL.
+func (s MetricsSnapshot) HasRetentionLeak() bool {
+	return s.RetentionLeaks > 0
+}

test/longhaul/backup/metrics_test.go

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+package backup
+
+import (
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Metrics", func() {
+	It("snapshots counters atomically", func() {
+		m := NewMetrics()
+		m.Scheduled.Add(3)
+		m.Completed.Add(2)
+		m.Failed.Add(1)
+		m.RetentionLeaks.Add(1)
+		m.LastChildCount.Store(7)
+		now := time.Now()
+		m.LastScheduledUnix.Store(now.Unix())
+
+		snap := m.Snapshot()
+		Expect(snap.Scheduled).To(Equal(int64(3)))
+		Expect(snap.Completed).To(Equal(int64(2)))
+		Expect(snap.Failed).To(Equal(int64(1)))
+		Expect(snap.RetentionLeaks).To(Equal(int64(1)))
+		Expect(snap.LastChildCount).To(Equal(int64(7)))
+		Expect(snap.LastScheduled.Unix()).To(Equal(now.Unix()))
+	})
+
+	It("reports zero LastScheduled when never scheduled", func() {
+		snap := NewMetrics().Snapshot()
+		Expect(snap.LastScheduled.IsZero()).To(BeTrue())
+	})
+
+	DescribeTable("HasRetentionLeak",
+		func(leaks int64, want bool) {
+			m := NewMetrics()
+			m.RetentionLeaks.Add(leaks)
+			Expect(m.Snapshot().HasRetentionLeak()).To(Equal(want))
+		},
+		Entry("clean", int64(0), false),
+		Entry("one leak", int64(1), true),
+		Entry("several leaks", int64(3), true),
+	)
+})

test/longhaul/backup/suite_test.go

@@ -0,0 +1,16 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+package backup
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestBackup(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Long Haul Backup Suite")
+}