feat: PoC failover reservation controller (#572)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

## Release Notes

* **New Features**
* Added failover reservations system for automatic VM failover handling
* Added new scheduling pipelines for failover reservation creation and
reuse
* Added visualization tool for monitoring reservations and VM
allocations

* **Documentation**
  * Added comprehensive failover reservations system documentation

* **Improvements**
  * Updated reservation domain label keys to cloud domain
* Extended reservation status with timestamp tracking for changes and
acknowledgments

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: umswmayj

.gitignore

@@ -4,6 +4,8 @@
 *.dll
 *.so
 *.dylib
+*.tgz
+
 build/**
 
 # Test binary, built with `go test -c`

Makefile

@@ -6,7 +6,7 @@ GOBIN=$(shell go env GOBIN)
 endif
 
 .PHONY: all
-all: crds deepcopy lint test
+all: crds deepcopy lint-fix format lint test
 
 .PHONY: help
 help: ## Display this help.

api/v1alpha1/reservation_types.go

@@ -29,7 +29,7 @@ const (
 
 	// LabelReservationType identifies the type of reservation.
 	// This label is present on all reservations to enable type-based filtering.
-	LabelReservationType = "reservations.cortex.sap.com/type"
+	LabelReservationType = "reservations.cortex.cloud/type"
 
 	// Reservation type label values
 	ReservationTypeLabelCommittedResource = "committed-resource"
@@ -152,6 +152,18 @@ type FailoverReservationStatus struct {
 	// Key: VM/instance UUID, Value: Host name where the VM is currently running.
 	// +kubebuilder:validation:Optional
 	Allocations map[string]string `json:"allocations,omitempty"`
+
+	// LastChanged tracks when the reservation was last modified.
+	// This is used to track pending changes that need acknowledgment.
+	// +kubebuilder:validation:Optional
+	LastChanged *metav1.Time `json:"lastChanged,omitempty"`
+
+	// AcknowledgedAt is the timestamp when the last change was acknowledged.
+	// When nil, the reservation is in a pending state awaiting acknowledgment.
+	// This does not affect the Ready condition - reservations are still considered
+	// ready even when not yet acknowledged.
+	// +kubebuilder:validation:Optional
+	AcknowledgedAt *metav1.Time `json:"acknowledgedAt,omitempty"`
 }
 
 // ReservationStatus defines the observed state of Reservation.
@@ -189,7 +201,7 @@ type ReservationStatus struct {
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:resource:scope=Cluster
-// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".metadata.labels['reservations\\.cortex\\.sap\\.com/type']"
+// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".metadata.labels['reservations\\.cortex\\.cloud/type']"
 // +kubebuilder:printcolumn:name="Host",type="string",JSONPath=".status.host"
 // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].status"
 

api/v1alpha1/zz_generated.deepcopy.go

@@ -42,13 +42,16 @@ import (
 	"github.com/cobaltcore-dev/cortex/internal/knowledge/kpis"
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/cinder"
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/explanation"
+	"github.com/cobaltcore-dev/cortex/internal/scheduling/external"
 	schedulinglib "github.com/cobaltcore-dev/cortex/internal/scheduling/lib"
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/machines"
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/manila"
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/nova"
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/pods"
+	"github.com/cobaltcore-dev/cortex/internal/scheduling/reservations"
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments"
 	reservationscontroller "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/controller"
+	"github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failover"
 	"github.com/cobaltcore-dev/cortex/pkg/conf"
 	"github.com/cobaltcore-dev/cortex/pkg/monitoring"
 	"github.com/cobaltcore-dev/cortex/pkg/multicluster"
@@ -142,6 +145,12 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	// Log the main configuration
+	setupLog.Info("loaded main configuration",
+		"enabledControllers", mainConfig.EnabledControllers,
+		"enabledTasks", mainConfig.EnabledTasks,
+		"leaderElectionID", mainConfig.LeaderElectionID)
+
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
 	// due to its vulnerabilities. More specifically, disabling http/2 will
 	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
@@ -350,6 +359,7 @@ func main() {
 		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "nova-deschedulings-executor") {
+		setupLog.Info("enabling controller", "controller", "nova-deschedulings-executor")
 		executorConfig := conf.GetConfigOrDie[nova.DeschedulingsExecutorConfig]()
 		novaClient := nova.NewNovaClient()
 		novaClientConfig := conf.GetConfigOrDie[nova.NovaClientConfig]()
@@ -379,6 +389,7 @@ func main() {
 		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "manila-decisions-pipeline-controller") {
+		setupLog.Info("enabling controller", "controller", "manila-decisions-pipeline-controller")
 		controller := &manila.FilterWeigherPipelineController{
 			Monitor: filterWeigherPipelineMonitor,
 		}
@@ -398,6 +409,7 @@ func main() {
 		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "cinder-decisions-pipeline-controller") {
+		setupLog.Info("enabling controller", "controller", "cinder-decisions-pipeline-controller")
 		controller := &cinder.FilterWeigherPipelineController{
 			Monitor: filterWeigherPipelineMonitor,
 		}
@@ -417,6 +429,7 @@ func main() {
 		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "ironcore-decisions-pipeline-controller") {
+		setupLog.Info("enabling controller", "controller", "ironcore-decisions-pipeline-controller")
 		controller := &machines.FilterWeigherPipelineController{
 			Monitor: filterWeigherPipelineMonitor,
 		}
@@ -435,6 +448,7 @@ func main() {
 		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "pods-decisions-pipeline-controller") {
+		setupLog.Info("enabling controller", "controller", "pods-decisions-pipeline-controller")
 		controller := &pods.FilterWeigherPipelineController{
 			Monitor: filterWeigherPipelineMonitor,
 		}
@@ -453,6 +467,7 @@ func main() {
 		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "explanation-controller") {
+		setupLog.Info("enabling controller", "controller", "explanation-controller")
 		// Setup a controller which will reconcile the history and explanation for
 		// decision resources.
 		explanationControllerConfig := conf.GetConfigOrDie[explanation.ControllerConfig]()
@@ -466,6 +481,7 @@ func main() {
 		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "reservations-controller") {
+		setupLog.Info("enabling controller", "controller", "reservations-controller")
 		monitor := reservationscontroller.NewControllerMonitor(multiclusterClient)
 		metrics.Registry.MustRegister(&monitor)
 		reservationsControllerConfig := conf.GetConfigOrDie[reservationscontroller.Config]()
@@ -480,6 +496,7 @@ func main() {
 		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "datasource-controllers") {
+		setupLog.Info("enabling controller", "controller", "datasource-controllers")
 		monitor := datasources.NewMonitor()
 		metrics.Registry.MustRegister(&monitor)
 		if err := (&openstack.OpenStackDatasourceReconciler{
@@ -502,6 +519,7 @@ func main() {
 		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "knowledge-controllers") {
+		setupLog.Info("enabling controller", "controller", "knowledge-controllers")
 		monitor := extractor.NewMonitor()
 		metrics.Registry.MustRegister(&monitor)
 		if err := (&extractor.KnowledgeReconciler{
@@ -523,6 +541,7 @@ func main() {
 		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "kpis-controller") {
+		setupLog.Info("enabling controller", "controller", "kpis-controller")
 		kpisControllerConfig := conf.GetConfigOrDie[kpis.ControllerConfig]()
 		if err := (&kpis.Controller{
 			Client: multiclusterClient,
@@ -532,6 +551,93 @@ func main() {
 			os.Exit(1)
 		}
 	}
+	if slices.Contains(mainConfig.EnabledControllers, "failover-reservations-controller") {
+		setupLog.Info("enabling controller", "controller", "failover-reservations-controller")
+		failoverConfig := conf.GetConfigOrDie[failover.FailoverConfig]()
+
+		// Apply defaults for unset values
+		defaults := failover.DefaultConfig()
+		if failoverConfig.DatasourceName == "" {
+			failoverConfig.DatasourceName = defaults.DatasourceName
+		}
+		if failoverConfig.SchedulerURL == "" {
+			failoverConfig.SchedulerURL = defaults.SchedulerURL
+		}
+		if failoverConfig.ReconcileInterval == 0 {
+			failoverConfig.ReconcileInterval = defaults.ReconcileInterval
+		}
+		if failoverConfig.Creator == "" {
+			failoverConfig.Creator = defaults.Creator
+		}
+		if failoverConfig.FlavorFailoverRequirements == nil {
+			failoverConfig.FlavorFailoverRequirements = defaults.FlavorFailoverRequirements
+		}
+		if failoverConfig.RevalidationInterval == 0 {
+			failoverConfig.RevalidationInterval = defaults.RevalidationInterval
+		}
+
+		// DatasourceName is still required - check after applying defaults
+		if failoverConfig.DatasourceName == "" {
+			setupLog.Error(nil, "failover-reservations-controller requires datasourceName to be configured")
+			os.Exit(1)
+		}
+
+		// The scheduler client calls the nova external scheduler API to get placement decisions
+		schedulerClient := reservations.NewSchedulerClient(failoverConfig.SchedulerURL)
+
+		// Defer the initialization of PostgresReader until the manager starts
+		// because the cache is not ready during setup
+		if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error {
+			// Create PostgresReader from the configured Datasource CRD
+			// This runs after the cache is started
+			postgresReader, err := external.NewPostgresReader(ctx, multiclusterClient, failoverConfig.DatasourceName)
+			if err != nil {
+				setupLog.Error(err, "unable to create postgres reader for failover controller",
+					"datasourceName", failoverConfig.DatasourceName)
+				return err
+			}
+
+			// Create NovaReader and DBVMSource
+			novaReader := external.NewNovaReader(postgresReader)
+			vmSource := failover.NewDBVMSource(novaReader)
+
+			// Create the unified failover controller
+			// It handles both:
+			// 1. Watch-based per-reservation reconciliation (acknowledgment, validation)
+			// 2. Periodic bulk VM processing (creating/assigning reservations)
+			failoverController := failover.NewFailoverReservationController(
+				multiclusterClient,
+				vmSource,
+				failoverConfig,
+				schedulerClient,
+			)
+
+			// Set up the watch-based reconciler for per-reservation reconciliation
+			if err := failoverController.SetupWithManager(mgr, multiclusterClient); err != nil {
+				setupLog.Error(err, "unable to set up failover reservation controller")
+				return err
+			}
+
+			setupLog.Info("failover-reservations-controller starting",
+				"datasourceName", failoverConfig.DatasourceName,
+				"schedulerURL", failoverConfig.SchedulerURL,
+				"reconcileInterval", failoverConfig.ReconcileInterval,
+				"revalidationInterval", failoverConfig.RevalidationInterval)
+
+			// Start the controller's periodic reconciliation loop
+			return failoverController.Start(ctx)
+		})); err != nil {
+			setupLog.Error(err, "unable to add failover controller to manager")
+			os.Exit(1)
+		}
+		setupLog.Info("failover-reservations-controller registered",
+			"datasourceName", failoverConfig.DatasourceName,
+			"schedulerURL", failoverConfig.SchedulerURL,
+			"reconcileInterval", failoverConfig.ReconcileInterval,
+			"revalidationInterval", failoverConfig.RevalidationInterval,
+			"trustHypervisorLocation", failoverConfig.TrustHypervisorLocation,
+			"maxVMsToProcess", failoverConfig.MaxVMsToProcess)
+	}
 
 	// +kubebuilder:scaffold:builder