Add toggle for nova input host safety measure (#616)

With this change we can now toggle the `limitHostsToRequest`
functionality inside the external scheduler api for nova. This is needed
because we will completely ignore nova's input soon, and nova will
handle keyerrors for unknown proposed hosts appropriately. Thus, before
we will completely remove this safety measure from the code, we add this
toggle to turn it off.

Author: PhilippMatthes

cmd/main.go

@@ -322,7 +322,8 @@ func main() {
 			setupLog.Error(err, "unable to create controller", "controller", "nova FilterWeigherPipelineController")
 			os.Exit(1)
 		}
-		nova.NewAPI(filterWeigherController).Init(mux)
+		novaAPIConfig := conf.GetConfigOrDie[nova.HTTPAPIConfig]()
+		nova.NewAPI(novaAPIConfig, filterWeigherController).Init(mux)
 
 		// Initialize commitments API for LIQUID interface
 		commitmentsAPI := commitments.NewAPI(multiclusterClient)

helm/bundles/cortex-nova/values.yaml

@@ -138,6 +138,9 @@ cortex-scheduling-controllers:
       - failover-reservations-controller
     enabledTasks:
       - nova-history-cleanup-task
+    # If true, the external scheduler API will limit the list of hosts in its
+    # response to those included in the scheduling request.
+    novaLimitHostsToRequest: true
     # CommittedResourceFlavorGroupPipelines maps flavor group IDs to pipeline names for CR reservations
     # This allows different scheduling strategies per flavor group (e.g., HANA vs GP)
     committedResourceFlavorGroupPipelines:

internal/scheduling/nova/external_scheduler_api.go

@@ -34,15 +34,23 @@ type HTTPAPI interface {
 	Init(*http.ServeMux)
 }
 
+type HTTPAPIConfig struct {
+	// NovaLimitHostsToRequest, if true, will filter the Nova scheduler response
+	// to only include hosts that were in the original request.
+	NovaLimitHostsToRequest bool `json:"novaLimitHostsToRequest,omitempty"`
+}
+
 type httpAPI struct {
 	monitor  scheduling.APIMonitor
 	delegate HTTPAPIDelegate
+	config   HTTPAPIConfig
 }
 
-func NewAPI(delegate HTTPAPIDelegate) HTTPAPI {
+func NewAPI(config HTTPAPIConfig, delegate HTTPAPIDelegate) HTTPAPI {
 	return &httpAPI{
 		monitor:  scheduling.NewSchedulerMonitor(),
 		delegate: delegate,
+		config:   config,
 	}
 }
 
@@ -222,7 +230,11 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req
 		return
 	}
 	hosts := decision.Status.Result.OrderedHosts
-	hosts = limitHostsToRequest(requestData, hosts)
+	if httpAPI.config.NovaLimitHostsToRequest {
+		hosts = limitHostsToRequest(requestData, hosts)
+		slog.Info("limited hosts to request",
+			"hosts", hosts, "originalHosts", decision.Status.Result.OrderedHosts)
+	}
 	response := api.ExternalSchedulerResponse{Hosts: hosts}
 	w.Header().Set("Content-Type", "application/json")
 	if err = json.NewEncoder(w).Encode(response); err != nil {

internal/scheduling/nova/external_scheduler_api_test.go

@@ -33,7 +33,7 @@ func (m *mockHTTPAPIDelegate) ProcessNewDecisionFromAPI(ctx context.Context, dec
 func TestNewAPI(t *testing.T) {
 	delegate := &mockHTTPAPIDelegate{}
 
-	api := NewAPI(delegate)
+	api := NewAPI(HTTPAPIConfig{}, delegate)
 
 	if api == nil {
 		t.Fatal("NewAPI returned nil")
@@ -55,7 +55,7 @@ func TestNewAPI(t *testing.T) {
 
 func TestHTTPAPI_Init(t *testing.T) {
 	delegate := &mockHTTPAPIDelegate{}
-	api := NewAPI(delegate)
+	api := NewAPI(HTTPAPIConfig{}, delegate)
 
 	mux := http.NewServeMux()
 	api.Init(mux)
@@ -73,7 +73,7 @@ func TestHTTPAPI_Init(t *testing.T) {
 
 func TestHTTPAPI_canRunScheduler(t *testing.T) {
 	delegate := &mockHTTPAPIDelegate{}
-	api := NewAPI(delegate).(*httpAPI)
+	api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI)
 
 	tests := []struct {
 		name        string
@@ -276,7 +276,7 @@ func TestHTTPAPI_NovaExternalScheduler(t *testing.T) {
 				},
 			}
 
-			api := NewAPI(delegate).(*httpAPI)
+			api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI)
 
 			var body *strings.Reader
 			if tt.body != "" {
@@ -327,7 +327,7 @@ func TestHTTPAPI_NovaExternalScheduler_DecisionCreation(t *testing.T) {
 		},
 	}
 
-	api := NewAPI(delegate).(*httpAPI)
+	api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI)
 
 	requestData := novaapi.ExternalSchedulerRequest{
 		Spec: novaapi.NovaObject[novaapi.NovaSpec]{
@@ -508,7 +508,7 @@ func TestLimitHostsToRequest(t *testing.T) {
 
 func TestHTTPAPI_inferPipelineName(t *testing.T) {
 	delegate := &mockHTTPAPIDelegate{}
-	api := NewAPI(delegate).(*httpAPI)
+	api := NewAPI(HTTPAPIConfig{}, delegate).(*httpAPI)
 
 	tests := []struct {
 		name           string