Support multicluster resource routing

cmd/main.go

@@ -18,6 +18,7 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -264,10 +265,14 @@ func main() {
 		setupLog.Error(err, "unable to add home cluster")
 		os.Exit(1)
 	}
+	hvGVK := schema.GroupVersionKind{Group: "kvm.cloud.sap", Version: "v1", Kind: "Hypervisor"}
 	multiclusterClient := &multicluster.Client{
 		HomeCluster:    homeCluster,
 		HomeRestConfig: restConfig,
 		HomeScheme:     scheme,
+		ResourceRouters: map[schema.GroupVersionKind]multicluster.ResourceRouter{
+			hvGVK: multicluster.HypervisorResourceRouter{},
+		},
 	}
 	multiclusterClientConfig := conf.GetConfigOrDie[multicluster.ClientConfig]()
 	if err := multiclusterClient.InitFromConf(ctx, mgr, multiclusterClientConfig); err != nil {

docs/guides/multicluster/cortex-remote.yaml → docs/guides/multicluster/cortex-remote-az-a.yaml

@@ -1,6 +1,6 @@
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
-name: cortex-remote
+name: cortex-remote-az-a
 nodes:
   - role: control-plane
     extraPortMappings:

docs/guides/multicluster/cortex-remote-az-b.yaml

@@ -0,0 +1,27 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+name: cortex-remote-az-b
+nodes:
+  - role: control-plane
+    extraPortMappings:
+    - containerPort: 6443
+      hostPort: 8445
+    extraMounts:
+      - hostPath: /tmp/root-ca-home.pem
+        containerPath: /etc/ca-certificates/root-ca.pem
+    kubeadmConfigPatches:
+      - |
+        kind: ClusterConfiguration
+        apiServer:
+          extraArgs:
+            oidc-client-id: "https://host.docker.internal:8443" # = audience
+            oidc-issuer-url: "https://host.docker.internal:8443"
+            oidc-username-claim: sub
+            oidc-ca-file: /etc/ca-certificates/root-ca.pem
+          certSANs:
+            - api-proxy
+            - api-proxy.default.svc
+            - api-proxy.default.svc.cluster.local
+            - localhost
+            - 127.0.0.1
+            - host.docker.internal

docs/guides/multicluster/hypervisors-az-a.yaml

@@ -0,0 +1,13 @@
+apiVersion: kvm.cloud.sap/v1
+kind: Hypervisor
+metadata:
+  name: hypervisor-1-az-a
+  labels:
+    topology.kubernetes.io/zone: cortex-remote-az-a
+---
+apiVersion: kvm.cloud.sap/v1
+kind: Hypervisor
+metadata:
+  name: hypervisor-2-az-a
+  labels:
+    topology.kubernetes.io/zone: cortex-remote-az-a

docs/guides/multicluster/hypervisors-az-b.yaml

@@ -0,0 +1,13 @@
+apiVersion: kvm.cloud.sap/v1
+kind: Hypervisor
+metadata:
+  name: hypervisor-1-az-b
+  labels:
+    topology.kubernetes.io/zone: cortex-remote-az-b
+---
+apiVersion: kvm.cloud.sap/v1
+kind: Hypervisor
+metadata:
+  name: hypervisor-2-az-b
+  labels:
+    topology.kubernetes.io/zone: cortex-remote-az-b

docs/guides/multicluster/readme.md

@@ -1,38 +1,42 @@
 # Cortex Multi-Cluster Testing
 
-Cortex provides support for multi-cluster deployments, where a "home" cluster hosts the cortex pods and one or more "remote" clusters are used to persist CRDs. A typical use case for this would be to offload the etcd storage for Cortex CRDs to a remote cluster, reducing the resource usage on the home cluster.
+Cortex provides support for multi-cluster deployments, where a "home" cluster hosts the cortex pods and one or more "remote" clusters are used to persist CRDs. A typical use case for this would be to offload the etcd storage for Cortex CRDs to a remote cluster, reducing the resource usage on the home cluster. Similarly, another use case is to have multiple remote clusters that maintain all the compute workloads and expose resources that Cortex needs to access, such as the `Hypervisor` resource.
 
-This guide will walk you through setting up a multi-cluster Cortex deployment using [kind](https://kind.sigs.k8s.io/). We will create two kind clusters: `cortex-home` and `cortex-remote`. The `cortex-home` cluster will host the Cortex control plane, while the `cortex-remote` cluster will be used to store CRDs.
+This guide will walk you through setting up a multi-cluster Cortex deployment using [kind](https://kind.sigs.k8s.io/). We will create three kind clusters: `cortex-home`, `cortex-remote-az-a`, and `cortex-remote-az-b`. The `cortex-home` cluster will host the Cortex control plane, while the `cortex-remote-az-a` and `cortex-remote-az-b` clusters will be used to store hypervisor CRDs.
 
-To store its CRDs in the `cortex-remote` cluster, the `cortex-home` cluster needs to be able to authenticate to the `cortex-remote` cluster's API server. We will achieve this by configuring the `cortex-remote` cluster to trust the service account tokens issued by the `cortex-home` cluster. In this way, no external OIDC provider is needed, because the `cortex-home` cluster's own OIDC issuer for service accounts acts as the identity provider.
+To store its CRDs in the `cortex-remote-*` clusters, the `cortex-home` cluster needs to be able to authenticate to the `cortex-remote-*` clusters' API servers. We will achieve this by configuring the `cortex-remote-*` clusters to trust the service account tokens issued by the `cortex-home` cluster. In this way, no external OIDC provider is needed, because the `cortex-home` cluster's own OIDC issuer for service accounts acts as the identity provider.
 
 Here is a diagram illustrating the authentication flow:
 
 ```mermaid
 sequenceDiagram
     participant Home as cortex-home
-    participant Remote as cortex-remote
+    participant RemoteA as cortex-remote-az-a
+    participant RemoteB as cortex-remote-az-b
     Home->>Home: Service Account Token Issued
-    Home->>Remote: API Request with Token
-    Remote->>Remote: Token Verified Against Home's OIDC Issuer
-    Remote->>Home: API Response
+    Home->>RemoteA: API Request with Token
+    RemoteA->>RemoteA: Token Verified Against Home's OIDC Issuer
+    RemoteA->>Home: API Response
+    Home->>RemoteB: API Request with Token
+    RemoteB->>RemoteB: Token Verified Against Home's OIDC Issuer
+    RemoteB->>Home: API Response
 ```
 
 ## Home Cluster Setup
 
-First we set up the `cortex-home` cluster. The provided kind configuration file `cortex-home.yaml` sets up the cluster with the necessary port mappings to allow communication between the two clusters. `cortex-home` will expose its API server on port `8443`, which `cortex-remote` will use to verify service account tokens through `https://host.docker.internal:8443`.
+First we set up the `cortex-home` cluster. The provided kind configuration file `cortex-home.yaml` sets up the cluster with the necessary port mappings to allow communication between the three clusters. `cortex-home` will expose its API server on port `8443`, which `cortex-remote-az-a` and `cortex-remote-az-b` will use to verify service account tokens through `https://host.docker.internal:8443`.
 
 ```bash
 kind create cluster --config docs/guides/multicluster/cortex-home.yaml
 ```
 
-Next, we need to expose the OIDC issuer endpoint of the `cortex-home` cluster's API server to the `cortex-remote` cluster. We do this by creating a `ClusterRoleBinding` that grants the `system:service-account-issuer-discovery` role to the `kube-system` service account in the `cortex-home` cluster.
+Next, we need to expose the OIDC issuer endpoint of the `cortex-home` cluster's API server to the `cortex-remote-*` clusters. We do this by creating a `ClusterRoleBinding` that grants the `system:service-account-issuer-discovery` role to the `kube-system` service account in the `cortex-home` cluster.
 
 ```bash
 kubectl --context kind-cortex-home apply -f docs/guides/multicluster/cortex-home-crb.yaml
 ```
 
-To talk back to the `cortex-home` cluster's OIDC endpoint, the `cortex-remote` cluster needs to trust the root CA certificate used by the `cortex-home` cluster's API server. We can extract this certificate from the `extension-apiserver-authentication` config map in the `kube-system` namespace, and save it to a temporary file for later use.
+To talk back to the `cortex-home` cluster's OIDC endpoint, the `cortex-remote-*` clusters need to trust the root CA certificate used by the `cortex-home` cluster's API server. We can extract this certificate from the `extension-apiserver-authentication` config map in the `kube-system` namespace, and save it to a temporary file for later use.
 
 ```bash
 kubectl --context kind-cortex-home --namespace kube-system \
@@ -42,67 +46,87 @@ kubectl --context kind-cortex-home --namespace kube-system \
 
 ## Remote Cluster Setup
 
-With all the prerequisites in place, we can now set up the `cortex-remote` cluster. We create the cluster using the provided kind configuration file `cortex-remote.yaml`. This configuration will tell the `cortex-remote` cluster to trust the `cortex-home` cluster's API server as OIDC issuer for service account token verification. Also, the `cortex-remote` cluster will trust the root CA certificate we extracted earlier. The `cortex-remote` apiserver will be accessible at `https://host.docker.internal:8444`.
+With all the prerequisites in place, we can now set up the `cortex-remote-*` clusters. We create the clusters using the provided kind configuration files `cortex-remote-az-a.yaml` and `cortex-remote-az-b.yaml`. These configurations will tell the `cortex-remote-*` clusters to trust the `cortex-home` cluster's API server as OIDC issuer for service account token verification. Also, the `cortex-remote-*` clusters will trust the root CA certificate we extracted earlier. The `cortex-remote-*` apiservers will be accessible at `https://host.docker.internal:8444` and `https://host.docker.internal:8445`, respectively.
 
 ```bash
-kind create cluster --config docs/guides/multicluster/cortex-remote.yaml
+kind create cluster --config docs/guides/multicluster/cortex-remote-az-a.yaml
+kind create cluster --config docs/guides/multicluster/cortex-remote-az-b.yaml
 ```
 
-Next, we need to create a `ClusterRoleBinding` in the `cortex-remote` cluster that grants service accounts coming from the `cortex-home` cluster access to the appropriate resources. We do this by applying the provided `cortex-remote-crb.yaml` file.
+Next, we need to create a `ClusterRoleBinding` in the `cortex-remote-*` clusters that grants service accounts coming from the `cortex-home` cluster access to the appropriate resources. We do this by applying the provided `cortex-remote-crb.yaml` file.
 
 ```bash
-kubectl --context kind-cortex-remote apply -f docs/guides/multicluster/cortex-remote-crb.yaml
+kubectl --context kind-cortex-remote-az-a apply -f docs/guides/multicluster/cortex-remote-crb.yaml
+kubectl --context kind-cortex-remote-az-b apply -f docs/guides/multicluster/cortex-remote-crb.yaml
 ```
 
 ## Deploying Cortex
 
-Before we launch cortex make sure that the CRDs are installed in the `cortex-remote` cluster.
+Before we launch cortex make sure that the CRDs are installed in the `cortex-remote-*` clusters.
 
 ```bash
-kubectl config use-context kind-cortex-remote
+kubectl config use-context kind-cortex-remote-az-a
+helm install helm/bundles/cortex-crds --generate-name
+kubectl config use-context kind-cortex-remote-az-b
 helm install helm/bundles/cortex-crds --generate-name
 ```
 
-Also, we need to extract the root CA certificate used by the `cortex-remote` cluster's API server, so that we can configure the cortex pods in the `cortex-home` cluster to trust it.
+Also, we need to extract the root CA certificate used by the `cortex-remote-*` clusters' API servers, so that we can configure the cortex pods in the `cortex-home` cluster to trust them.
 
 ```bash
-kubectl --context kind-cortex-remote --namespace kube-system \
+kubectl --context kind-cortex-remote-az-a --namespace kube-system \
+  get configmap extension-apiserver-authentication \
+  -o jsonpath="{.data['client-ca-file']}" > /tmp/root-ca-remote-az-a.pem
+kubectl --context kind-cortex-remote-az-b --namespace kube-system \
   get configmap extension-apiserver-authentication \
-  -o jsonpath="{.data['client-ca-file']}" > /tmp/root-ca-remote.pem
+  -o jsonpath="{.data['client-ca-file']}" > /tmp/root-ca-remote-az-b.pem
 ```
 
-Now we can deploy cortex to the `cortex-home` cluster, configuring it to use the `cortex-remote` cluster for CRD storage. We create a temporary Helm values override file that specifies the API server URL and root CA certificate for the `cortex-remote` cluster. In this example, we are configuring the `decisions.cortex.cloud/v1alpha1` resource to be stored in the `cortex-remote` cluster.
+Now we can deploy cortex to the `cortex-home` cluster, configuring it to use the `cortex-remote-*` clusters for CRD storage. We create a temporary Helm values override file that specifies the API server URLs and root CA certificate for the `cortex-remote-*` clusters. In this example, we are configuring the `kvm.cloud.sap/v1/Hypervisor` resource to be stored in the `cortex-remote-*` clusters.
 
 ```bash
 export TILT_OVERRIDES_PATH=/tmp/cortex-values.yaml
 tee $TILT_OVERRIDES_PATH <<EOF
 global:
   conf:
-    apiServerOverrides:
-    - gvk: cortex.cloud/v1alpha1/DecisionList
-      host: https://host.docker.internal:8444
-      caCert: |
-$(cat /tmp/root-ca-remote.pem | sed 's/^/        /')
-    - gvk: cortex.cloud/v1alpha1/Decision
-      host: https://host.docker.internal:8444
-      caCert: |
-$(cat /tmp/root-ca-remote.pem | sed 's/^/        /')
+    apiservers:
+      remotes:
+      - host: https://host.docker.internal:8443
+        gvks:
+        - kvm.cloud.sap/v1/Hypervisor
+        - kvm.cloud.sap/v1/HypervisorList
+        labels:
+          az: cortex-remote-az-a
+        caCert: |
+          $(cat /tmp/root-ca-home.pem | sed 's/^/        /')
+      - host: https://host.docker.internal:8444
+        gvks:
+        - kvm.cloud.sap/v1/Hypervisor
+        - kvm.cloud.sap/v1/HypervisorList
+        labels:
+          az: cortex-remote-az-b
+        caCert: |
+          $(cat /tmp/root-ca-remote-az-b.pem | sed 's/^/        /')
 EOF
 ```
 
+Additionally, we will add some hypervisors cortex can reconcile on:
+```bash
+kubectl --context kind-cortex-remote-az-a apply -f docs/guides/multicluster/hypervisors-az-a.yaml
+kubectl --context kind-cortex-remote-az-b apply -f docs/guides/multicluster/hypervisors-az-b.yaml
+```
+
 Now we can start Cortex using Tilt, which will pick up the Helm values override file we just created.
 
 ```bash
 kubectl config use-context kind-cortex-home
-tilt up
+export ACTIVE_DEPLOYMENTS="nova" && tilt up
 ```
 
 ## Outcome
 
-With Cortex running in the `cortex-home` cluster and configured to use the `cortex-remote` cluster for CRD storage, we can verify that everything is working as expected with the following command:
+With Cortex running in the `cortex-home` cluster and configured to use the `cortex-remote-*` clusters for hypervisors, we can verify that everything is working as expected with the following command:
 
 ```bash
-kubectl --context kind-cortex-remote get decisions
+TODO
 ```
-
-This command should return the list of `Decision` resources stored in the `cortex-remote` cluster, confirming that Cortex is successfully using the remote cluster for CRD persistence.

helm/bundles/cortex-cinder/values.yaml

@@ -70,6 +70,24 @@ cortex: &cortex
   prometheus: {enable: false}
   conf: &cortexConf
     schedulingDomain: cinder
+    apiservers:
+      home:
+        gvks:
+          - cortex.cloud/v1alpha1/Decision
+          - cortex.cloud/v1alpha1/DecisionList
+          - cortex.cloud/v1alpha1/Descheduling
+          - cortex.cloud/v1alpha1/DeschedulingList
+          - cortex.cloud/v1alpha1/Pipeline
+          - cortex.cloud/v1alpha1/PipelineList
+          - cortex.cloud/v1alpha1/Knowledge
+          - cortex.cloud/v1alpha1/KnowledgeList
+          - cortex.cloud/v1alpha1/Datasource
+          - cortex.cloud/v1alpha1/DatasourceList
+          - cortex.cloud/v1alpha1/KPI
+          - cortex.cloud/v1alpha1/KPIList
+          - cortex.cloud/v1alpha1/Reservation
+          - cortex.cloud/v1alpha1/ReservationList
+          - v1/Secret
     keystoneSecretRef:
       name: cortex-cinder-openstack-keystone
       namespace: default

helm/bundles/cortex-ironcore/values.yaml

@@ -30,6 +30,30 @@ cortex:
   conf:
     # The operator will only touch CRs with this scheduling domain name.
     schedulingDomain: machines
+    apiservers:
+      home:
+        gvks:
+          - cortex.cloud/v1alpha1/Decision
+          - cortex.cloud/v1alpha1/DecisionList
+          - cortex.cloud/v1alpha1/Descheduling
+          - cortex.cloud/v1alpha1/DeschedulingList
+          - cortex.cloud/v1alpha1/Pipeline
+          - cortex.cloud/v1alpha1/PipelineList
+          - cortex.cloud/v1alpha1/Knowledge
+          - cortex.cloud/v1alpha1/KnowledgeList
+          - cortex.cloud/v1alpha1/Datasource
+          - cortex.cloud/v1alpha1/DatasourceList
+          - cortex.cloud/v1alpha1/KPI
+          - cortex.cloud/v1alpha1/KPIList
+          - cortex.cloud/v1alpha1/Reservation
+          - cortex.cloud/v1alpha1/ReservationList
+          - compute.ironcore.dev/v1alpha1/Machine
+          - compute.ironcore.dev/v1alpha1/MachineList
+          - compute.ironcore.dev/v1alpha1/MachinePool
+          - compute.ironcore.dev/v1alpha1/MachinePoolList
+          - compute.ironcore.dev/v1alpha1/MachineClass
+          - compute.ironcore.dev/v1alpha1/MachineClassList
+          - v1/Secret
     enabledControllers:
       - ironcore-decisions-pipeline-controller
       - explanation-controller

helm/bundles/cortex-manila/values.yaml

@@ -70,6 +70,24 @@ cortex: &cortex
   prometheus: {enable: false}
   conf: &cortexConf
     schedulingDomain: manila
+    apiservers:
+      home:
+        gvks:
+          - cortex.cloud/v1alpha1/Decision
+          - cortex.cloud/v1alpha1/DecisionList
+          - cortex.cloud/v1alpha1/Descheduling
+          - cortex.cloud/v1alpha1/DeschedulingList
+          - cortex.cloud/v1alpha1/Pipeline
+          - cortex.cloud/v1alpha1/PipelineList
+          - cortex.cloud/v1alpha1/Knowledge
+          - cortex.cloud/v1alpha1/KnowledgeList
+          - cortex.cloud/v1alpha1/Datasource
+          - cortex.cloud/v1alpha1/DatasourceList
+          - cortex.cloud/v1alpha1/KPI
+          - cortex.cloud/v1alpha1/KPIList
+          - cortex.cloud/v1alpha1/Reservation
+          - cortex.cloud/v1alpha1/ReservationList
+          - v1/Secret
     keystoneSecretRef:
       name: cortex-manila-openstack-keystone
       namespace: default

helm/bundles/cortex-nova/values.yaml

@@ -80,6 +80,26 @@ cortex: &cortex
   prometheus: {enable: false}
   conf: &cortexConf
     schedulingDomain: nova
+    apiservers:
+      home:
+        gvks:
+          - cortex.cloud/v1alpha1/Decision
+          - cortex.cloud/v1alpha1/DecisionList
+          - cortex.cloud/v1alpha1/Descheduling
+          - cortex.cloud/v1alpha1/DeschedulingList
+          - cortex.cloud/v1alpha1/Pipeline
+          - cortex.cloud/v1alpha1/PipelineList
+          - cortex.cloud/v1alpha1/Knowledge
+          - cortex.cloud/v1alpha1/KnowledgeList
+          - cortex.cloud/v1alpha1/Datasource
+          - cortex.cloud/v1alpha1/DatasourceList
+          - cortex.cloud/v1alpha1/KPI
+          - cortex.cloud/v1alpha1/KPIList
+          - cortex.cloud/v1alpha1/Reservation
+          - cortex.cloud/v1alpha1/ReservationList
+          - kvm.cloud.sap/v1/Hypervisor
+          - kvm.cloud.sap/v1/HypervisorList
+          - v1/Secret
     keystoneSecretRef:
       name: cortex-nova-openstack-keystone
       namespace: default

helm/bundles/cortex-pods/values.yaml

@@ -30,6 +30,27 @@ cortex:
   conf:
     # The operator will only touch CRs with this scheduling domain name.
     schedulingDomain: pods
+    apiservers:
+      home:
+        gvks:
+          - cortex.cloud/v1alpha1/Decision
+          - cortex.cloud/v1alpha1/DecisionList
+          - cortex.cloud/v1alpha1/Descheduling
+          - cortex.cloud/v1alpha1/DeschedulingList
+          - cortex.cloud/v1alpha1/Pipeline
+          - cortex.cloud/v1alpha1/PipelineList
+          - cortex.cloud/v1alpha1/Knowledge
+          - cortex.cloud/v1alpha1/KnowledgeList
+          - cortex.cloud/v1alpha1/Datasource
+          - cortex.cloud/v1alpha1/DatasourceList
+          - cortex.cloud/v1alpha1/KPI
+          - cortex.cloud/v1alpha1/KPIList
+          - cortex.cloud/v1alpha1/Reservation
+          - cortex.cloud/v1alpha1/ReservationList
+          - v1/Secret
+          - v1/Pod
+          - v1/NodeList
+          - v1/Binding
     enabledControllers:
       - pods-decisions-pipeline-controller
       - explanation-controller