From e81131ca571643df75e194d0ce623173dfa65cda Mon Sep 17 00:00:00 2001 From: Alex Savanovich <40720931+savme@users.noreply.github.com> Date: Wed, 29 Jul 2026 20:14:54 +0300 Subject: [PATCH 1/4] feat: add /scale subresource to WorkloadDeployment --- api/v1alpha/workloaddeployment_types.go | 12 + api/v1alpha/zz_generated.deepcopy.go | 5 + ...ute.datumapis.com_workloaddeployments.yaml | 14 + config/components/controller_rbac/role.yaml | 9 + docs/api/instances.md | 375 +++++++++++++++- docs/api/workloaddeployments.md | 398 ++++++++++++++++- docs/api/workloads.md | 415 +++++++++++++++++- .../instancecontrol/instancecontrol.go | 1 + .../stateful/stateful_control.go | 5 +- .../stateful/stateful_control_test.go | 49 ++- internal/controller/teardown.go | 3 + internal/controller/workload_controller.go | 1 + .../controller/workload_controller_test.go | 48 ++ .../workloaddeployment_controller.go | 32 +- .../workloaddeployment_controller_test.go | 110 ++++- .../workloaddeployment_location_test.go | 1 + 16 files changed, 1431 insertions(+), 47 deletions(-) diff --git a/api/v1alpha/workloaddeployment_types.go b/api/v1alpha/workloaddeployment_types.go index c1ec37f0..0c6c6380 100644 --- a/api/v1alpha/workloaddeployment_types.go +++ b/api/v1alpha/workloaddeployment_types.go @@ -33,6 +33,12 @@ type WorkloadDeploymentSpec struct { // // +kubebuilder:validation:Required ScaleSettings HorizontalScaleSettings `json:"scaleSettings"` + + // Replicas is the current desired replica target for this deployment. When + // unset, the deployment reconciles to scaleSettings.minReplicas. + // + // +kubebuilder:validation:Optional + Replicas *int32 `json:"replicas,omitempty"` } // WorkloadDeploymentStatus defines the observed state of WorkloadDeployment @@ -65,6 +71,11 @@ type WorkloadDeploymentStatus struct { // The number of instances which are ready. ReadyReplicas int32 `json:"readyReplicas"` + // Selector is the label selector that identifies Pods backing this deployment. + // + // +kubebuilder:validation:Optional + Selector string `json:"selector,omitempty"` + // The most recent generation observed by the deployment controller. When // this matches metadata.generation, the controller has reconciled the // latest spec (e.g. a restart request). @@ -90,6 +101,7 @@ const ( // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.selector // +kubebuilder:metadata:annotations="discovery.miloapis.com/parent-contexts=Project" // WorkloadDeployment is the Schema for the workloaddeployments API diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index ca5b2830..65d1e1e9 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -984,6 +984,11 @@ func (in *WorkloadDeploymentSpec) DeepCopyInto(out *WorkloadDeploymentSpec) { out.WorkloadRef = in.WorkloadRef in.Template.DeepCopyInto(&out.Template) in.ScaleSettings.DeepCopyInto(&out.ScaleSettings) + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadDeploymentSpec. diff --git a/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml b/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml index f7f9fd22..75b77feb 100644 --- a/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml +++ b/config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml @@ -78,6 +78,12 @@ spec: placementName: description: The placement in the workload which is driving a deployment type: string + replicas: + description: |- + Replicas is the current desired replica target for this deployment. When + unset, the deployment reconciles to scaleSettings.minReplicas. + format: int32 + type: integer scaleSettings: description: Scale settings such as minimum and maximum replica counts. properties: @@ -1173,6 +1179,10 @@ spec: description: The number of instances created format: int32 type: integer + selector: + description: Selector is the label selector that identifies Pods backing + this deployment. + type: string suspended: description: |- Suspended, when true, requests that all instances managed by this deployment @@ -1197,4 +1207,8 @@ spec: served: true storage: true subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas status: {} diff --git a/config/components/controller_rbac/role.yaml b/config/components/controller_rbac/role.yaml index d3f92436..e0c6beb5 100644 --- a/config/components/controller_rbac/role.yaml +++ b/config/components/controller_rbac/role.yaml @@ -96,3 +96,12 @@ rules: - get - list - watch +- apiGroups: + - services.miloapis.com + resources: + - serviceconsumers + - services + verbs: + - get + - list + - watch diff --git a/docs/api/instances.md b/docs/api/instances.md index 580ab30c..b603ad99 100644 --- a/docs/api/instances.md +++ b/docs/api/instances.md @@ -53,14 +53,16 @@ Instance is the Schema for the instances API spec object - InstanceSpec defines the desired state of Instance
+ Spec defines the desired state of an Instance.
false status object - InstanceStatus defines the observed state of Instance
+ Status defines the current state of an Instance.
+
+ Default: map[conditions:[map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Programmed] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Available] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for controller reason:Pending status:Unknown type:Ready] map[lastTransitionTime:1970-01-01T00:00:00Z message:Waiting for quota evaluation reason:PendingEvaluation status:Unknown type:QuotaGranted]]]
false @@ -72,7 +74,7 @@ Instance is the Schema for the instances API -InstanceSpec defines the desired state of Instance +Spec defines the desired state of an Instance. @@ -97,11 +99,26 @@ InstanceSpec defines the desired state of Instance The runtime type of the instance, such as a container sandbox or a VM.
+ + + + + + + + + + @@ -536,6 +553,30 @@ used by the instance.
The name of the container.
+ + + + + + + + + + @@ -545,6 +586,18 @@ used by the instance.
so replicate the structure here too.
+ + + + + @@ -590,7 +643,8 @@ EnvVar represents an environment variable present in a Container. @@ -650,6 +704,14 @@ Source for the environment variable's value. Cannot be used if value is not empt spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.
+ + + + + @@ -751,6 +813,66 @@ spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podI
true
controllerobject + Controller contains settings driven by the controller managing the instance.
+ 		false
locationobject + The location which the instance has been scheduled to
+ 		false
volumes []object -
+ Volumes that must be available to attach to an instance's containers or +Virtual Machine.
 false
true
args[]string + Arguments to the entrypoint, overriding the image's CMD. Combined with +Command: when Command is also set the resulting invocation is +append(Command, Args...). When only Args is set it overrides CMD while +preserving the image's ENTRYPOINT. + +If neither Command nor Args is set, the image's own ENTRYPOINT and CMD +are used unchanged.
+ 		false
command[]string + Entrypoint array to run in the container image, overriding the image's +ENTRYPOINT. Each element is a separate token, not a shell command — to run a +shell command use: ["sh", "-c", "my command"]. + +If not provided, the container image's own ENTRYPOINT is used.
+ 		false
env []object false
envFrom[]object + List of sources to populate environment variables in the container. +The keys defined within a source must be a C_IDENTIFIER. All invalid +keys will be reported as an event when the container is starting. When a +key exists in multiple sources, the value associated with the last source +will take precedence. Values defined by an Env with a duplicate key will +take precedence.
+ 		false
ports []objectname string - Name of the environment variable. Must be a C_IDENTIFIER.
+ Name of the environment variable. +May consist of any printable ASCII characters except '='.
 true
false
fileKeyRefobject + FileKeyRef selects a key of the env file. +Requires the EnvFiles feature gate to be enabled.
+ 		false
resourceFieldRef object
+### Instance.spec.runtime.sandbox.containers[index].env[index].valueFrom.fileKeyRef +[↩ Parent](#instancespecruntimesandboxcontainersindexenvindexvaluefrom) + + + +FileKeyRef selects a key of the env file. +Requires the EnvFiles feature gate to be enabled. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The key within the env file. An invalid key will prevent the pod from starting. +The keys defined within a source may consist of any printable ASCII characters except '='. +During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters.
+ 		true
pathstring + The path within the volume from which to select the file. +Must be relative and may not contain the '..' path or start with '..'.
+ 		true
volumeNamestring + The name of the volume mount containing the env file.
+ 		true
optionalboolean + Specify whether the file or its key must be defined. If the file or key +does not exist, then the env var is not published. +If optional is set to true and the specified key does not exist, +the environment variable will not be set in the Pod's containers. + +If optional is set to false and the specified key does not exist, +an error will be returned during Pod creation.
+
+ Default: false
+ 		false
+ + ### Instance.spec.runtime.sandbox.containers[index].env[index].valueFrom.resourceFieldRef [↩ Parent](#instancespecruntimesandboxcontainersindexenvindexvaluefrom) @@ -840,6 +962,117 @@ More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/nam +### Instance.spec.runtime.sandbox.containers[index].envFrom[index] +[↩ Parent](#instancespecruntimesandboxcontainersindex) + + + +EnvFromSource represents a source for a set of ConfigMaps or Secrets to be +used as environment variables in a container. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
configMapRefobject + The ConfigMap to select from.
+ 		false
prefixstring + An optional identifier to prepend to each key in the referenced +ConfigMap or Secret. Must be a valid C_IDENTIFIER.
+ 		false
secretRefobject + The Secret to select from.
+ 		false
+ + +### Instance.spec.runtime.sandbox.containers[index].envFrom[index].configMapRef +[↩ Parent](#instancespecruntimesandboxcontainersindexenvfromindex) + + + +The ConfigMap to select from. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of the ConfigMap in the same namespace as the Workload.
+ 		true
optionalboolean + Specify whether the ConfigMap must be defined.
+ 		false
+ + +### Instance.spec.runtime.sandbox.containers[index].envFrom[index].secretRef +[↩ Parent](#instancespecruntimesandboxcontainersindexenvfromindex) + + + +The Secret to select from. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of the Secret in the same namespace as the Workload.
+ 		true
optionalboolean + Specify whether the Secret must be defined.
+ 		false
+ + ### Instance.spec.runtime.sandbox.containers[index].ports[index] [↩ Parent](#instancespecruntimesandboxcontainersindex) @@ -1108,6 +1341,102 @@ If not specified, this field defaults to TCP.
+### Instance.spec.controller +[↩ Parent](#instancespec) + + + +Controller contains settings driven by the controller managing the instance. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
templateHashstring + TemplateHash is the hash of the instance template applied for this instance.
+ 		true
schedulingGates[]object + SchedulingGates is a list of gates that must be satisfied before the +instance can be scheduled.
+ 		false
+ + +### Instance.spec.controller.schedulingGates[index] +[↩ Parent](#instancespeccontroller) + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + The name of the gate.
+ 		true
+ + +### Instance.spec.location +[↩ Parent](#instancespec) + + + +The location which the instance has been scheduled to + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of a datum location
+ 		true
namespacestring + Namespace for the datum location
+ 		true
+ + ### Instance.spec.volumes[index] [↩ Parent](#instancespec) @@ -1690,7 +2019,7 @@ mode, like fsGroup, and the result can be other mode bits set.
-InstanceStatus defines the observed state of Instance +Status defines the current state of an Instance. @@ -1709,6 +2038,13 @@ InstanceStatus defines the observed state of Instance Known condition types are: "Available", "Progressing"
+ + + + + @@ -1797,6 +2133,33 @@ with respect to the current state of the instance.
false
controllerobject + Controller contains status information about the controller managing the instance.
+ 		false
networkInterfaces []object
+### Instance.status.controller +[↩ Parent](#instancestatus) + + + +Controller contains status information about the controller managing the instance. + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
observedTemplateHashstring + ObservedTemplateHash is the hash of the instance template applied for this instance.
+ 		true
+ + ### Instance.status.networkInterfaces[index] [↩ Parent](#instancestatus) diff --git a/docs/api/workloaddeployments.md b/docs/api/workloaddeployments.md index 63b905d1..e3e5b079 100644 --- a/docs/api/workloaddeployments.md +++ b/docs/api/workloaddeployments.md @@ -118,6 +118,16 @@ WorkloadDeploymentSpec defines the desired state of WorkloadDeployment The workload that a deployment belongs to
true + + replicas + integer + + Replicas is the current desired replica target for this deployment. When +unset, the deployment reconciles to scaleSettings.minReplicas.
+
+ Format: int32
+ + false @@ -139,6 +149,16 @@ Scale settings such as minimum and maximum replica counts. + instanceManagementPolicy + string + + Controls how instances are managed during scale up and down, as well as +during maintenance events.
+
+ Default: OrderedReady
+ + true + minReplicas integer @@ -338,11 +358,26 @@ Describes the desired configuration of an instance The runtime type of the instance, such as a container sandbox or a VM.
true + + controller + object + + Controller contains settings driven by the controller managing the instance.
+ + false + + location + object + + The location which the instance has been scheduled to
+ + false volumes []object -
+ Volumes that must be available to attach to an instance's containers or +Virtual Machine.
false @@ -777,6 +812,30 @@ used by the instance.
The name of the container.
true + + args + []string + + Arguments to the entrypoint, overriding the image's CMD. Combined with +Command: when Command is also set the resulting invocation is +append(Command, Args...). When only Args is set it overrides CMD while +preserving the image's ENTRYPOINT. + +If neither Command nor Args is set, the image's own ENTRYPOINT and CMD +are used unchanged.
+ + false + + command + []string + + Entrypoint array to run in the container image, overriding the image's +ENTRYPOINT. Each element is a separate token, not a shell command — to run a +shell command use: ["sh", "-c", "my command"]. + +If not provided, the container image's own ENTRYPOINT is used.
+ + false env []object @@ -786,6 +845,18 @@ used by the instance.
so replicate the structure here too.
false + + envFrom + []object + + List of sources to populate environment variables in the container. +The keys defined within a source must be a C_IDENTIFIER. All invalid +keys will be reported as an event when the container is starting. When a +key exists in multiple sources, the value associated with the last source +will take precedence. Values defined by an Env with a duplicate key will +take precedence.
+ + false ports []object @@ -831,7 +902,8 @@ EnvVar represents an environment variable present in a Container. name string - Name of the environment variable. Must be a C_IDENTIFIER.
+ Name of the environment variable. +May consist of any printable ASCII characters except '='.
true @@ -891,6 +963,14 @@ Source for the environment variable's value. Cannot be used if value is not empt spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.
false + + fileKeyRef + object + + FileKeyRef selects a key of the env file. +Requires the EnvFiles feature gate to be enabled.
+ + false resourceFieldRef object @@ -992,6 +1072,66 @@ spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podI +### WorkloadDeployment.spec.template.spec.runtime.sandbox.containers[index].env[index].valueFrom.fileKeyRef +[↩ Parent](#workloaddeploymentspectemplatespecruntimesandboxcontainersindexenvindexvaluefrom) + + + +FileKeyRef selects a key of the env file. +Requires the EnvFiles feature gate to be enabled. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The key within the env file. An invalid key will prevent the pod from starting. +The keys defined within a source may consist of any printable ASCII characters except '='. +During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters.
+ 		true
pathstring + The path within the volume from which to select the file. +Must be relative and may not contain the '..' path or start with '..'.
+ 		true
volumeNamestring + The name of the volume mount containing the env file.
+ 		true
optionalboolean + Specify whether the file or its key must be defined. If the file or key +does not exist, then the env var is not published. +If optional is set to true and the specified key does not exist, +the environment variable will not be set in the Pod's containers. + +If optional is set to false and the specified key does not exist, +an error will be returned during Pod creation.
+
+ Default: false
+ 		false
+ + ### WorkloadDeployment.spec.template.spec.runtime.sandbox.containers[index].env[index].valueFrom.resourceFieldRef [↩ Parent](#workloaddeploymentspectemplatespecruntimesandboxcontainersindexenvindexvaluefrom) @@ -1081,6 +1221,117 @@ More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/nam +### WorkloadDeployment.spec.template.spec.runtime.sandbox.containers[index].envFrom[index] +[↩ Parent](#workloaddeploymentspectemplatespecruntimesandboxcontainersindex) + + + +EnvFromSource represents a source for a set of ConfigMaps or Secrets to be +used as environment variables in a container. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
configMapRefobject + The ConfigMap to select from.
+ 		false
prefixstring + An optional identifier to prepend to each key in the referenced +ConfigMap or Secret. Must be a valid C_IDENTIFIER.
+ 		false
secretRefobject + The Secret to select from.
+ 		false
+ + +### WorkloadDeployment.spec.template.spec.runtime.sandbox.containers[index].envFrom[index].configMapRef +[↩ Parent](#workloaddeploymentspectemplatespecruntimesandboxcontainersindexenvfromindex) + + + +The ConfigMap to select from. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of the ConfigMap in the same namespace as the Workload.
+ 		true
optionalboolean + Specify whether the ConfigMap must be defined.
+ 		false
+ + +### WorkloadDeployment.spec.template.spec.runtime.sandbox.containers[index].envFrom[index].secretRef +[↩ Parent](#workloaddeploymentspectemplatespecruntimesandboxcontainersindexenvfromindex) + + + +The Secret to select from. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of the Secret in the same namespace as the Workload.
+ 		true
optionalboolean + Specify whether the Secret must be defined.
+ 		false
+ + ### WorkloadDeployment.spec.template.spec.runtime.sandbox.containers[index].ports[index] [↩ Parent](#workloaddeploymentspectemplatespecruntimesandboxcontainersindex) @@ -1349,6 +1600,102 @@ If not specified, this field defaults to TCP.
+### WorkloadDeployment.spec.template.spec.controller +[↩ Parent](#workloaddeploymentspectemplatespec) + + + +Controller contains settings driven by the controller managing the instance. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
templateHashstring + TemplateHash is the hash of the instance template applied for this instance.
+ 		true
schedulingGates[]object + SchedulingGates is a list of gates that must be satisfied before the +instance can be scheduled.
+ 		false
+ + +### WorkloadDeployment.spec.template.spec.controller.schedulingGates[index] +[↩ Parent](#workloaddeploymentspectemplatespeccontroller) + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + The name of the gate.
+ 		true
+ + +### WorkloadDeployment.spec.template.spec.location +[↩ Parent](#workloaddeploymentspectemplatespec) + + + +The location which the instance has been scheduled to + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of a datum location
+ 		true
namespacestring + Namespace for the datum location
+ 		true
+ + ### WorkloadDeployment.spec.template.spec.volumes[index] [↩ Parent](#workloaddeploymentspectemplatespec) @@ -2035,8 +2382,8 @@ WorkloadDeploymentStatus defines the observed state of WorkloadDeployment currentReplicas integer - The number of instances created by a deployment and have the latest -deployment generation settings applied.
+ The number of instances which have the latest workload settings applied +and are programmed (a subset of UpdatedReplicas that are ready to serve).

Format: int32
@@ -2045,7 +2392,16 @@ deployment generation settings applied.
desiredReplicas integer - The desired number of instances to be managed by a deployment.
+ The desired number of instances
+
+ Format: int32
+ + true + + readyReplicas + integer + + The number of instances which are ready.

Format: int32
@@ -2054,7 +2410,19 @@ deployment generation settings applied.
replicas integer - The number of instances created by a deployment
+ The number of instances created
+
+ Format: int32
+ + true + + updatedReplicas + integer + + The number of instances updated to the latest template revision, i.e. +whose observed template hash matches the desired template, regardless of +readiness. Lags Replicas during a rolling update or restart, then catches +back up — making an in-progress roll observable.

Format: int32
@@ -2074,6 +2442,24 @@ Known condition types are: "Available", "Progressing"
The location which the deployment has been scheduled to
false + + observedGeneration + integer + + The most recent generation observed by the deployment controller. When +this matches metadata.generation, the controller has reconciled the +latest spec (e.g. a restart request).
+
+ Format: int64
+ + false + + selector + string + + Selector is the label selector that identifies Pods backing this deployment.
+ + false diff --git a/docs/api/workloads.md b/docs/api/workloads.md index c7742714..df10dcc0 100644 --- a/docs/api/workloads.md +++ b/docs/api/workloads.md @@ -160,6 +160,16 @@ Scale settings such as minimum and maximum replica counts. + instanceManagementPolicy + string + + Controls how instances are managed during scale up and down, as well as +during maintenance events.
+
+ Default: OrderedReady
+ + true + minReplicas integer @@ -359,11 +369,26 @@ Describes the desired configuration of an instance The runtime type of the instance, such as a container sandbox or a VM.
true + + controller + object + + Controller contains settings driven by the controller managing the instance.
+ + false + + location + object + + The location which the instance has been scheduled to
+ + false volumes []object -
+ Volumes that must be available to attach to an instance's containers or +Virtual Machine.
false @@ -798,6 +823,30 @@ used by the instance.
The name of the container.
true + + args + []string + + Arguments to the entrypoint, overriding the image's CMD. Combined with +Command: when Command is also set the resulting invocation is +append(Command, Args...). When only Args is set it overrides CMD while +preserving the image's ENTRYPOINT. + +If neither Command nor Args is set, the image's own ENTRYPOINT and CMD +are used unchanged.
+ + false + + command + []string + + Entrypoint array to run in the container image, overriding the image's +ENTRYPOINT. Each element is a separate token, not a shell command — to run a +shell command use: ["sh", "-c", "my command"]. + +If not provided, the container image's own ENTRYPOINT is used.
+ + false env []object @@ -807,6 +856,18 @@ used by the instance.
so replicate the structure here too.
false + + envFrom + []object + + List of sources to populate environment variables in the container. +The keys defined within a source must be a C_IDENTIFIER. All invalid +keys will be reported as an event when the container is starting. When a +key exists in multiple sources, the value associated with the last source +will take precedence. Values defined by an Env with a duplicate key will +take precedence.
+ + false ports []object @@ -852,7 +913,8 @@ EnvVar represents an environment variable present in a Container. name string - Name of the environment variable. Must be a C_IDENTIFIER.
+ Name of the environment variable. +May consist of any printable ASCII characters except '='.
true @@ -912,6 +974,14 @@ Source for the environment variable's value. Cannot be used if value is not empt spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.
false + + fileKeyRef + object + + FileKeyRef selects a key of the env file. +Requires the EnvFiles feature gate to be enabled.
+ + false resourceFieldRef object @@ -1013,6 +1083,66 @@ spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podI +### Workload.spec.template.spec.runtime.sandbox.containers[index].env[index].valueFrom.fileKeyRef +[↩ Parent](#workloadspectemplatespecruntimesandboxcontainersindexenvindexvaluefrom) + + + +FileKeyRef selects a key of the env file. +Requires the EnvFiles feature gate to be enabled. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The key within the env file. An invalid key will prevent the pod from starting. +The keys defined within a source may consist of any printable ASCII characters except '='. +During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters.
+ 		true
pathstring + The path within the volume from which to select the file. +Must be relative and may not contain the '..' path or start with '..'.
+ 		true
volumeNamestring + The name of the volume mount containing the env file.
+ 		true
optionalboolean + Specify whether the file or its key must be defined. If the file or key +does not exist, then the env var is not published. +If optional is set to true and the specified key does not exist, +the environment variable will not be set in the Pod's containers. + +If optional is set to false and the specified key does not exist, +an error will be returned during Pod creation.
+
+ Default: false
+ 		false
+ + ### Workload.spec.template.spec.runtime.sandbox.containers[index].env[index].valueFrom.resourceFieldRef [↩ Parent](#workloadspectemplatespecruntimesandboxcontainersindexenvindexvaluefrom) @@ -1102,6 +1232,117 @@ More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/nam +### Workload.spec.template.spec.runtime.sandbox.containers[index].envFrom[index] +[↩ Parent](#workloadspectemplatespecruntimesandboxcontainersindex) + + + +EnvFromSource represents a source for a set of ConfigMaps or Secrets to be +used as environment variables in a container. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
configMapRefobject + The ConfigMap to select from.
+ 		false
prefixstring + An optional identifier to prepend to each key in the referenced +ConfigMap or Secret. Must be a valid C_IDENTIFIER.
+ 		false
secretRefobject + The Secret to select from.
+ 		false
+ + +### Workload.spec.template.spec.runtime.sandbox.containers[index].envFrom[index].configMapRef +[↩ Parent](#workloadspectemplatespecruntimesandboxcontainersindexenvfromindex) + + + +The ConfigMap to select from. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of the ConfigMap in the same namespace as the Workload.
+ 		true
optionalboolean + Specify whether the ConfigMap must be defined.
+ 		false
+ + +### Workload.spec.template.spec.runtime.sandbox.containers[index].envFrom[index].secretRef +[↩ Parent](#workloadspectemplatespecruntimesandboxcontainersindexenvfromindex) + + + +The Secret to select from. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of the Secret in the same namespace as the Workload.
+ 		true
optionalboolean + Specify whether the Secret must be defined.
+ 		false
+ + ### Workload.spec.template.spec.runtime.sandbox.containers[index].ports[index] [↩ Parent](#workloadspectemplatespecruntimesandboxcontainersindex) @@ -1370,6 +1611,102 @@ If not specified, this field defaults to TCP.
+### Workload.spec.template.spec.controller +[↩ Parent](#workloadspectemplatespec) + + + +Controller contains settings driven by the controller managing the instance. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
templateHashstring + TemplateHash is the hash of the instance template applied for this instance.
+ 		true
schedulingGates[]object + SchedulingGates is a list of gates that must be satisfied before the +instance can be scheduled.
+ 		false
+ + +### Workload.spec.template.spec.controller.schedulingGates[index] +[↩ Parent](#workloadspectemplatespeccontroller) + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + The name of the gate.
+ 		true
+ + +### Workload.spec.template.spec.location +[↩ Parent](#workloadspectemplatespec) + + + +The location which the instance has been scheduled to + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of a datum location
+ 		true
namespacestring + Namespace for the datum location
+ 		true
+ + ### Workload.spec.template.spec.volumes[index] [↩ Parent](#workloadspectemplatespec) @@ -2022,8 +2359,17 @@ WorkloadStatus defines the observed state of Workload currentReplicas integer - The number of instances created by a placement and have the latest -workload generation settings applied.
+ The number of instances which have the latest workload settings applied +and are programmed (a subset of UpdatedReplicas that are ready to serve).
+
+ Format: int32
+ + true + + deployments + integer + + The number of deployments that currently exist

Format: int32
@@ -2032,7 +2378,16 @@ workload generation settings applied.
desiredReplicas integer - The desired number of instances to be managed by a placement.
+ The desired number of instances
+
+ Format: int32
+ + true + + readyReplicas + integer + + The number of instances which are ready.

Format: int32
@@ -2041,7 +2396,19 @@ workload generation settings applied.
replicas integer - The number of instances created by a placement
+ The number of instances that currently exist
+
+ Format: int32
+ + true + + updatedReplicas + integer + + The number of instances updated to the latest template revision (their +observed template hash matches the desired template), regardless of +readiness. Lags Replicas during a rolling update or restart, then catches +back up — making an in-progress roll observable.

Format: int32
@@ -2061,6 +2428,15 @@ Known condition types are: "Available", "Progressing"
The status of the workload gateway if configured.
false + + observedGeneration + integer + + The most recent generation observed by the workload controller.
+
+ Format: int64
+ + false placements []object @@ -2536,8 +2912,8 @@ RouteGroupKind indicates the group and kind of a Route resource. currentReplicas integer - The number of instances created by a placement and have the latest -workload generation settings applied.
+ The number of instances which have the latest workload settings applied +and are programmed (a subset of UpdatedReplicas that are ready to serve).

Format: int32
@@ -2546,7 +2922,7 @@ workload generation settings applied.
desiredReplicas integer - The desired number of instances to be managed by a placement.
+ The desired number of instances

Format: int32
@@ -2558,11 +2934,30 @@ workload generation settings applied.
The name of the placement
true + + readyReplicas + integer + + The number of instances which are ready.
+
+ Format: int32
+ + true replicas integer - The number of instances created by a placement
+ The number of instances that currently exist
+
+ Format: int32
+ + true + + updatedReplicas + integer + + The number of instances updated to the latest template revision, regardless +of readiness. Lags Replicas during a rolling update or restart.

Format: int32
diff --git a/internal/controller/instancecontrol/instancecontrol.go b/internal/controller/instancecontrol/instancecontrol.go index d2c83692..56ed2ebb 100644 --- a/internal/controller/instancecontrol/instancecontrol.go +++ b/internal/controller/instancecontrol/instancecontrol.go @@ -19,6 +19,7 @@ type Strategy interface { ctx context.Context, scheme *runtime.Scheme, deployment *v1alpha.WorkloadDeployment, + desiredReplicas int32, currentInstances []v1alpha.Instance, ) ([]Action, error) } diff --git a/internal/controller/instancecontrol/stateful/stateful_control.go b/internal/controller/instancecontrol/stateful/stateful_control.go index ee64cb60..c9be87d3 100644 --- a/internal/controller/instancecontrol/stateful/stateful_control.go +++ b/internal/controller/instancecontrol/stateful/stateful_control.go @@ -57,6 +57,7 @@ func (c *statefulControl) GetActions( ctx context.Context, scheme *runtime.Scheme, deployment *v1alpha.WorkloadDeployment, + desiredReplicas int32, currentInstances []v1alpha.Instance, ) ([]instancecontrol.Action, error) { instanceTemplateHash := instancecontrol.ComputeHash(deployment.Spec.Template) @@ -75,7 +76,7 @@ func (c *statefulControl) GetActions( // Instances that are desired to exist. We do not currently support the // concept of a partition, so will fill the entire slice. - desiredInstances := make([]*v1alpha.Instance, deployment.Spec.ScaleSettings.MinReplicas) + desiredInstances := make([]*v1alpha.Instance, desiredReplicas) for _, instance := range currentInstances { instanceIndex := getInstanceOrdinal(instance.Name) @@ -88,7 +89,7 @@ func (c *statefulControl) GetActions( // It's possible that the incoming currentInstances will have gaps in // instances, so fill them in. - for i := range deployment.Spec.ScaleSettings.MinReplicas { + for i := range desiredReplicas { if desiredInstances[i] == nil { desiredInstances[i] = &v1alpha.Instance{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/controller/instancecontrol/stateful/stateful_control_test.go b/internal/controller/instancecontrol/stateful/stateful_control_test.go index 63300be2..dc0557f2 100644 --- a/internal/controller/instancecontrol/stateful/stateful_control_test.go +++ b/internal/controller/instancecontrol/stateful/stateful_control_test.go @@ -36,7 +36,7 @@ func TestFreshDeployment(t *testing.T) { // No instances var currentInstances []v1alpha.Instance - actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) assert.NoError(t, err) assert.Len(t, actions, 2) @@ -50,6 +50,21 @@ func TestFreshDeployment(t *testing.T) { assert.True(t, actions[1].IsSkipped()) } +func TestFreshDeployment_UsesDesiredReplicasInput(t *testing.T) { + ctx := context.Background() + control := NewWithOptions(Options{}) + + deployment := getWorkloadDeployment("test-fresh-deploy", 1) + + actions, err := control.GetActions(ctx, scheme, deployment, 3, nil) + + assert.NoError(t, err) + assert.Len(t, actions, 3) + assert.Equal(t, "test-fresh-deploy-0", actions[0].Object.GetName()) + assert.Equal(t, "test-fresh-deploy-1", actions[1].Object.GetName()) + assert.Equal(t, "test-fresh-deploy-2", actions[2].Object.GetName()) +} + // TestFreshDeployment_InstanceHasOwnerReference verifies that Instances produced // by GetActions carry a controller owner reference to the WorkloadDeployment. // The simplified WD finalizer relies on Kubernetes GC to cascade Instance @@ -59,7 +74,7 @@ func TestFreshDeployment_InstanceHasOwnerReference(t *testing.T) { control := NewWithOptions(Options{}) deployment := getWorkloadDeployment("test-wd", 1) - actions, err := control.GetActions(ctx, scheme, deployment, nil) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, nil) require.NoError(t, err) require.Len(t, actions, 1) @@ -88,7 +103,7 @@ func TestUpdateWithAllReadyInstances(t *testing.T) { deployment.Spec.Template.Spec.Runtime.Sandbox.Containers[0].Image = "test-image-update" - actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) assert.NoError(t, err) assert.Len(t, actions, 2) @@ -118,7 +133,7 @@ func TestScaleUpWithNotReadyInstance(t *testing.T) { }) currentInstances = append(currentInstances, *notReadyInstance) - actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) assert.NoError(t, err) assert.Len(t, actions, 2) @@ -145,7 +160,7 @@ func TestScaleUpWithDeletingReadyInstance(t *testing.T) { deletingInstance.DeletionTimestamp = ptr.To(metav1.Now()) currentInstances = append(currentInstances, *deletingInstance) - actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) assert.NoError(t, err) assert.Len(t, actions, 2) @@ -169,7 +184,7 @@ func TestScaleDownWithAllReadyInstances(t *testing.T) { currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 0)) currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 1)) - actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) assert.NoError(t, err) assert.Len(t, actions, 1) @@ -189,7 +204,7 @@ func TestNetworkingEnabledAddsNetworkGate(t *testing.T) { deployment := getWorkloadDeployment("test-deploy-net-on", 1) var currentInstances []v1alpha.Instance - actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) assert.NoError(t, err) assert.Len(t, actions, 1) @@ -220,7 +235,7 @@ func TestNetworkingDisabledOmitsNetworkGate(t *testing.T) { deployment := getWorkloadDeployment("test-deploy-net-off", 1) var currentInstances []v1alpha.Instance - actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) assert.NoError(t, err) assert.Len(t, actions, 1) @@ -252,7 +267,7 @@ func TestInstanceLabels_FourNewLabelsStamped(t *testing.T) { deployment := getWorkloadDeployment("test-labels-deploy", 1) var currentInstances []v1alpha.Instance - actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) assert.NoError(t, err) assert.Len(t, actions, 1) @@ -289,7 +304,7 @@ func TestInstanceLabels_RefreshedOnRecreate(t *testing.T) { deployment.Spec.Template.Spec.Runtime.Sandbox.Containers[0].Image = "updated-image" // First reconcile: the drifted instance is deleted (recreate), not updated. - actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) assert.NoError(t, err) assert.Len(t, actions, 1) assert.Equal(t, instancecontrol.ActionTypeDelete, actions[0].ActionType()) @@ -297,7 +312,7 @@ func TestInstanceLabels_RefreshedOnRecreate(t *testing.T) { // Next reconcile, after the old instance has been fully deleted and is gone: // the empty slot is refilled by the create path, which stamps the labels. - actions, err = control.GetActions(ctx, scheme, deployment, nil) + actions, err = control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, nil) assert.NoError(t, err) assert.Len(t, actions, 1) assert.Equal(t, instancecontrol.ActionTypeCreate, actions[0].ActionType()) @@ -328,7 +343,7 @@ func TestInstanceLocation_SetWhenDeploymentStatusLocationPresent(t *testing.T) { } var currentInstances []v1alpha.Instance - actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) assert.NoError(t, err) assert.Len(t, actions, 1) @@ -353,7 +368,7 @@ func TestInstanceLocation_NilWhenDeploymentStatusLocationAbsent(t *testing.T) { // deployment.Status.Location is intentionally not set (nil) var currentInstances []v1alpha.Instance - actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) assert.NoError(t, err, "instance creation must succeed even when Status.Location is nil") assert.Len(t, actions, 1, "exactly one create action must be produced") @@ -393,7 +408,7 @@ func TestLabelBackfill_NotReadyMatchingHash(t *testing.T) { // Instance 1: needs to be created (nil in desiredInstances), so we only provide instance0. currentInstances := []v1alpha.Instance{*instance0} - actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) assert.NoError(t, err) @@ -465,7 +480,7 @@ func TestLabelBackfill_Idempotent(t *testing.T) { } currentInstances := []v1alpha.Instance{*instance} - actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) assert.NoError(t, err) @@ -490,7 +505,7 @@ func TestLabelBackfill_ReadyInstanceCorrected(t *testing.T) { delete(instance.Labels, v1alpha.CityCodeLabel) currentInstances := []v1alpha.Instance{*instance} - actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) assert.NoError(t, err) @@ -553,7 +568,7 @@ func TestLabelBackfill_DoesNotAffectRollingUpdate(t *testing.T) { deployment.Spec.Template.Spec.Runtime.Sandbox.Containers[0].Image = "rolling-update-image" currentInstances := []v1alpha.Instance{*instance0, *instance1} - actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + actions, err := control.GetActions(ctx, scheme, deployment, deployment.Spec.ScaleSettings.MinReplicas, currentInstances) assert.NoError(t, err) diff --git a/internal/controller/teardown.go b/internal/controller/teardown.go index 44df2cdd..9a287ee7 100644 --- a/internal/controller/teardown.go +++ b/internal/controller/teardown.go @@ -51,6 +51,9 @@ func NewComputeTeardown( } } +// +kubebuilder:rbac:groups=services.miloapis.com,resources=serviceconsumers,verbs=get;list;watch +// +kubebuilder:rbac:groups=services.miloapis.com,resources=services,verbs=get;list;watch + // TeardownConsumer implements consumer.Teardown. func (ct *ComputeTeardown) TeardownConsumer( ctx context.Context, diff --git a/internal/controller/workload_controller.go b/internal/controller/workload_controller.go index 861ed736..f97eecb2 100644 --- a/internal/controller/workload_controller.go +++ b/internal/controller/workload_controller.go @@ -490,6 +490,7 @@ func (r *WorkloadReconciler) getDeploymentsForWorkload( CityCode: cityCode, Template: workload.Spec.Template, ScaleSettings: placement.ScaleSettings, + Replicas: new(placement.ScaleSettings.MinReplicas), }, }) } diff --git a/internal/controller/workload_controller_test.go b/internal/controller/workload_controller_test.go index e8c01ae8..d8dbef80 100644 --- a/internal/controller/workload_controller_test.go +++ b/internal/controller/workload_controller_test.go @@ -8,12 +8,15 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" computev1alpha "go.datum.net/compute/api/v1alpha" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" ) // makeWorkload builds a Workload with the given generation for use in @@ -79,6 +82,51 @@ func runReconcileWorkloadStatus(t *testing.T, workload *computev1alpha.Workload, return cond } +func TestGetDeploymentsForWorkload_InitializesReplicas(t *testing.T) { + t.Parallel() + + workload := &computev1alpha.Workload{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-workload", + Namespace: testDefaultNamespace, + UID: types.UID("workload-uid"), + }, + Spec: computev1alpha.WorkloadSpec{ + Placements: []computev1alpha.WorkloadPlacement{ + { + Name: testDefaultPlacement, + CityCodes: []string{"DFW"}, + ScaleSettings: computev1alpha.HorizontalScaleSettings{ + MinReplicas: 2, + }, + }, + }, + }, + } + location := &networkingv1alpha.LocationBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "dfw"}, + Spec: networkingv1alpha.LocationBindingSpec{ + LocationRef: corev1.LocalObjectReference{Name: "dfw"}, + Topology: map[string]string{"topology.datum.net/city-code": "DFW"}, + }, + } + + s := newNetworkingScheme() + cl := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(location). + WithIndex(&computev1alpha.WorkloadDeployment{}, deploymentWorkloadUIDIndex, deploymentWorkloadUIDIndexFunc). + Build() + r := &WorkloadReconciler{} + + desired, orphaned, err := r.getDeploymentsForWorkload(context.Background(), cl, workload) + require.NoError(t, err) + require.Empty(t, orphaned) + require.Len(t, desired, 1) + require.NotNil(t, desired[0].Spec.Replicas) + assert.Equal(t, int32(2), *desired[0].Spec.Replicas) +} + // TestReconcileWorkloadStatus_AllDeploymentsSameReason verifies that when all // deployments share the same blocking reason, that reason is propagated to the // Workload Available condition with ObservedGeneration set correctly. diff --git a/internal/controller/workloaddeployment_controller.go b/internal/controller/workloaddeployment_controller.go index 059ce361..25155a99 100644 --- a/internal/controller/workloaddeployment_controller.go +++ b/internal/controller/workloaddeployment_controller.go @@ -11,6 +11,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -60,6 +61,21 @@ type WorkloadDeploymentReconciler struct { enableReferencedDataGate bool } +func effectiveDesiredReplicas(deployment *computev1alpha.WorkloadDeployment) int32 { + if !deployment.DeletionTimestamp.IsZero() { + return 0 + } + if deployment.Spec.Replicas != nil { + return *deployment.Spec.Replicas + } + return deployment.Spec.ScaleSettings.MinReplicas +} + +func workloadDeploymentPodSelector(deployment *computev1alpha.WorkloadDeployment) string { + set := labels.Set{computev1alpha.WorkloadDeploymentUIDLabel: string(deployment.GetUID())} + return set.AsSelector().String() +} + // +kubebuilder:rbac:groups=compute.datumapis.com,resources=workloaddeployments,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=compute.datumapis.com,resources=workloaddeployments/status,verbs=get;update;patch // +kubebuilder:rbac:groups=compute.datumapis.com,resources=workloaddeployments/finalizers,verbs=update @@ -104,6 +120,14 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco if !deployment.DeletionTimestamp.IsZero() { return ctrl.Result{}, nil } + if deployment.Spec.Replicas == nil { + base := deployment.DeepCopy() + deployment.Spec.Replicas = new(deployment.Spec.ScaleSettings.MinReplicas) + if err := cl.GetClient().Patch(ctx, &deployment, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, fmt.Errorf("failed initializing deployment replicas: %w", err) + } + return ctrl.Result{Requeue: true}, nil + } logger.Info("reconciling deployment") defer logger.Info("reconcile complete") @@ -116,6 +140,7 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco listOpts := client.MatchingLabels{ computev1alpha.WorkloadDeploymentUIDLabel: string(deployment.GetUID()), } + desiredReplicas := effectiveDesiredReplicas(&deployment) var instances computev1alpha.InstanceList if err := cl.GetClient().List(ctx, &instances, listOpts); err != nil { @@ -127,7 +152,7 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco EnableReferencedDataGate: r.enableReferencedDataGate, }) - actions, err := instanceControl.GetActions(ctx, cl.GetScheme(), &deployment, instances.Items) + actions, err := instanceControl.GetActions(ctx, cl.GetScheme(), &deployment, desiredReplicas, instances.Items) if err != nil { return ctrl.Result{}, fmt.Errorf("failed getting instance control actions: %w", err) } @@ -177,10 +202,6 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco // their gates removed. replicas := len(instances.Items) - desiredReplicas := deployment.Spec.ScaleSettings.MinReplicas - if dt := deployment.DeletionTimestamp; !dt.IsZero() { - desiredReplicas = 0 - } currentReplicas, updatedReplicas, readyReplicas, quotaBlockedReplicas, referencedDataBlockedReplicas, err := r.reconcileInstanceGates(ctx, cl.GetClient(), &deployment, instances.Items, networkReady) if err != nil { @@ -192,6 +213,7 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco deployment.Status.UpdatedReplicas = int32(updatedReplicas) deployment.Status.DesiredReplicas = desiredReplicas deployment.Status.ReadyReplicas = int32(readyReplicas) + deployment.Status.Selector = workloadDeploymentPodSelector(&deployment) deployment.Status.ObservedGeneration = deployment.Generation switch { diff --git a/internal/controller/workloaddeployment_controller_test.go b/internal/controller/workloaddeployment_controller_test.go index 45dfbea6..fb877c05 100644 --- a/internal/controller/workloaddeployment_controller_test.go +++ b/internal/controller/workloaddeployment_controller_test.go @@ -55,6 +55,7 @@ func wdControllerTestDeployment(minReplicas int32) *computev1alpha.WorkloadDeplo CityCode: wdControllerTestCityCode, PlacementName: testDefaultPlacement, WorkloadRef: computev1alpha.WorkloadReference{Name: wdControllerTestWorkload}, + Replicas: new(minReplicas), ScaleSettings: computev1alpha.HorizontalScaleSettings{ MinReplicas: minReplicas, // Always present in production: the API server defaults the policy @@ -374,6 +375,62 @@ func TestReconcileInstanceGates_ClearsNetworkSchedulingGate(t *testing.T) { }) } +func TestEffectiveDesiredReplicas(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + mutate func(*computev1alpha.WorkloadDeployment) + wantCount int32 + }{ + { + name: "falls back to minReplicas when unset", + mutate: func(deployment *computev1alpha.WorkloadDeployment) { deployment.Spec.Replicas = nil }, + wantCount: 2, + }, + { + name: "uses spec replicas when set", + mutate: func(deployment *computev1alpha.WorkloadDeployment) { + deployment.Spec.Replicas = new(int32(5)) + }, + wantCount: 5, + }, + { + name: "deletion scales to zero", + mutate: func(deployment *computev1alpha.WorkloadDeployment) { + deployment.Spec.Replicas = new(int32(5)) + now := metav1.Now() + deployment.DeletionTimestamp = &now + }, + wantCount: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + deployment := wdControllerTestDeployment(2) + if tt.mutate != nil { + tt.mutate(deployment) + } + + assert.Equal(t, tt.wantCount, effectiveDesiredReplicas(deployment)) + }) + } +} + +func TestWorkloadDeploymentPodSelector(t *testing.T) { + t.Parallel() + + deployment := wdControllerTestDeployment(1) + + assert.Equal(t, + "compute.datumapis.com/workload-deployment-uid=wd-uid-test", + workloadDeploymentPodSelector(deployment), + ) +} + // newTestWDReconciler builds a WorkloadDeploymentReconciler wired to a fake // project cluster with the controller finalizer pre-registered, mirroring // SetupWithManager. Networking is disabled so Reconcile treats the network as @@ -421,7 +478,7 @@ func TestWorkloadDeploymentReconcile_FinalizerAddRequeues(t *testing.T) { // Second reconcile (post-requeue) proceeds past the finalizer branch and // publishes status: ObservedGeneration tracks the deployment generation and - // DesiredReplicas reflects scale settings. + // DesiredReplicas reflects the effective desired count. result, err = r.Reconcile(context.Background(), req) require.NoError(t, err) assert.Equal(t, ctrl.Result{}, result) @@ -429,10 +486,61 @@ func TestWorkloadDeploymentReconcile_FinalizerAddRequeues(t *testing.T) { require.NoError(t, cl.Get(context.Background(), req.NamespacedName, &updated)) assert.Equal(t, updated.Generation, updated.Status.ObservedGeneration) assert.Equal(t, int32(1), updated.Status.DesiredReplicas) + assert.Equal(t, workloadDeploymentPodSelector(&updated), updated.Status.Selector) assert.True(t, apimeta.IsStatusConditionTrue(updated.Status.Conditions, computev1alpha.WorkloadDeploymentReplicasReady), "no instances are quota-blocked, so ReplicasReady must be true") } +func TestWorkloadDeploymentReconcile_UsesSpecReplicas(t *testing.T) { + t.Parallel() + + deployment := wdControllerTestDeployment(1) + deployment.Spec.Replicas = new(int32(3)) + cl := newProjectFakeClient(deployment) + r := newTestWDReconciler(cl) + req := mcreconcile.Request{ + ClusterName: testCluster, + Request: ctrl.Request{ + NamespacedName: types.NamespacedName{Name: wdControllerTestName, Namespace: wdControllerTestNS}, + }, + } + + _, err := r.Reconcile(context.Background(), req) + require.NoError(t, err) + _, err = r.Reconcile(context.Background(), req) + require.NoError(t, err) + + var updated computev1alpha.WorkloadDeployment + require.NoError(t, cl.Get(context.Background(), req.NamespacedName, &updated)) + assert.Equal(t, int32(3), updated.Status.DesiredReplicas) + assert.Equal(t, workloadDeploymentPodSelector(&updated), updated.Status.Selector) +} + +func TestWorkloadDeploymentReconcile_InitializesReplicas(t *testing.T) { + t.Parallel() + + deployment := wdControllerTestDeployment(2) + deployment.Spec.Replicas = nil + deployment.Finalizers = []string{workloadControllerFinalizer} + cl := newProjectFakeClient(deployment) + r := newTestWDReconciler(cl) + req := mcreconcile.Request{ + ClusterName: testCluster, + Request: ctrl.Request{ + NamespacedName: types.NamespacedName{Name: wdControllerTestName, Namespace: wdControllerTestNS}, + }, + } + + result, err := r.Reconcile(context.Background(), req) + require.NoError(t, err) + assert.Equal(t, ctrl.Result{Requeue: true}, result) + + var updated computev1alpha.WorkloadDeployment + require.NoError(t, cl.Get(context.Background(), req.NamespacedName, &updated)) + require.NotNil(t, updated.Spec.Replicas) + assert.Equal(t, int32(2), *updated.Spec.Replicas) +} + // ─── wdRefDataCondChanged tests ─────────────────────────────────────────────── // TestWdRefDataCondChanged_BothNil verifies that two nil conditions are treated diff --git a/internal/controller/workloaddeployment_location_test.go b/internal/controller/workloaddeployment_location_test.go index dd41381a..d8b6aa8e 100644 --- a/internal/controller/workloaddeployment_location_test.go +++ b/internal/controller/workloaddeployment_location_test.go @@ -174,6 +174,7 @@ func TestWorkloadDeploymentReconcile_NoMatchingLocation_SetsCondition(t *testing Spec: computev1alpha.WorkloadDeploymentSpec{ CityCode: locTestCityCode, WorkloadRef: computev1alpha.WorkloadReference{Name: "location-test-workload"}, + Replicas: new(int32(1)), ScaleSettings: computev1alpha.HorizontalScaleSettings{ MinReplicas: 1, // Production deployments always carry the kubebuilder-defaulted From 50b063946ed8fb361c68b4b8e270811436849221 Mon Sep 17 00:00:00 2001 From: Alex Savanovich <40720931+savme@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:47:45 +0200 Subject: [PATCH 2/4] feat: add HPA controller for workload deployments --- cmd/main.go | 33 +- config/components/controller_rbac/role.yaml | 12 + internal/controller/teardown.go | 12 +- internal/controller/testing_helpers_test.go | 2 + .../workloaddeployment_hpa_controller.go | 206 ++++++++++++ .../workloaddeployment_hpa_controller_test.go | 298 ++++++++++++++++++ 6 files changed, 556 insertions(+), 7 deletions(-) create mode 100644 internal/controller/workloaddeployment_hpa_controller.go create mode 100644 internal/controller/workloaddeployment_hpa_controller_test.go diff --git a/cmd/main.go b/cmd/main.go index 24c8bb74..ab19da30 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -16,6 +16,7 @@ import ( // to ensure that exec-entrypoint and run can make use of them. "github.com/KimMachineGun/automemlimit/memlimit" "golang.org/x/sync/errgroup" + autoscalingv2 "k8s.io/api/autoscaling/v2" _ "k8s.io/client-go/plugin/pkg/client/auth" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -31,6 +32,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -97,6 +99,19 @@ func init() { // +kubebuilder:scaffold:scheme } +func managedResourceGVKs(s *runtime.Scheme, objs ...client.Object) ([]schema.GroupVersionKind, error) { + gvks := make([]schema.GroupVersionKind, 0, len(objs)) + for _, obj := range objs { + gvk, err := apiutil.GVKForObject(obj, s) + if err != nil { + return nil, err + } + gvks = append(gvks, gvk) + } + + return gvks, nil +} + //nolint:gocyclo // main wires all controller paths; complexity is inherent to startup sequencing func main() { @@ -361,6 +376,11 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "WorkloadDeployment") os.Exit(1) } + + if err = (&controller.WorkloadDeploymentHPAReconciler{}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "WorkloadDeploymentHPA") + os.Exit(1) + } } if enableCellControllers { @@ -560,6 +580,15 @@ func initializeClusterDiscovery( return nil, nil, "", nil, fmt.Errorf("unable to create root client for service-catalog: %w", err) } + managedResources, err := managedResourceGVKs( + scheme, + &computev1alpha.Instance{}, + &autoscalingv2.HorizontalPodAutoscaler{}, + ) + if err != nil { + return nil, nil, "", nil, fmt.Errorf("unable to resolve managed resource GVKs: %w", err) + } + provider, err = consumerprovider.New(providerMgr, consumerprovider.Options{ RootClient: rootClient, Scheme: scheme, @@ -570,9 +599,7 @@ func initializeClusterDiscovery( o.Cache.DefaultTransform = cache.TransformStripManagedFields() }, }, - ManagedResources: []schema.GroupVersionKind{ - computev1alpha.GroupVersion.WithKind("Instance"), - }, + ManagedResources: managedResources, Teardowns: []consumerprovider.Teardown{ controller.NewComputeTeardown(quotaClientManager, federationClient, scheme), }, diff --git a/config/components/controller_rbac/role.yaml b/config/components/controller_rbac/role.yaml index e0c6beb5..b945fe9b 100644 --- a/config/components/controller_rbac/role.yaml +++ b/config/components/controller_rbac/role.yaml @@ -24,6 +24,18 @@ rules: verbs: - get - list +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - compute.datumapis.com resources: diff --git a/internal/controller/teardown.go b/internal/controller/teardown.go index 9a287ee7..18625d1d 100644 --- a/internal/controller/teardown.go +++ b/internal/controller/teardown.go @@ -18,10 +18,14 @@ import ( quotametrics "go.datum.net/compute/internal/quota" ) -// labelServiceName is the label key the consumer provider uses to scope -// deactivation cleanup. Every Instance the compute operator creates in a -// consumer project carries this label so disengage can target them by service. -const labelServiceName = "services.miloapis.com/service-name" +// labelServiceName and labelServiceValue are the label key and value the +// consumer provider uses to scope deactivation cleanup. Every resource the +// compute operator creates in a consumer project carries this label so disengage +// can target them by service. +const ( + labelServiceName = "services.miloapis.com/service-name" + labelServiceValue = "compute.datumapis.com" +) // ComputeTeardown implements consumer.Teardown. It is invoked after the // consumer provider has cancelled the per-cluster context and marked labeled diff --git a/internal/controller/testing_helpers_test.go b/internal/controller/testing_helpers_test.go index 17b24f47..52b3ca36 100644 --- a/internal/controller/testing_helpers_test.go +++ b/internal/controller/testing_helpers_test.go @@ -7,6 +7,7 @@ import ( "fmt" "sync" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/events" @@ -26,6 +27,7 @@ import ( // cluster (corev1 + compute). func newProjectScheme() *runtime.Scheme { s := runtime.NewScheme() + _ = autoscalingv2.AddToScheme(s) _ = corev1.AddToScheme(s) _ = computev1alpha.AddToScheme(s) return s diff --git a/internal/controller/workloaddeployment_hpa_controller.go b/internal/controller/workloaddeployment_hpa_controller.go new file mode 100644 index 00000000..6ba5e6cf --- /dev/null +++ b/internal/controller/workloaddeployment_hpa_controller.go @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mccontext "sigs.k8s.io/multicluster-runtime/pkg/context" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +// WorkloadDeploymentHPAReconciler manages cell-local HorizontalPodAutoscalers +// for WorkloadDeployments that opt into load-driven autoscaling. +type WorkloadDeploymentHPAReconciler struct { + mgr mcmanager.Manager +} + +// +kubebuilder:rbac:groups=compute.datumapis.com,resources=workloaddeployments,verbs=get;list;watch +// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete + +func (r *WorkloadDeploymentHPAReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + cl, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + + ctx = mccontext.WithCluster(ctx, req.ClusterName) + + var deployment computev1alpha.WorkloadDeployment + if err := cl.GetClient().Get(ctx, req.NamespacedName, &deployment); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if !deployment.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + + if !workloadDeploymentAutoscalingEnabled(&deployment) { + return ctrl.Result{}, deleteWorkloadDeploymentHPA(ctx, cl.GetClient(), &deployment) + } + + logger.Info("reconciling deployment HPA") + defer logger.Info("deployment HPA reconcile complete") + + metrics, err := workloadDeploymentHPAMetrics(&deployment) + if err != nil { + return ctrl.Result{}, err + } + + hpa := autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: deployment.Name, + Namespace: deployment.Namespace, + }, + } + + _, err = controllerutil.CreateOrPatch(ctx, cl.GetClient(), &hpa, func() error { + if hpa.UID != "" && !metav1.IsControlledBy(&hpa, &deployment) { + return fmt.Errorf("HPA %s/%s already exists and is not controlled by WorkloadDeployment %s/%s", + hpa.Namespace, hpa.Name, deployment.Namespace, deployment.Name) + } + + if err := controllerutil.SetControllerReference(&deployment, &hpa, cl.GetScheme()); err != nil { + return err + } + + hpa.Labels = workloadDeploymentHPALabels(&deployment) + + hpa.Spec = autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: computev1alpha.GroupVersion.String(), + Kind: "WorkloadDeployment", + Name: deployment.Name, + }, + MinReplicas: new(deployment.Spec.ScaleSettings.MinReplicas), + MaxReplicas: *deployment.Spec.ScaleSettings.MaxReplicas, + Metrics: metrics, + } + + return nil + }) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed reconciling deployment HPA: %w", err) + } + + return ctrl.Result{}, nil +} + +func workloadDeploymentAutoscalingEnabled(deployment *computev1alpha.WorkloadDeployment) bool { + return deployment.Spec.ScaleSettings.MaxReplicas != nil && len(deployment.Spec.ScaleSettings.Metrics) > 0 +} + +func deleteWorkloadDeploymentHPA(ctx context.Context, c client.Client, deployment *computev1alpha.WorkloadDeployment) error { + var hpa autoscalingv2.HorizontalPodAutoscaler + if err := c.Get(ctx, client.ObjectKey{Namespace: deployment.Namespace, Name: deployment.Name}, &hpa); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed fetching deployment HPA: %w", err) + } + if !metav1.IsControlledBy(&hpa, deployment) { + return nil + } + + if err := c.Delete(ctx, &hpa); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("failed deleting deployment HPA: %w", err) + } + + return nil +} + +func workloadDeploymentHPALabels(deployment *computev1alpha.WorkloadDeployment) map[string]string { + return map[string]string{ + labelServiceName: labelServiceValue, + computev1alpha.WorkloadDeploymentUIDLabel: string(deployment.UID), + computev1alpha.WorkloadDeploymentNameLabel: deployment.Name, + computev1alpha.WorkloadNameLabel: deployment.Spec.WorkloadRef.Name, + computev1alpha.PlacementNameLabel: deployment.Spec.PlacementName, + computev1alpha.CityCodeLabel: deployment.Spec.CityCode, + } +} + +func workloadDeploymentHPAMetrics(deployment *computev1alpha.WorkloadDeployment) ([]autoscalingv2.MetricSpec, error) { + metrics := make([]autoscalingv2.MetricSpec, 0, len(deployment.Spec.ScaleSettings.Metrics)) + for i, metric := range deployment.Spec.ScaleSettings.Metrics { + if metric.Resource == nil { + return nil, fmt.Errorf("metric %d has no resource source", i) + } + + if metric.Resource.Name != corev1.ResourceCPU && metric.Resource.Name != corev1.ResourceMemory { + return nil, fmt.Errorf("metric %d uses unsupported resource %q", i, metric.Resource.Name) + } + + target, err := workloadDeploymentHPAMetricTarget(metric.Resource.Target) + if err != nil { + return nil, fmt.Errorf("metric %d has invalid target: %w", i, err) + } + + metrics = append(metrics, autoscalingv2.MetricSpec{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: metric.Resource.Name, + Target: target, + }, + }) + } + + return metrics, nil +} + +func workloadDeploymentHPAMetricTarget(target computev1alpha.MetricTarget) (autoscalingv2.MetricTarget, error) { + setTargets := 0 + if target.Value != nil { + setTargets++ + } + if target.AverageValue != nil { + setTargets++ + } + if target.AverageUtilization != nil { + setTargets++ + } + if setTargets != 1 { + return autoscalingv2.MetricTarget{}, fmt.Errorf("exactly one target value must be set") + } + + if target.Value != nil { + value := target.Value.DeepCopy() + return autoscalingv2.MetricTarget{Type: autoscalingv2.ValueMetricType, Value: &value}, nil + } + + if target.AverageValue != nil { + averageValue := target.AverageValue.DeepCopy() + return autoscalingv2.MetricTarget{Type: autoscalingv2.AverageValueMetricType, AverageValue: &averageValue}, nil + } + + return autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: new(*target.AverageUtilization), + }, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *WorkloadDeploymentHPAReconciler) SetupWithManager(mgr mcmanager.Manager) error { + r.mgr = mgr + + return mcbuilder.ControllerManagedBy(mgr). + For(&computev1alpha.WorkloadDeployment{}, mcbuilder.WithEngageWithLocalCluster(false)). + Owns(&autoscalingv2.HorizontalPodAutoscaler{}, mcbuilder.WithPredicates(predicate.GenerationChangedPredicate{})). + Complete(r) +} diff --git a/internal/controller/workloaddeployment_hpa_controller_test.go b/internal/controller/workloaddeployment_hpa_controller_test.go new file mode 100644 index 00000000..df6cca4e --- /dev/null +++ b/internal/controller/workloaddeployment_hpa_controller_test.go @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +const ( + hpaTestCluster = "test-cluster" + hpaTestUID = "existing-hpa" +) + +func hpaTestDeployment() *computev1alpha.WorkloadDeployment { + averageUtilization := int32(75) + maxReplicas := int32(10) + return &computev1alpha.WorkloadDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: wdControllerTestName, + Namespace: wdControllerTestNS, + UID: wdControllerTestUID, + }, + Spec: computev1alpha.WorkloadDeploymentSpec{ + CityCode: wdControllerTestCityCode, + PlacementName: testDefaultPlacement, + WorkloadRef: computev1alpha.WorkloadReference{Name: wdControllerTestWorkload}, + ScaleSettings: computev1alpha.HorizontalScaleSettings{ + MinReplicas: 2, + MaxReplicas: new(maxReplicas), + Metrics: []computev1alpha.MetricSpec{ + { + Resource: &computev1alpha.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: computev1alpha.MetricTarget{ + AverageUtilization: new(averageUtilization), + }, + }, + }, + }, + InstanceManagementPolicy: computev1alpha.OrderedReadyInstanceManagementPolicyType, + }, + }, + } +} + +func newHPAReconciler(cl client.Client) *WorkloadDeploymentHPAReconciler { + return &WorkloadDeploymentHPAReconciler{ + mgr: newFakeMCManager(hpaTestCluster, newFakeCluster(cl)), + } +} + +func reconcileHPA(t *testing.T, r *WorkloadDeploymentHPAReconciler, deployment *computev1alpha.WorkloadDeployment) { + t.Helper() + _, err := r.Reconcile(context.Background(), mcreconcile.Request{ + ClusterName: multicluster.ClusterName(hpaTestCluster), + Request: reconcile.Request{NamespacedName: types.NamespacedName{Namespace: deployment.Namespace, Name: deployment.Name}}, + }) + require.NoError(t, err) +} + +func TestWorkloadDeploymentHPAReconciler_CreatesHPA(t *testing.T) { + t.Parallel() + + deployment := hpaTestDeployment() + cl := newProjectFakeClient(deployment) + reconcileHPA(t, newHPAReconciler(cl), deployment) + + var hpa autoscalingv2.HorizontalPodAutoscaler + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(deployment), &hpa)) + + assert.Equal(t, computev1alpha.GroupVersion.String(), hpa.Spec.ScaleTargetRef.APIVersion) + assert.Equal(t, "WorkloadDeployment", hpa.Spec.ScaleTargetRef.Kind) + assert.Equal(t, deployment.Name, hpa.Spec.ScaleTargetRef.Name) + require.NotNil(t, hpa.Spec.MinReplicas) + assert.Equal(t, int32(2), *hpa.Spec.MinReplicas) + assert.Equal(t, int32(10), hpa.Spec.MaxReplicas) + + require.Len(t, hpa.Spec.Metrics, 1) + assert.Equal(t, autoscalingv2.ResourceMetricSourceType, hpa.Spec.Metrics[0].Type) + require.NotNil(t, hpa.Spec.Metrics[0].Resource) + assert.Equal(t, corev1.ResourceCPU, hpa.Spec.Metrics[0].Resource.Name) + assert.Equal(t, autoscalingv2.UtilizationMetricType, hpa.Spec.Metrics[0].Resource.Target.Type) + require.NotNil(t, hpa.Spec.Metrics[0].Resource.Target.AverageUtilization) + assert.Equal(t, int32(75), *hpa.Spec.Metrics[0].Resource.Target.AverageUtilization) + + assert.Equal(t, map[string]string{ + labelServiceName: labelServiceValue, + computev1alpha.WorkloadDeploymentUIDLabel: string(deployment.UID), + computev1alpha.WorkloadDeploymentNameLabel: deployment.Name, + computev1alpha.WorkloadNameLabel: deployment.Spec.WorkloadRef.Name, + computev1alpha.PlacementNameLabel: deployment.Spec.PlacementName, + computev1alpha.CityCodeLabel: deployment.Spec.CityCode, + }, hpa.Labels) + require.Len(t, hpa.OwnerReferences, 1) + assert.Equal(t, deployment.Name, hpa.OwnerReferences[0].Name) + assert.True(t, *hpa.OwnerReferences[0].Controller) +} + +func TestWorkloadDeploymentHPAReconciler_UpdatesHPA(t *testing.T) { + t.Parallel() + + deployment := hpaTestDeployment() + existing := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: deployment.Name, + Namespace: deployment.Namespace, + UID: hpaTestUID, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(deployment, computev1alpha.GroupVersion.WithKind("WorkloadDeployment")), + }, + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{APIVersion: "apps/v1", Kind: "Deployment", Name: "old"}, + MinReplicas: new(int32(1)), + MaxReplicas: 1, + }, + } + cl := newProjectFakeClient(deployment, existing) + reconcileHPA(t, newHPAReconciler(cl), deployment) + + var hpa autoscalingv2.HorizontalPodAutoscaler + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(deployment), &hpa)) + assert.Equal(t, computev1alpha.GroupVersion.String(), hpa.Spec.ScaleTargetRef.APIVersion) + assert.Equal(t, "WorkloadDeployment", hpa.Spec.ScaleTargetRef.Kind) + assert.Equal(t, int32(10), hpa.Spec.MaxReplicas) +} + +func TestWorkloadDeploymentHPAReconciler_DoesNotAdoptUnownedHPA(t *testing.T) { + t.Parallel() + + deployment := hpaTestDeployment() + existing := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{Name: deployment.Name, Namespace: deployment.Namespace, UID: hpaTestUID}, + } + cl := newProjectFakeClient(deployment, existing) + r := newHPAReconciler(cl) + + _, err := r.Reconcile(context.Background(), mcreconcile.Request{ + ClusterName: multicluster.ClusterName(hpaTestCluster), + Request: reconcile.Request{NamespacedName: types.NamespacedName{Namespace: deployment.Namespace, Name: deployment.Name}}, + }) + require.Error(t, err) + + var hpa autoscalingv2.HorizontalPodAutoscaler + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(deployment), &hpa)) + assert.Empty(t, hpa.OwnerReferences) +} + +func TestWorkloadDeploymentHPAReconciler_DeletesHPAWhenAutoscalingDisabled(t *testing.T) { + t.Parallel() + + deployment := hpaTestDeployment() + deployment.Spec.ScaleSettings.MaxReplicas = nil + existing := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: deployment.Name, + Namespace: deployment.Namespace, + UID: hpaTestUID, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(deployment, computev1alpha.GroupVersion.WithKind("WorkloadDeployment")), + }, + }, + } + cl := newProjectFakeClient(deployment, existing) + reconcileHPA(t, newHPAReconciler(cl), deployment) + + var hpa autoscalingv2.HorizontalPodAutoscaler + err := cl.Get(context.Background(), client.ObjectKeyFromObject(deployment), &hpa) + assert.True(t, apierrors.IsNotFound(err)) +} + +func TestWorkloadDeploymentHPAReconciler_DoesNotDeleteUnownedHPA(t *testing.T) { + t.Parallel() + + deployment := hpaTestDeployment() + deployment.Spec.ScaleSettings.MaxReplicas = nil + existing := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{Name: deployment.Name, Namespace: deployment.Namespace, UID: hpaTestUID}, + } + cl := newProjectFakeClient(deployment, existing) + reconcileHPA(t, newHPAReconciler(cl), deployment) + + var hpa autoscalingv2.HorizontalPodAutoscaler + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(deployment), &hpa)) + assert.Empty(t, hpa.OwnerReferences) +} + +func TestWorkloadDeploymentHPAMetrics(t *testing.T) { + t.Parallel() + + value := resource.MustParse("100m") + averageValue := resource.MustParse("256Mi") + averageUtilization := int32(80) + + tests := []struct { + name string + metrics []computev1alpha.MetricSpec + wantType autoscalingv2.MetricTargetType + wantErr bool + wantMetric corev1.ResourceName + }{ + { + name: "value", + metrics: []computev1alpha.MetricSpec{{Resource: &computev1alpha.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: computev1alpha.MetricTarget{Value: &value}, + }}}, + wantType: autoscalingv2.ValueMetricType, + wantMetric: corev1.ResourceCPU, + }, + { + name: "average value", + metrics: []computev1alpha.MetricSpec{{Resource: &computev1alpha.ResourceMetricSource{ + Name: corev1.ResourceMemory, + Target: computev1alpha.MetricTarget{AverageValue: &averageValue}, + }}}, + wantType: autoscalingv2.AverageValueMetricType, + wantMetric: corev1.ResourceMemory, + }, + { + name: "average utilization", + metrics: []computev1alpha.MetricSpec{{Resource: &computev1alpha.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: computev1alpha.MetricTarget{AverageUtilization: &averageUtilization}, + }}}, + wantType: autoscalingv2.UtilizationMetricType, + wantMetric: corev1.ResourceCPU, + }, + { + name: "missing resource", + metrics: []computev1alpha.MetricSpec{{}}, + wantErr: true, + }, + { + name: "unsupported resource", + metrics: []computev1alpha.MetricSpec{{Resource: &computev1alpha.ResourceMetricSource{ + Name: corev1.ResourceEphemeralStorage, + Target: computev1alpha.MetricTarget{AverageUtilization: &averageUtilization}, + }}}, + wantErr: true, + }, + { + name: "multiple targets", + metrics: []computev1alpha.MetricSpec{{Resource: &computev1alpha.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: computev1alpha.MetricTarget{ + Value: &value, + AverageUtilization: &averageUtilization, + }, + }}}, + wantErr: true, + }, + { + name: "missing target", + metrics: []computev1alpha.MetricSpec{{Resource: &computev1alpha.ResourceMetricSource{ + Name: corev1.ResourceCPU, + }}}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + deployment := hpaTestDeployment() + deployment.Spec.ScaleSettings.Metrics = tt.metrics + + metrics, err := workloadDeploymentHPAMetrics(deployment) + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + require.Len(t, metrics, 1) + require.NotNil(t, metrics[0].Resource) + assert.Equal(t, tt.wantMetric, metrics[0].Resource.Name) + assert.Equal(t, tt.wantType, metrics[0].Resource.Target.Type) + }) + } +} From 77e6dcab229e4d03c212980b6adb900ae59e30bd Mon Sep 17 00:00:00 2001 From: Alex Savanovich <40720931+savme@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:10:20 +0200 Subject: [PATCH 3/4] fix: name WorkloadDeployment HPA controller --- internal/controller/workloaddeployment_hpa_controller.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/controller/workloaddeployment_hpa_controller.go b/internal/controller/workloaddeployment_hpa_controller.go index 6ba5e6cf..b1cd0aff 100644 --- a/internal/controller/workloaddeployment_hpa_controller.go +++ b/internal/controller/workloaddeployment_hpa_controller.go @@ -200,6 +200,7 @@ func (r *WorkloadDeploymentHPAReconciler) SetupWithManager(mgr mcmanager.Manager r.mgr = mgr return mcbuilder.ControllerManagedBy(mgr). + Named("workload-deployment-hpa"). For(&computev1alpha.WorkloadDeployment{}, mcbuilder.WithEngageWithLocalCluster(false)). Owns(&autoscalingv2.HorizontalPodAutoscaler{}, mcbuilder.WithPredicates(predicate.GenerationChangedPredicate{})). Complete(r) From fe2906629e835cedc9e5f2301fd9b046e1dbff7c Mon Sep 17 00:00:00 2001 From: Alex Savanovich <40720931+savme@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:56:36 +0200 Subject: [PATCH 4/4] test: add WorkloadDeployment HPA lifecycle chainsaw tests --- .github/workflows/test-e2e.yml | 13 +- .gitignore | 2 + Makefile | 22 +-- Taskfile.yaml | 132 ++++++++++++++++++ test/e2e/.chainsaw.yaml | 14 ++ test/e2e/e2e_test.go | 6 - .../hpa-lifecycle/chainsaw-test.yaml | 21 +++ .../workloaddeployment/hpa-lifecycle/hpa.yaml | 25 ++++ .../hpa-lifecycle/workloaddeployment.yaml | 32 +++++ .../hpa-stray/chainsaw-test.yaml | 24 ++++ .../hpa-stray/stray-hpa.yaml | 21 +++ .../hpa-stray/workloaddeployment.yaml | 32 +++++ .../hpa-update/chainsaw-test.yaml | 20 +++ .../hpa-update/hpa-initial.yaml | 15 ++ .../hpa-update/hpa-updated.yaml | 15 ++ .../hpa-update/workloaddeployment.yaml | 32 +++++ 16 files changed, 399 insertions(+), 27 deletions(-) create mode 100644 Taskfile.yaml create mode 100644 test/e2e/.chainsaw.yaml delete mode 100644 test/e2e/e2e_test.go create mode 100644 test/e2e/workloaddeployment/hpa-lifecycle/chainsaw-test.yaml create mode 100644 test/e2e/workloaddeployment/hpa-lifecycle/hpa.yaml create mode 100644 test/e2e/workloaddeployment/hpa-lifecycle/workloaddeployment.yaml create mode 100644 test/e2e/workloaddeployment/hpa-stray/chainsaw-test.yaml create mode 100644 test/e2e/workloaddeployment/hpa-stray/stray-hpa.yaml create mode 100644 test/e2e/workloaddeployment/hpa-stray/workloaddeployment.yaml create mode 100644 test/e2e/workloaddeployment/hpa-update/chainsaw-test.yaml create mode 100644 test/e2e/workloaddeployment/hpa-update/hpa-initial.yaml create mode 100644 test/e2e/workloaddeployment/hpa-update/hpa-updated.yaml create mode 100644 test/e2e/workloaddeployment/hpa-update/workloaddeployment.yaml diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index b2eda8c3..837e97fc 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -1,5 +1,8 @@ name: E2E Tests +env: + TASK_X_REMOTE_TASKFILES: 1 + on: push: pull_request: @@ -17,6 +20,9 @@ jobs: with: go-version-file: go.mod + - name: Setup Task + uses: arduino/setup-task@v2 + - name: Install the latest version of kind run: | curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 @@ -26,10 +32,5 @@ jobs: - name: Verify kind installation run: kind version - - name: Create kind cluster - run: kind create cluster - - name: Running Test e2e - run: | - go mod tidy - make test-e2e + run: make test-e2e diff --git a/.gitignore b/.gitignore index d5cc564d..3092d0a0 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ bin/ # Local e2e environment artefacts (Kind kubeconfigs, etc.) tmp/ +.test-infra/ +.task/ diff --git a/Makefile b/Makefile index 3d6a3e2e..6a5a68b5 100644 --- a/Makefile +++ b/Makefile @@ -64,22 +64,14 @@ vet: ## Run go vet against code. test: manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out -# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. -# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. -# Prometheus and CertManager are installed by default; skip with: -# - PROMETHEUS_INSTALL_SKIP=true -# - CERT_MANAGER_INSTALL_SKIP=true .PHONY: test-e2e -test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. - # @command -v kind >/dev/null 2>&1 || { \ - # echo "Kind is not installed. Please install Kind manually."; \ - # exit 1; \ - # } - # @kind get clusters | grep -q 'kind' || { \ - # echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ - # exit 1; \ - # } - # go test ./test/e2e/ -v -ginkgo.v +test-e2e: manifests generate fmt vet ## Run Chainsaw e2e tests in an isolated test-infra Kind cluster. + @set -e; \ + export TASK_X_REMOTE_TASKFILES=1; \ + cleanup() { task e2e:down || true; }; \ + trap cleanup EXIT; \ + task e2e:up; \ + task e2e .PHONY: lint lint: golangci-lint ## Run golangci-lint linter diff --git a/Taskfile.yaml b/Taskfile.yaml new file mode 100644 index 00000000..b53f9004 --- /dev/null +++ b/Taskfile.yaml @@ -0,0 +1,132 @@ +version: '3' + +dotenv: ['.env'] + +includes: + # Remote Taskfile from datum-cloud/test-infra. Requires + # TASK_X_REMOTE_TASKFILES=1. See: https://taskfile.dev/experiments/remote-taskfiles + test-infra: + taskfile: https://raw.githubusercontent.com/datum-cloud/test-infra/{{.TEST_INFRA_REPO_REF}}/Taskfile.yml + checksum: 1d057f69ef11b9fb7a94bbe7723f50f9508098cb2481f375f38ca5d55533d93e + vars: + CLUSTER_NAME: '{{.E2E_CLUSTER}}' + REPO_REF: '{{.TEST_INFRA_REPO_REF}}' + WAIT_TIMEOUT: '{{.WAIT_TIMEOUT}}' + +vars: + WAIT_TIMEOUT: '300s' + TOOL_DIR: '{{.USER_WORKING_DIR}}/bin' + IMG_NAME: 'compute' + IMG_TAG: 'e2e' + IMG: '{{.IMG_NAME}}:{{.IMG_TAG}}' + E2E_CLUSTER: 'test-infra' + E2E_KUBECONFIG: '.test-infra/kubeconfig' + TEST_INFRA_REPO_REF: 'v0.7.1' + CHAINSAW_VERSION: 'v0.2.15' + +tasks: + default: + cmds: + - task --list + silent: true + + dev:build: + desc: Build the compute manager container image + silent: true + cmds: + - docker build -t {{.IMG}} . + + e2e:up: + desc: Bring up a test-infra Kind cluster with the compute cell controller + silent: true + env: + KUBECONFIG: '{{.E2E_KUBECONFIG}}' + cmds: + - task: test-infra:cluster-up + vars: + CLUSTER_NAME: '{{.E2E_CLUSTER}}' + - task: dev:build + - task: test-infra:kind-load-image + vars: + CLUSTER_NAME: '{{.E2E_CLUSTER}}' + IMAGES: '{{.IMG}}' + - task test-infra:kubectl -- apply -k config/base/crd + - task test-infra:kubectl -- create namespace compute-system --dry-run=client -o yaml | task test-infra:kubectl -- apply -f - + - task test-infra:kubectl -- apply -k config/overlays/cell + - | + task test-infra:kubectl -- -n compute-system apply -f - <<'EOF' + apiVersion: v1 + kind: ConfigMap + metadata: + name: compute-config + data: + config.yaml: | + apiVersion: apiserver.config.datumapis.com/v1alpha1 + kind: WorkloadOperator + metricsServer: + bindAddress: "0" + discovery: {} + EOF + - task test-infra:kubectl -- -n compute-system set image deployment/compute-manager manager={{.IMG}} + - task test-infra:kubectl -- -n compute-system patch deployment compute-manager --type=json -p='[{"op":"replace","path":"/spec/template/spec/containers/0/imagePullPolicy","value":"IfNotPresent"}]' + - task test-infra:kubectl -- -n compute-system rollout status deployment/compute-manager --timeout=180s + + e2e: + desc: Run Chainsaw e2e tests against the isolated Kind cluster + silent: true + deps: + - e2e:preflight + - install:chainsaw + env: + KUBECONFIG: '{{.E2E_KUBECONFIG}}' + cmds: + - '{{.TOOL_DIR}}/chainsaw test test/e2e --config test/e2e/.chainsaw.yaml' + + install:chainsaw: + desc: Install Chainsaw locally + silent: true + cmds: + - | + mkdir -p {{.TOOL_DIR}} + if [ ! -f "{{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}}" ]; then + GOBIN={{.TOOL_DIR}} go install github.com/kyverno/chainsaw@{{.CHAINSAW_VERSION}} + mv {{.TOOL_DIR}}/chainsaw {{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}} + fi + ln -sf {{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}} {{.TOOL_DIR}}/chainsaw + status: + - test -f {{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}} + + e2e:preflight: + internal: true + silent: true + env: + KUBECONFIG: '{{.E2E_KUBECONFIG}}' + cmds: + - | + if [ ! -f "$KUBECONFIG" ]; then + echo "missing e2e kubeconfig at $KUBECONFIG" + echo "run: TASK_X_REMOTE_TASKFILES=1 task e2e:up" + exit 1 + fi + - | + context="$(kubectl config current-context 2>/dev/null || true)" + if [ "$context" != "kind-{{.E2E_CLUSTER}}" ]; then + echo "e2e kubeconfig points at $context, expected kind-{{.E2E_CLUSTER}}" + echo "run: TASK_X_REMOTE_TASKFILES=1 task e2e:up" + exit 1 + fi + - | + if ! kubectl get crd workloaddeployments.compute.datumapis.com >/dev/null 2>&1; then + echo "Compute CRDs are not installed in the e2e cluster" + echo "run: TASK_X_REMOTE_TASKFILES=1 task e2e:up" + exit 1 + fi + - kubectl -n compute-system rollout status deployment/compute-manager --timeout=30s + + e2e:down: + desc: Tear down the e2e test-infra Kind cluster + silent: true + cmds: + - task: test-infra:cluster-down + vars: + CLUSTER_NAME: '{{.E2E_CLUSTER}}' diff --git a/test/e2e/.chainsaw.yaml b/test/e2e/.chainsaw.yaml new file mode 100644 index 00000000..bd19c7a6 --- /dev/null +++ b/test/e2e/.chainsaw.yaml @@ -0,0 +1,14 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Configuration +metadata: + name: compute-e2e +spec: + timeouts: + apply: 60s + assert: 120s + cleanup: 120s + delete: 60s + error: 60s + exec: 60s + parallel: 1 + skipDelete: false diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go deleted file mode 100644 index 86de3c02..00000000 --- a/test/e2e/e2e_test.go +++ /dev/null @@ -1,6 +0,0 @@ -package e2e - -// This file exists due to the inability to bypass e2e generation when using -// kubebuilder to generate resources. -// -// See: https://github.com/kubernetes-sigs/kubebuilder/issues/4509 diff --git a/test/e2e/workloaddeployment/hpa-lifecycle/chainsaw-test.yaml b/test/e2e/workloaddeployment/hpa-lifecycle/chainsaw-test.yaml new file mode 100644 index 00000000..b247c4ee --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-lifecycle/chainsaw-test.yaml @@ -0,0 +1,21 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: workloaddeployment-hpa-lifecycle +spec: + steps: + - name: create autoscaled WorkloadDeployment + try: + - apply: + file: workloaddeployment.yaml + - assert: + file: hpa.yaml + - name: disable autoscaling + try: + - script: + content: | + kubectl -n "$NAMESPACE" patch workloaddeployment.compute.datumapis.com chainsaw-hpa --type=json \ + -p='[{"op":"remove","path":"/spec/scaleSettings/maxReplicas"},{"op":"remove","path":"/spec/scaleSettings/metrics"}]' + - script: + content: | + kubectl -n "$NAMESPACE" wait --for=delete horizontalpodautoscaler.autoscaling/chainsaw-hpa --timeout=60s diff --git a/test/e2e/workloaddeployment/hpa-lifecycle/hpa.yaml b/test/e2e/workloaddeployment/hpa-lifecycle/hpa.yaml new file mode 100644 index 00000000..91abd3eb --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-lifecycle/hpa.yaml @@ -0,0 +1,25 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: chainsaw-hpa + namespace: ($namespace) + labels: + services.miloapis.com/service-name: compute.datumapis.com + compute.datumapis.com/workload-deployment-name: chainsaw-hpa + compute.datumapis.com/workload-name: chainsaw-workload + compute.datumapis.com/placement-name: chainsaw-placement + compute.datumapis.com/city-code: DFW +spec: + scaleTargetRef: + apiVersion: compute.datumapis.com/v1alpha + kind: WorkloadDeployment + name: chainsaw-hpa + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 75 diff --git a/test/e2e/workloaddeployment/hpa-lifecycle/workloaddeployment.yaml b/test/e2e/workloaddeployment/hpa-lifecycle/workloaddeployment.yaml new file mode 100644 index 00000000..3813b88d --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-lifecycle/workloaddeployment.yaml @@ -0,0 +1,32 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: chainsaw-hpa + namespace: ($namespace) +spec: + workloadRef: + name: chainsaw-workload + uid: chainsaw-workload-uid + placementName: chainsaw-placement + cityCode: DFW + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + sandbox: + containers: + - name: app + image: registry.k8s.io/pause:3.10 + networkInterfaces: + - network: + name: default + scaleSettings: + minReplicas: 2 + maxReplicas: 10 + metrics: + - resource: + name: cpu + target: + averageUtilization: 75 + instanceManagementPolicy: OrderedReady diff --git a/test/e2e/workloaddeployment/hpa-stray/chainsaw-test.yaml b/test/e2e/workloaddeployment/hpa-stray/chainsaw-test.yaml new file mode 100644 index 00000000..d9fd6f6d --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-stray/chainsaw-test.yaml @@ -0,0 +1,24 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: workloaddeployment-hpa-stray +spec: + steps: + - name: create stray HPA + try: + - apply: + file: stray-hpa.yaml + - name: create matching autoscaled WorkloadDeployment + try: + - apply: + file: workloaddeployment.yaml + - assert: + file: stray-hpa.yaml + - name: disable autoscaling + try: + - script: + content: | + kubectl -n "$NAMESPACE" patch workloaddeployment.compute.datumapis.com chainsaw-stray --type=json \ + -p='[{"op":"remove","path":"/spec/scaleSettings/maxReplicas"},{"op":"remove","path":"/spec/scaleSettings/metrics"}]' + - assert: + file: stray-hpa.yaml diff --git a/test/e2e/workloaddeployment/hpa-stray/stray-hpa.yaml b/test/e2e/workloaddeployment/hpa-stray/stray-hpa.yaml new file mode 100644 index 00000000..ecb69f49 --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-stray/stray-hpa.yaml @@ -0,0 +1,21 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: chainsaw-stray + namespace: ($namespace) + labels: + e2e.compute.datumapis.com/stray: "true" +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: not-owned-by-compute + minReplicas: 1 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 diff --git a/test/e2e/workloaddeployment/hpa-stray/workloaddeployment.yaml b/test/e2e/workloaddeployment/hpa-stray/workloaddeployment.yaml new file mode 100644 index 00000000..3a855fa1 --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-stray/workloaddeployment.yaml @@ -0,0 +1,32 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: chainsaw-stray + namespace: ($namespace) +spec: + workloadRef: + name: chainsaw-workload + uid: chainsaw-workload-uid + placementName: chainsaw-placement + cityCode: DFW + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + sandbox: + containers: + - name: app + image: registry.k8s.io/pause:3.10 + networkInterfaces: + - network: + name: default + scaleSettings: + minReplicas: 2 + maxReplicas: 10 + metrics: + - resource: + name: cpu + target: + averageUtilization: 75 + instanceManagementPolicy: OrderedReady diff --git a/test/e2e/workloaddeployment/hpa-update/chainsaw-test.yaml b/test/e2e/workloaddeployment/hpa-update/chainsaw-test.yaml new file mode 100644 index 00000000..9f73939c --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-update/chainsaw-test.yaml @@ -0,0 +1,20 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: workloaddeployment-hpa-update +spec: + steps: + - name: create autoscaled WorkloadDeployment + try: + - apply: + file: workloaddeployment.yaml + - assert: + file: hpa-initial.yaml + - name: update autoscaling settings + try: + - script: + content: | + kubectl -n "$NAMESPACE" patch workloaddeployment.compute.datumapis.com chainsaw-hpa-update --type=merge \ + -p='{"spec":{"scaleSettings":{"minReplicas":3,"maxReplicas":12,"metrics":[{"resource":{"name":"memory","target":{"averageValue":"512Mi"}}}]}}}' + - assert: + file: hpa-updated.yaml diff --git a/test/e2e/workloaddeployment/hpa-update/hpa-initial.yaml b/test/e2e/workloaddeployment/hpa-update/hpa-initial.yaml new file mode 100644 index 00000000..8c07db42 --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-update/hpa-initial.yaml @@ -0,0 +1,15 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: chainsaw-hpa-update + namespace: ($namespace) +spec: + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 75 diff --git a/test/e2e/workloaddeployment/hpa-update/hpa-updated.yaml b/test/e2e/workloaddeployment/hpa-update/hpa-updated.yaml new file mode 100644 index 00000000..592f246f --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-update/hpa-updated.yaml @@ -0,0 +1,15 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: chainsaw-hpa-update + namespace: ($namespace) +spec: + minReplicas: 3 + maxReplicas: 12 + metrics: + - type: Resource + resource: + name: memory + target: + type: AverageValue + averageValue: 512Mi diff --git a/test/e2e/workloaddeployment/hpa-update/workloaddeployment.yaml b/test/e2e/workloaddeployment/hpa-update/workloaddeployment.yaml new file mode 100644 index 00000000..eea78f22 --- /dev/null +++ b/test/e2e/workloaddeployment/hpa-update/workloaddeployment.yaml @@ -0,0 +1,32 @@ +apiVersion: compute.datumapis.com/v1alpha +kind: WorkloadDeployment +metadata: + name: chainsaw-hpa-update + namespace: ($namespace) +spec: + workloadRef: + name: chainsaw-workload + uid: chainsaw-workload-uid + placementName: chainsaw-placement + cityCode: DFW + template: + spec: + runtime: + resources: + instanceType: datumcloud/d1-standard-2 + sandbox: + containers: + - name: app + image: registry.k8s.io/pause:3.10 + networkInterfaces: + - network: + name: default + scaleSettings: + minReplicas: 2 + maxReplicas: 10 + metrics: + - resource: + name: cpu + target: + averageUtilization: 75 + instanceManagementPolicy: OrderedReady