test: add WorkloadDeployment HPA lifecycle chainsaw tests

.github/workflows/test-e2e.yml

@@ -1,5 +1,8 @@
 name: E2E Tests
 
+env:
+  TASK_X_REMOTE_TASKFILES: 1
+
 on:
   push:
   pull_request:
@@ -17,6 +20,9 @@ jobs:
         with:
           go-version-file: go.mod
 
+      - name: Setup Task
+        uses: arduino/setup-task@v2
+
       - name: Install the latest version of kind
         run: |
           curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64
@@ -26,10 +32,5 @@ jobs:
       - name: Verify kind installation
         run: kind version
 
-      - name: Create kind cluster
-        run: kind create cluster
-
       - name: Running Test e2e
-        run: |
-          go mod tidy
-          make test-e2e
+        run: make test-e2e

.gitignore

@@ -28,3 +28,5 @@ bin/
 
 # Local e2e environment artefacts (Kind kubeconfigs, etc.)
 tmp/
+.test-infra/
+.task/

Makefile

@@ -64,22 +64,14 @@ vet: ## Run go vet against code.
 test: manifests generate fmt vet envtest ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
 
-# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
-# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
-# Prometheus and CertManager are installed by default; skip with:
-# - PROMETHEUS_INSTALL_SKIP=true
-# - CERT_MANAGER_INSTALL_SKIP=true
 .PHONY: test-e2e
-test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
-	# @command -v kind >/dev/null 2>&1 || { \
-	# 	echo "Kind is not installed. Please install Kind manually."; \
-	# 	exit 1; \
-	# }
-	# @kind get clusters | grep -q 'kind' || { \
-	# 	echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \
-	# 	exit 1; \
-	# }
-	# go test ./test/e2e/ -v -ginkgo.v
+test-e2e: manifests generate fmt vet ## Run Chainsaw e2e tests in an isolated test-infra Kind cluster.
+	@set -e; \
+	export TASK_X_REMOTE_TASKFILES=1; \
+	cleanup() { task e2e:down || true; }; \
+	trap cleanup EXIT; \
+	task e2e:up; \
+	task e2e
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter

Taskfile.yaml

@@ -0,0 +1,132 @@
+version: '3'
+
+dotenv: ['.env']
+
+includes:
+  # Remote Taskfile from datum-cloud/test-infra. Requires
+  # TASK_X_REMOTE_TASKFILES=1. See: https://taskfile.dev/experiments/remote-taskfiles
+  test-infra:
+    taskfile: https://raw.githubusercontent.com/datum-cloud/test-infra/{{.TEST_INFRA_REPO_REF}}/Taskfile.yml
+    checksum: 1d057f69ef11b9fb7a94bbe7723f50f9508098cb2481f375f38ca5d55533d93e
+    vars:
+      CLUSTER_NAME: '{{.E2E_CLUSTER}}'
+      REPO_REF: '{{.TEST_INFRA_REPO_REF}}'
+      WAIT_TIMEOUT: '{{.WAIT_TIMEOUT}}'
+
+vars:
+  WAIT_TIMEOUT: '300s'
+  TOOL_DIR: '{{.USER_WORKING_DIR}}/bin'
+  IMG_NAME: 'compute'
+  IMG_TAG: 'e2e'
+  IMG: '{{.IMG_NAME}}:{{.IMG_TAG}}'
+  E2E_CLUSTER: 'test-infra'
+  E2E_KUBECONFIG: '.test-infra/kubeconfig'
+  TEST_INFRA_REPO_REF: 'v0.7.1'
+  CHAINSAW_VERSION: 'v0.2.15'
+
+tasks:
+  default:
+    cmds:
+      - task --list
+    silent: true
+
+  dev:build:
+    desc: Build the compute manager container image
+    silent: true
+    cmds:
+      - docker build -t {{.IMG}} .
+
+  e2e:up:
+    desc: Bring up a test-infra Kind cluster with the compute cell controller
+    silent: true
+    env:
+      KUBECONFIG: '{{.E2E_KUBECONFIG}}'
+    cmds:
+      - task: test-infra:cluster-up
+        vars:
+          CLUSTER_NAME: '{{.E2E_CLUSTER}}'
+      - task: dev:build
+      - task: test-infra:kind-load-image
+        vars:
+          CLUSTER_NAME: '{{.E2E_CLUSTER}}'
+          IMAGES: '{{.IMG}}'
+      - task test-infra:kubectl -- apply -k config/base/crd
+      - task test-infra:kubectl -- create namespace compute-system --dry-run=client -o yaml | task test-infra:kubectl -- apply -f -
+      - task test-infra:kubectl -- apply -k config/overlays/cell
+      - |
+        task test-infra:kubectl -- -n compute-system apply -f - <<'EOF'
+        apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: compute-config
+        data:
+          config.yaml: |
+            apiVersion: apiserver.config.datumapis.com/v1alpha1
+            kind: WorkloadOperator
+            metricsServer:
+              bindAddress: "0"
+            discovery: {}
+        EOF
+      - task test-infra:kubectl -- -n compute-system set image deployment/compute-manager manager={{.IMG}}
+      - task test-infra:kubectl -- -n compute-system patch deployment compute-manager --type=json -p='[{"op":"replace","path":"/spec/template/spec/containers/0/imagePullPolicy","value":"IfNotPresent"}]'
+      - task test-infra:kubectl -- -n compute-system rollout status deployment/compute-manager --timeout=180s
+
+  e2e:
+    desc: Run Chainsaw e2e tests against the isolated Kind cluster
+    silent: true
+    deps:
+      - e2e:preflight
+      - install:chainsaw
+    env:
+      KUBECONFIG: '{{.E2E_KUBECONFIG}}'
+    cmds:
+      - '{{.TOOL_DIR}}/chainsaw test test/e2e --config test/e2e/.chainsaw.yaml'
+
+  install:chainsaw:
+    desc: Install Chainsaw locally
+    silent: true
+    cmds:
+      - |
+        mkdir -p {{.TOOL_DIR}}
+        if [ ! -f "{{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}}" ]; then
+          GOBIN={{.TOOL_DIR}} go install github.com/kyverno/chainsaw@{{.CHAINSAW_VERSION}}
+          mv {{.TOOL_DIR}}/chainsaw {{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}}
+        fi
+        ln -sf {{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}} {{.TOOL_DIR}}/chainsaw
+    status:
+      - test -f {{.TOOL_DIR}}/chainsaw-{{.CHAINSAW_VERSION}}
+
+  e2e:preflight:
+    internal: true
+    silent: true
+    env:
+      KUBECONFIG: '{{.E2E_KUBECONFIG}}'
+    cmds:
+      - |
+        if [ ! -f "$KUBECONFIG" ]; then
+          echo "missing e2e kubeconfig at $KUBECONFIG"
+          echo "run: TASK_X_REMOTE_TASKFILES=1 task e2e:up"
+          exit 1
+        fi
+      - |
+        context="$(kubectl config current-context 2>/dev/null || true)"
+        if [ "$context" != "kind-{{.E2E_CLUSTER}}" ]; then
+          echo "e2e kubeconfig points at $context, expected kind-{{.E2E_CLUSTER}}"
+          echo "run: TASK_X_REMOTE_TASKFILES=1 task e2e:up"
+          exit 1
+        fi
+      - |
+        if ! kubectl get crd workloaddeployments.compute.datumapis.com >/dev/null 2>&1; then
+          echo "Compute CRDs are not installed in the e2e cluster"
+          echo "run: TASK_X_REMOTE_TASKFILES=1 task e2e:up"
+          exit 1
+        fi
+      - kubectl -n compute-system rollout status deployment/compute-manager --timeout=30s
+
+  e2e:down:
+    desc: Tear down the e2e test-infra Kind cluster
+    silent: true
+    cmds:
+      - task: test-infra:cluster-down
+        vars:
+          CLUSTER_NAME: '{{.E2E_CLUSTER}}'

api/v1alpha/workloaddeployment_types.go

@@ -33,6 +33,12 @@ type WorkloadDeploymentSpec struct {
 	//
 	// +kubebuilder:validation:Required
 	ScaleSettings HorizontalScaleSettings `json:"scaleSettings"`
+
+	// Replicas is the current desired replica target for this deployment. When
+	// unset, the deployment reconciles to scaleSettings.minReplicas.
+	//
+	// +kubebuilder:validation:Optional
+	Replicas *int32 `json:"replicas,omitempty"`
 }
 
 // WorkloadDeploymentStatus defines the observed state of WorkloadDeployment
@@ -65,6 +71,11 @@ type WorkloadDeploymentStatus struct {
 	// The number of instances which are ready.
 	ReadyReplicas int32 `json:"readyReplicas"`
 
+	// Selector is the label selector that identifies Pods backing this deployment.
+	//
+	// +kubebuilder:validation:Optional
+	Selector string `json:"selector,omitempty"`
+
 	// The most recent generation observed by the deployment controller. When
 	// this matches metadata.generation, the controller has reconciled the
 	// latest spec (e.g. a restart request).
@@ -90,6 +101,7 @@ const (
 
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
+// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.selector
 // +kubebuilder:metadata:annotations="discovery.miloapis.com/parent-contexts=Project"
 
 // WorkloadDeployment is the Schema for the workloaddeployments API

cmd/main.go

@@ -16,6 +16,7 @@ import (
 	// to ensure that exec-entrypoint and run can make use of them.
 	"github.com/KimMachineGun/automemlimit/memlimit"
 	"golang.org/x/sync/errgroup"
+	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -31,6 +32,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 	"sigs.k8s.io/controller-runtime/pkg/cluster"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -97,6 +99,19 @@ func init() {
 	// +kubebuilder:scaffold:scheme
 }
 
+func managedResourceGVKs(s *runtime.Scheme, objs ...client.Object) ([]schema.GroupVersionKind, error) {
+	gvks := make([]schema.GroupVersionKind, 0, len(objs))
+	for _, obj := range objs {
+		gvk, err := apiutil.GVKForObject(obj, s)
+		if err != nil {
+			return nil, err
+		}
+		gvks = append(gvks, gvk)
+	}
+
+	return gvks, nil
+}
+
 //nolint:gocyclo // main wires all controller paths; complexity is inherent to startup sequencing
 func main() {
 
@@ -361,6 +376,11 @@ func main() {
 			setupLog.Error(err, "unable to create controller", "controller", "WorkloadDeployment")
 			os.Exit(1)
 		}
+
+		if err = (&controller.WorkloadDeploymentHPAReconciler{}).SetupWithManager(mgr); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "WorkloadDeploymentHPA")
+			os.Exit(1)
+		}
 	}
 
 	if enableCellControllers {
@@ -560,6 +580,15 @@ func initializeClusterDiscovery(
 				return nil, nil, "", nil, fmt.Errorf("unable to create root client for service-catalog: %w", err)
 			}
 
+			managedResources, err := managedResourceGVKs(
+				scheme,
+				&computev1alpha.Instance{},
+				&autoscalingv2.HorizontalPodAutoscaler{},
+			)
+			if err != nil {
+				return nil, nil, "", nil, fmt.Errorf("unable to resolve managed resource GVKs: %w", err)
+			}
+
 			provider, err = consumerprovider.New(providerMgr, consumerprovider.Options{
 				RootClient:   rootClient,
 				Scheme:       scheme,
@@ -570,9 +599,7 @@ func initializeClusterDiscovery(
 						o.Cache.DefaultTransform = cache.TransformStripManagedFields()
 					},
 				},
-				ManagedResources: []schema.GroupVersionKind{
-					computev1alpha.GroupVersion.WithKind("Instance"),
-				},
+				ManagedResources: managedResources,
 				Teardowns: []consumerprovider.Teardown{
 					controller.NewComputeTeardown(quotaClientManager, federationClient, scheme),
 				},

config/base/crd/bases/compute.datumapis.com_workloaddeployments.yaml

@@ -78,6 +78,12 @@ spec:
               placementName:
                 description: The placement in the workload which is driving a deployment
                 type: string
+              replicas:
+                description: |-
+                  Replicas is the current desired replica target for this deployment. When
+                  unset, the deployment reconciles to scaleSettings.minReplicas.
+                format: int32
+                type: integer
               scaleSettings:
                 description: Scale settings such as minimum and maximum replica counts.
                 properties:
@@ -1173,6 +1179,10 @@ spec:
                 description: The number of instances created
                 format: int32
                 type: integer
+              selector:
+                description: Selector is the label selector that identifies Pods backing
+                  this deployment.
+                type: string
               suspended:
                 description: |-
                   Suspended, when true, requests that all instances managed by this deployment
@@ -1197,4 +1207,8 @@ spec:
     served: true
     storage: true
     subresources:
+      scale:
+        labelSelectorPath: .status.selector
+        specReplicasPath: .spec.replicas
+        statusReplicasPath: .status.replicas
       status: {}

config/components/controller_rbac/role.yaml

@@ -24,6 +24,18 @@ rules:
   verbs:
   - get
   - list
+- apiGroups:
+  - autoscaling
+  resources:
+  - horizontalpodautoscalers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - compute.datumapis.com
   resources:
@@ -96,3 +108,12 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - services.miloapis.com
+  resources:
+  - serviceconsumers
+  - services
+  verbs:
+  - get
+  - list
+  - watch