DNM: complete cluster autoscaling branch, for CI and reference

Cargo.toml

@@ -24,6 +24,7 @@ members = [
     "src/cloud-resources",
     "src/cluster",
     "src/cluster-client",
+    "src/cluster-controller",
     "src/clusterd",
     "src/compute",
     "src/compute-client",
@@ -153,6 +154,7 @@ default-members = [
     "src/cloud-resources",
     "src/cluster",
     "src/cluster-client",
+    "src/cluster-controller",
     "src/clusterd",
     "src/compute",
     "src/compute-client",

doc/user/content/reference/system-catalog/mz_internal.md

@@ -165,6 +165,34 @@ The `mz_cluster_schedules` table shows the `SCHEDULE` option specified for each
 | `type`                              | [`text`]     | `on-refresh`, or `manual`. Default: `manual`                   |
 | `refresh_hydration_time_estimate`   | [`interval`] | The interval given in the `HYDRATION TIME ESTIMATE` option.    |
 
+## `mz_cluster_reconfigurations`
+
+The `mz_cluster_reconfigurations` table shows the in-flight graceful reconfiguration of each managed
+cluster that has one. A row is present only while a background `ALTER CLUSTER` is converging on a
+new configuration; the realized (current) shape is in
+[`mz_clusters`](../mz_catalog/#mz_clusters).
+
+<!-- RELATION_SPEC mz_internal.mz_cluster_reconfigurations -->
+| Field          | Type             | Meaning                                                        |
+|----------------|------------------|----------------------------------------------------------------|
+| `cluster_id`   | [`text`]         | The ID of the cluster. Corresponds to [`mz_clusters.id`](../mz_catalog/#mz_clusters). |
+| `deadline`     | [`mz_timestamp`] | The deadline by which the reconfiguration must complete; after it passes the `on_timeout` action applies. Compare against `mz_now()` to distinguish an in-progress reconfiguration from one past its deadline. |
+| `on_timeout`   | [`text`]         | The action applied if `deadline` passes before the target hydrates: `commit` (cut over to the not-yet-hydrated target) or `rollback` (revert to the pre-reconfiguration shape). |
+| `target`       | [`jsonb`]        | The config shape the cluster is reconfiguring to, as JSON: `size`, `replication_factor`, `availability_zones`, and `logging`. The realized (current) shape is in `mz_clusters`. |
+
+## `mz_cluster_auto_scaling_strategies`
+
+The `mz_cluster_auto_scaling_strategies` table shows the configured `AUTO SCALING STRATEGY` of each
+managed cluster that has one, together with any in-flight autoscaling state. A row is present while
+a strategy is configured or an autoscaling action is running.
+
+<!-- RELATION_SPEC mz_internal.mz_cluster_auto_scaling_strategies -->
+| Field          | Type      | Meaning                                                        |
+|----------------|-----------|----------------------------------------------------------------|
+| `cluster_id`   | [`text`]  | The ID of the cluster. Corresponds to [`mz_clusters.id`](../mz_catalog/#mz_clusters). |
+| `strategy`     | [`jsonb`] | **Unstable** The configured autoscaling policy, as JSON. Currently an `on_hydration` sub-policy carrying its `hydration_size` and optional `linger_duration`. |
+| `state`        | [`jsonb`] | **Unstable** The in-flight autoscaling runtime state, as JSON keyed by strategy, or `NULL` when nothing is running. Currently a `burst` key carrying the active hydration burst: its `burst_size`, `linger_duration`, and `steady_hydrated_at`. |
+
 ## `mz_cluster_replica_metrics`
 
 The `mz_cluster_replica_metrics` view gives the last known CPU and RAM utilization statistics

misc/python/materialize/mzcompose/__init__.py

@@ -96,6 +96,16 @@ def get_minimal_system_parameters(
         "enable_refresh_every_mvs": "true",
         "enable_replacement_materialized_views": "true",
         "enable_cluster_schedule_refresh": "true",
+        # The cluster controller and background ALTER CLUSTER land dark in
+        # production (the dyncfg defaults stay false); force them on for the test
+        # harness so CI exercises the controller owning the managed-cluster
+        # replica set. The real production default flip is a separate rollout.
+        "enable_cluster_controller": (
+            "true" if version >= MzVersion.parse_mz("v26.29.0-dev") else "false"
+        ),
+        "enable_background_alter_cluster": (
+            "true" if version >= MzVersion.parse_mz("v26.29.0-dev") else "false"
+        ),
         "enable_s3_tables_region_check": "false",
         "enable_statement_lifecycle_logging": "true",
         "enable_storage_introspection_logs": "true",
@@ -673,6 +683,10 @@ def get_default_system_parameters(
     "enable_mcp_developer_query_tool",
     "mcp_max_response_size",
     "user_id_pool_batch_size",
+    "cluster_controller_tick_interval",
+    "default_cluster_reconfiguration_timeout",
+    "enable_hydration_burst",
+    "default_hydration_burst_linger",
 ]
 
 

misc/python/materialize/parallel_workload/action.py

@@ -1864,6 +1864,12 @@ def __init__(
             "oidc_group_role_sync_strict",
             "console_oidc_client_id",
             "console_oidc_scopes",
+            "enable_cluster_controller",
+            "cluster_controller_tick_interval",
+            "enable_background_alter_cluster",
+            "default_cluster_reconfiguration_timeout",
+            "enable_hydration_burst",
+            "default_hydration_burst_linger",
         ]
 
     def run(self, exe: Executor) -> bool: