diff --git a/acceptance/bundle/resources/registered_models/aliases_converge/databricks.yml.tmpl b/acceptance/bundle/resources/registered_models/aliases_converge/databricks.yml.tmpl new file mode 100644 index 00000000000..43550247edc --- /dev/null +++ b/acceptance/bundle/resources/registered_models/aliases_converge/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: deploy-registered-models-aliases-$UNIQUE_NAME + +resources: + registered_models: + my_registered_model: + name: my-registered-model-aliases-$UNIQUE_NAME + comment: $COMMENT + catalog_name: main + schema_name: default + # Aliases live on model versions and GET does not echo them back + # (DoRead uses IncludeAliases=false), so a config that sets them used to + # report a perpetual in-place update and never converge. + aliases: + - alias_name: champion + id: alias-champion diff --git a/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml b/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/registered_models/aliases_converge/output.txt b/acceptance/bundle/resources/registered_models/aliases_converge/output.txt new file mode 100644 index 00000000000..f1ac429ace8 --- /dev/null +++ b/acceptance/bundle/resources/registered_models/aliases_converge/output.txt @@ -0,0 +1,57 @@ + +>>> export COMMENT=original comment + +=== Initial deployment with aliases set +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-registered-models-aliases-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan is a no-op: GET never echoes aliases, so they must not drift +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== The config-set aliases are skipped as input_only (confirms the matched rule) +>>> [CLI] bundle plan --output json +{ + "action": "skip", + "reason": "input_only", + "old": [ + { + "alias_name": "champion", + "id": "alias-champion" + } + ], + "new": [ + { + "alias_name": "champion", + "id": "alias-champion" + } + ] +} + +=== Edit the comment: an in-place update, not a recreate +>>> [CLI] bundle plan +update registered_models.my_registered_model + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-registered-models-aliases-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Re-plan after the edit converges: no further changes +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.registered_models.my_registered_model + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-registered-models-aliases-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/registered_models/aliases_converge/script b/acceptance/bundle/resources/registered_models/aliases_converge/script new file mode 100644 index 00000000000..ca9dc8f700a --- /dev/null +++ b/acceptance/bundle/resources/registered_models/aliases_converge/script @@ -0,0 +1,27 @@ +echo "*" > .gitignore + +trace export COMMENT="original comment" +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Initial deployment with aliases set" +trace $CLI bundle deploy + +title "Plan is a no-op: GET never echoes aliases, so they must not drift" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + +title "The config-set aliases are skipped as input_only (confirms the matched rule)" +trace $CLI bundle plan --output json | jq '.plan[].changes.aliases' + +title "Edit the comment: an in-place update, not a recreate" +export COMMENT="updated comment" +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle plan | contains.py "Plan: 0 to add, 1 to change, 0 to delete" +trace $CLI bundle deploy + +title "Re-plan after the edit converges: no further changes" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" diff --git a/acceptance/bundle/resources/registered_models/aliases_converge/test.toml b/acceptance/bundle/resources/registered_models/aliases_converge/test.toml new file mode 100644 index 00000000000..b06ed5ceaab --- /dev/null +++ b/acceptance/bundle/resources/registered_models/aliases_converge/test.toml @@ -0,0 +1,5 @@ +RecordRequests = false + +# The aliases input_only classification lives in the direct engine, so scope +# this regression there. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/databricks.yml.tmpl b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/databricks.yml.tmpl new file mode 100644 index 00000000000..ce2db8adb30 --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: lifecycle-started-edit-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + sql_warehouses: + mywarehouse: + name: $UNIQUE_NAME + cluster_size: "2X-Small" + auto_stop_mins: 10 + lifecycle: + started: false diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml new file mode 100644 index 00000000000..d0abd00ab97 --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/output.txt b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/output.txt new file mode 100644 index 00000000000..4379e7528b8 --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/output.txt @@ -0,0 +1,67 @@ + +=== Deploy with started=false: warehouse created and then stopped +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //sql/warehouses +{ + "method": "POST", + "path": "/api/2.0/sql/warehouses", + "body": { + "auto_stop_mins": 10, + "cluster_size": "2X-Small", + "enable_photon": true, + "max_num_clusters": 1, + "name": "[UNIQUE_NAME]", + "spot_instance_policy": "COST_OPTIMIZED" + } +} +{ + "method": "POST", + "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/stop" +} + +>>> errcode [CLI] warehouses get [WAREHOUSE_ID] +"STOPPED" + +=== Edit a field with started=false: Edit restarts the warehouse, so a Stop must follow +>>> update_file.py databricks.yml auto_stop_mins: 10 auto_stop_mins: 20 + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //sql/warehouses +{ + "method": "POST", + "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/edit", + "body": { + "auto_stop_mins": 20, + "cluster_size": "2X-Small", + "enable_photon": true, + "max_num_clusters": 1, + "name": "[UNIQUE_NAME]", + "spot_instance_policy": "COST_OPTIMIZED" + } +} +{ + "method": "POST", + "path": "/api/2.0/sql/warehouses/[WAREHOUSE_ID]/stop" +} + +>>> errcode [CLI] warehouses get [WAREHOUSE_ID] +"STOPPED" + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.sql_warehouses.mywarehouse + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/script b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/script new file mode 100644 index 00000000000..2ec69c28804 --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/script @@ -0,0 +1,24 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Deploy with started=false: warehouse created and then stopped" +trace errcode $CLI bundle deploy +WAREHOUSE_ID=$($CLI bundle summary -o json | jq -r '.resources.sql_warehouses.mywarehouse.id') +add_repl.py "$WAREHOUSE_ID" "WAREHOUSE_ID" +trace print_requests.py //sql/warehouses +rm -f out.requests.txt +{ trace errcode $CLI warehouses get "$WAREHOUSE_ID" | jq '.state'; } || true + +# Editing a warehouse restarts it, so a started=false config must follow the Edit with a +# Stop; before the fix this redeploy hung waiting for STOPPED. +title "Edit a field with started=false: Edit restarts the warehouse, so a Stop must follow" +trace update_file.py databricks.yml "auto_stop_mins: 10" "auto_stop_mins: 20" +trace errcode $CLI bundle deploy +trace print_requests.py //sql/warehouses +rm -f out.requests.txt +{ trace errcode $CLI warehouses get "$WAREHOUSE_ID" | jq '.state'; } || true diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/test.toml new file mode 100644 index 00000000000..57d6adc378c --- /dev/null +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/test.toml @@ -0,0 +1,11 @@ +Local = true +RecordRequests = true + +# Starting warehouses can take a while on infra under load. +# Omit this test to avoid timeouts. +Cloud = false + +Ignore = [".databricks", "databricks.yml"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index e5323c81cb4..e01801393ce 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -337,6 +337,11 @@ resources: # Backend-computed; the user never sets it. Not annotated output_only in the spec. - field: browse_only reason: output_only + # Aliases are managed on model versions through a separate API, and DoRead + # passes IncludeAliases=false, so GET never echoes them back. Without this a + # config that sets aliases reports a perpetual update (remote stays empty). + - field: aliases + reason: input_only provided_id_fields: # The name can technically be updated without recreate. We recreate for now though # to match TF implementation. diff --git a/bundle/direct/dresources/sql_warehouse.go b/bundle/direct/dresources/sql_warehouse.go index 146dee5294d..864ee99e039 100644 --- a/bundle/direct/dresources/sql_warehouse.go +++ b/bundle/direct/dresources/sql_warehouse.go @@ -134,7 +134,8 @@ func hasWarehouseChanges(entry *PlanEntry) bool { // DoUpdate updates the warehouse in place. func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config *SqlWarehouseState, entry *PlanEntry) (*SqlWarehouseRemote, error) { - if hasWarehouseChanges(entry) { + edited := hasWarehouseChanges(entry) + if edited { request := sql.EditWarehouseRequest{ AutoStopMins: config.AutoStopMins, Channel: config.Channel, @@ -169,6 +170,11 @@ func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config * desiredStarted := *config.Lifecycle.Started alreadyRunning := remoteWarehouseIsRunning(entry) + if edited { + // Editing a warehouse restarts it (Edit returns a WaitGetWarehouseRunning + // waiter), so reconcile from RUNNING, not the stale pre-plan state. + alreadyRunning = true + } if desiredStarted && !alreadyRunning { // lifecycle.started=true: fire Start; WaitAfterUpdate polls for RUNNING. _, err := r.client.Warehouses.Start(ctx, sql.StartRequest{Id: id})