From 7b28c1825ba8eda1bfe798b3183c19fe1a523140 Mon Sep 17 00:00:00 2001 From: Ronald Zhang Date: Wed, 8 Jul 2026 00:26:33 +0000 Subject: [PATCH 1/3] Add cascade field to pipeline resource for destroy Add a `cascade` field to the pipeline bundle resource that controls whether destroying a pipeline also deletes its datasets (materialized views, streaming tables, and views). When unset, the server default applies (cascade); set `cascade: false` to retain the datasets on destroy. The field is delete-time only: it is not part of the pipeline spec, so it is never sent on create/update. The direct engine persists it in state (via a PipelineState wrapper around CreatePipeline, mirroring the sql_warehouse lifecycle pattern) and reads it at delete time, force-sending cascade so an explicit false survives query-string omitempty (honored for query params as of databricks-sdk-go v0.152.0). A cascade-only change is a state-only update with no pipeline API call. cascade is classified input_only so it does not show remote drift. The terraform engine drops the field for now: the pinned provider has no such attribute yet (pending terraform-provider-databricks#5860), so terraform-engine support is a follow-up. Co-authored-by: Isaac --- NEXT_CHANGELOG.md | 1 + acceptance/bundle/refschema/out.fields.txt | 1 + .../destroy-pipeline-cascade/databricks.yml | 7 ++ .../destroy-pipeline-cascade/out.test.toml | 3 + .../destroy-pipeline-cascade/output.txt | 56 ++++++++++++++++ .../destroy/destroy-pipeline-cascade/script | 15 +++++ .../destroy-pipeline-cascade/test.toml | 5 ++ bundle/config/resources/pipeline.go | 4 ++ .../terraform/tfdyn/convert_pipeline.go | 3 +- bundle/direct/dresources/pipeline.go | 65 ++++++++++++++++--- bundle/direct/dresources/resources.yml | 4 ++ bundle/internal/schema/annotations.yml | 3 + bundle/schema/jsonschema.json | 6 +- 13 files changed, 161 insertions(+), 12 deletions(-) create mode 100644 acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml create mode 100644 acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml create mode 100644 acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt create mode 100644 acceptance/pipelines/destroy/destroy-pipeline-cascade/script create mode 100644 acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 30a64c54100..85baff1386e 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -7,6 +7,7 @@ ### CLI ### Bundles +* Added a `cascade` field to the pipeline resource to control whether destroying a pipeline also cascades to its datasets. When unset, the server default applies; set `cascade: false` to retain the datasets on destroy. ### Dependency updates diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 7f9cf9c94f5..99595340f02 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -2273,6 +2273,7 @@ resources.models.*.permissions[*].service_principal_name string ALL resources.models.*.permissions[*].user_name string ALL resources.pipelines.*.allow_duplicate_names bool ALL resources.pipelines.*.budget_policy_id string ALL +resources.pipelines.*.cascade *bool ALL resources.pipelines.*.catalog string ALL resources.pipelines.*.cause string REMOTE resources.pipelines.*.channel string ALL diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml new file mode 100644 index 00000000000..5e0b4f9b9c1 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-pipeline-destroy-cascade +resources: + pipelines: + my_pipeline: + name: test-pipeline-cascade + cascade: false diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt new file mode 100644 index 00000000000..d13113ef0f7 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt @@ -0,0 +1,56 @@ + +=== Deploy a pipeline configured with cascade: false +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] + +>>> print_requests.py //api/2.0/pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/state/metadata.json" + }, + "edition": "ADVANCED", + "name": "test-pipeline-cascade" + } +} + +=== Change only cascade (false -> true): expect a state-only update, no pipeline update API call +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] + +>>> print_requests.py //api/2.0/pipelines + +=== Destroy: delete reads the updated cascade=true from persisted state +>>> [CLI] pipelines destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.my_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them: + delete resources.pipelines.my_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //api/2.0/pipelines +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[UUID]", + "q": { + "cascade": "true" + } +} diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/script b/acceptance/pipelines/destroy/destroy-pipeline-cascade/script new file mode 100644 index 00000000000..e3124520050 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/script @@ -0,0 +1,15 @@ +title "Deploy a pipeline configured with cascade: false" +trace $CLI pipelines deploy + +# Flush the deploy requests: cascade is delete-time-only, so the create request must not carry it. +trace print_requests.py //api/2.0/pipelines + +title "Change only cascade (false -> true): expect a state-only update, no pipeline update API call" +update_file.py databricks.yml "cascade: false" "cascade: true" +trace $CLI pipelines deploy +# No POST/PATCH to /pipelines is expected here; the cascade-only change is persisted to state. +trace print_requests.py //api/2.0/pipelines + +title "Destroy: delete reads the updated cascade=true from persisted state" +trace $CLI pipelines destroy --auto-approve +trace print_requests.py //api/2.0/pipelines diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml new file mode 100644 index 00000000000..bd95018f1c6 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml @@ -0,0 +1,5 @@ +# --cascade only affects the direct engine's delete request; the terraform engine deletes +# pipelines through the provider and ignores it. Restrict to direct so the recorded request +# is meaningful and the single variant does not diverge. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index c442a01a641..7fc64dde896 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -16,6 +16,10 @@ type Pipeline struct { pipelines.CreatePipeline //nolint CreatePipeline also defines Id field with the same json tag "id" Permissions []PipelinePermission `json:"permissions,omitempty"` + + // Optional parameter that controls whether destroying the pipeline also cascades to datasets. + // By default, the server will cascade the pipeline deletion to its datasets. + Cascade *bool `json:"cascade,omitempty"` } func (p *Pipeline) UnmarshalJSON(b []byte) error { diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline.go b/bundle/deploy/terraform/tfdyn/convert_pipeline.go index 944ce0858c7..04b652e16c5 100644 --- a/bundle/deploy/terraform/tfdyn/convert_pipeline.go +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline.go @@ -21,7 +21,8 @@ func convertPipelineResource(ctx context.Context, vin dyn.Value) (dyn.Value, err return dyn.InvalidValue, err } - vout, err = dyn.DropKeys(vout, []string{"dry_run"}) + // Current Terraform provider does not support the cascade attribute yet + vout, err = dyn.DropKeys(vout, []string{"dry_run", "cascade"}) if err != nil { return dyn.InvalidValue, err } diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index 20d2eb08245..c151acee88c 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -10,11 +10,32 @@ import ( "github.com/databricks/databricks-sdk-go/service/pipelines" ) +// PipelineState is the state type for Pipeline resources. It extends CreatePipeline with +// the Cascade field, a delete-time setting that is not part of the pipeline spec +type PipelineState struct { + pipelines.CreatePipeline + + // Cascade controls whether deleting the pipeline also deletes its datasets (MVs, STs, + // Views). Nil means the server default (cascade) applies. Read from persisted state at + // delete time; never sent on create/update. + Cascade *bool `json:"cascade,omitempty"` +} + +func (s *PipelineState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s PipelineState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + // PipelineRemote is the return type for DoRead. It embeds CreatePipeline so that all // paths in StateType are valid paths in RemoteType. type PipelineRemote struct { pipelines.CreatePipeline + Cascade *bool `json:"cascade,omitempty"` + // Remote-specific fields from pipelines.GetPipelineResponse Cause string `json:"cause,omitempty"` ClusterId string `json:"cluster_id,omitempty"` @@ -49,12 +70,18 @@ func (*ResourcePipeline) New(client *databricks.WorkspaceClient) *ResourcePipeli } } -func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *pipelines.CreatePipeline { - return &input.CreatePipeline +func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *PipelineState { + return &PipelineState{ + CreatePipeline: input.CreatePipeline, + Cascade: input.Cascade, + } } -func (*ResourcePipeline) RemapState(remote *PipelineRemote) *pipelines.CreatePipeline { - return &remote.CreatePipeline +func (*ResourcePipeline) RemapState(remote *PipelineRemote) *PipelineState { + return &PipelineState{ + CreatePipeline: remote.CreatePipeline, + Cascade: remote.Cascade, + } } func (r *ResourcePipeline) DoRead(ctx context.Context, id string) (*PipelineRemote, error) { @@ -109,7 +136,9 @@ func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { } } return &PipelineRemote{ - CreatePipeline: createPipeline, + CreatePipeline: createPipeline, + // cascade is input-only; the GET response never carries it, so leave it nil. + Cascade: nil, Cause: p.Cause, ClusterId: p.ClusterId, CreatorUserName: p.CreatorUserName, @@ -124,15 +153,22 @@ func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { } } -func (r *ResourcePipeline) DoCreate(ctx context.Context, config *pipelines.CreatePipeline) (string, *PipelineRemote, error) { - response, err := r.client.Pipelines.Create(ctx, *config) +func (r *ResourcePipeline) DoCreate(ctx context.Context, config *PipelineState) (string, *PipelineRemote, error) { + response, err := r.client.Pipelines.Create(ctx, config.CreatePipeline) if err != nil { return "", nil, err } return response.PipelineId, nil, nil } -func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *pipelines.CreatePipeline, _ *PlanEntry) (*PipelineRemote, error) { +func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *PipelineState, entry *PlanEntry) (*PipelineRemote, error) { + // cascade is a delete-time-only setting with no update API, so a change to it alone must + // persist to state without a pipeline Update call. This mirrors sql_warehouse's handling + // of its non-spec lifecycle field. + if !entry.Changes.HasChangeExcept("cascade") { + return nil, nil + } + request := pipelines.EditPipeline{ AllowDuplicateNames: config.AllowDuplicateNames, BudgetPolicyId: config.BudgetPolicyId, @@ -174,8 +210,17 @@ func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *pipe return nil, r.client.Pipelines.Update(ctx, request) } -func (r *ResourcePipeline) DoDelete(ctx context.Context, id string, _ *pipelines.CreatePipeline) error { - return r.client.Pipelines.DeleteByPipelineId(ctx, id) +func (r *ResourcePipeline) DoDelete(ctx context.Context, id string, state *PipelineState) error { + if state.Cascade == nil { + // No explicit cascade in config: preserve the backend default (cascade). + return r.client.Pipelines.DeleteByPipelineId(ctx, id) + } + return r.client.Pipelines.Delete(ctx, pipelines.DeletePipelineRequest{ + PipelineId: id, + Cascade: *state.Cascade, + Force: false, + ForceSendFields: []string{"Cascade"}, + }) } // Note, terraform provider either diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 6c8cd3973ed..fdfc94c9de7 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -171,6 +171,10 @@ resources: reason: "!drop" - field: run_as reason: input_only + # cascade is a delete-time-only setting + # it is never returned by GET, so suppress remote drift for it. + - field: cascade + reason: input_only ignore_local_changes: - field: deployment.version_id diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index a868bdc98f7..407d7879475 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1229,6 +1229,9 @@ resources: path: ./pipeline.py ``` "$fields": + "cascade": + "description": |- + Whether destroying the pipeline also cascades to its datasets. Defaults to true. Set to false to keep the datasets when the pipeline is deleted. Only affects the delete operation. "ingestion_definition": "$fields": "objects": diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 87154f7945f..f72d190f3d1 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1357,6 +1357,10 @@ "description": "[Public Preview] Budget policy of this pipeline.", "$ref": "#/$defs/string" }, + "cascade": { + "description": "Whether destroying the pipeline also cascades to its datasets. Defaults to true. Set to false to keep the datasets when the pipeline is deleted. Only affects the delete operation.", + "$ref": "#/$defs/bool" + }, "catalog": { "description": "A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog.", "$ref": "#/$defs/string" @@ -14867,4 +14871,4 @@ } }, "additionalProperties": {} -} \ No newline at end of file +} From 7df2def069df07a7ec64c79ee69e4f278dc4a614 Mon Sep 17 00:00:00 2001 From: Ronald Zhang Date: Thu, 9 Jul 2026 16:33:01 +0000 Subject: [PATCH 2/3] Address review: rename to cascade_on_destroy, add terraform guard and message fix - Rename the pipeline config field cascade -> cascade_on_destroy, matching the Lakebase purge_on_delete precedent (per review discussion). - Add ValidateCascadeOnDestroy mutator: hard error when the field is used with the terraform engine, which does not support it yet. - Fix the destroy approval message so a pipeline with cascade_on_destroy: false is not described as deleting its STs/MVs, and mention the property otherwise. - Comment why ForceSendFields is needed in DoDelete. - Regenerate schema, refschema, and affected acceptance outputs. Co-authored-by: Isaac --- NEXT_CHANGELOG.md | 2 +- .../cli_defaults/output.txt | 2 +- .../job_pipeline_task/output.txt | 2 +- .../pipeline_fields/output.txt | 2 +- .../resolve_variables/output.txt | 2 +- .../validation_errors/output.txt | 2 +- .../deploy/pipeline-config-dots/output.txt | 2 +- .../deploy/snapshot-comparison/output.txt | 4 +- .../bundle/destroy/all-resources/output.txt | 2 +- .../destroy/jobs-and-pipeline/output.txt | 2 +- acceptance/bundle/migrate/runas/output.txt | 2 +- acceptance/bundle/refschema/out.fields.txt | 2 +- .../pipelines_recreate/output.txt | 2 +- .../resource_deps/remote_app_url/output.txt | 2 +- .../pipelines/current_can_manage/output.txt | 2 +- .../pipelines/current_is_owner/output.txt | 2 +- .../pipelines/empty_list/output.txt | 2 +- .../pipelines/other_can_manage/output.txt | 2 +- .../pipelines/other_is_owner/output.txt | 2 +- .../permissions/pipelines/update/output.txt | 2 +- .../allow-duplicate-names/output.txt | 2 +- .../pipelines/auto-approve/output.txt | 2 +- .../pipelines/lakeflow-pipeline/output.txt | 2 +- .../change-ingestion-definition/output.txt | 2 +- .../recreate-keys/change-storage/output.txt | 2 +- .../resources/pipelines/recreate/output.txt | 2 +- .../resources/pipelines/update/output.txt | 2 +- .../resources/schemas/auto-approve/output.txt | 2 +- .../bundle/summary/modified_status/output.txt | 2 +- .../integration_classic/output.txt | 4 +- .../pipelines/destroy/auto-approve/output.txt | 2 +- .../databricks.yml | 7 ++ .../out.test.toml | 3 + .../output.txt | 8 +++ .../cascade-on-destroy-terraform-error/script | 2 + .../test.toml | 8 +++ .../destroy-pipeline-cascade/databricks.yml | 2 +- .../destroy-pipeline-cascade/output.txt | 22 ++++-- .../destroy/destroy-pipeline-cascade/script | 15 ++-- .../destroy-pipeline-cascade/test.toml | 6 +- .../destroy/destroy-pipeline/output.txt | 2 +- .../pipelines/destroy/force-lock/output.txt | 2 +- acceptance/pipelines/e2e/output.txt | 2 +- .../mutator/validate_cascade_on_destroy.go | 45 ++++++++++++ .../validate_cascade_on_destroy_test.go | 68 +++++++++++++++++++ bundle/config/resources/pipeline.go | 8 ++- .../terraform/tfdyn/convert_pipeline.go | 6 +- bundle/direct/dresources/pipeline.go | 39 ++++++----- bundle/direct/dresources/resources.yml | 4 +- bundle/internal/schema/annotations.yml | 4 +- bundle/phases/destroy.go | 36 +++++++++- bundle/phases/messages.go | 5 +- bundle/phases/plan.go | 20 ++++++ bundle/schema/jsonschema.json | 6 +- 54 files changed, 303 insertions(+), 83 deletions(-) create mode 100644 acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/databricks.yml create mode 100644 acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/out.test.toml create mode 100644 acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/output.txt create mode 100644 acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/script create mode 100644 acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/test.toml create mode 100644 bundle/config/mutator/validate_cascade_on_destroy.go create mode 100644 bundle/config/mutator/validate_cascade_on_destroy_test.go diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 85baff1386e..d67d0387ab6 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -7,7 +7,7 @@ ### CLI ### Bundles -* Added a `cascade` field to the pipeline resource to control whether destroying a pipeline also cascades to its datasets. When unset, the server default applies; set `cascade: false` to retain the datasets on destroy. +* Added a `cascade_on_destroy` field to the pipeline resource to control whether destroying a pipeline also deletes its datasets (MVs, STs, Views). When unset, the server default applies; set `cascade_on_destroy: false` to retain the datasets on destroy. Supported with the direct deployment engine. ### Dependency updates diff --git a/acceptance/bundle/config-remote-sync/cli_defaults/output.txt b/acceptance/bundle/config-remote-sync/cli_defaults/output.txt index 307a945ea93..bdcf3f6fdce 100644 --- a/acceptance/bundle/config-remote-sync/cli_defaults/output.txt +++ b/acceptance/bundle/config-remote-sync/cli_defaults/output.txt @@ -57,7 +57,7 @@ The following resources will be deleted: delete resources.pipelines.pipeline1 This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.pipeline1 All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt b/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt index 54a1e342788..35d6d03d3c5 100644 --- a/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt +++ b/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt @@ -41,7 +41,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt b/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt index e14b94f76f5..17573e9e043 100644 --- a/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt +++ b/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt @@ -60,7 +60,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/resolve_variables/output.txt b/acceptance/bundle/config-remote-sync/resolve_variables/output.txt index 388f7ab3bbf..b65460cc725 100644 --- a/acceptance/bundle/config-remote-sync/resolve_variables/output.txt +++ b/acceptance/bundle/config-remote-sync/resolve_variables/output.txt @@ -143,7 +143,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/validation_errors/output.txt b/acceptance/bundle/config-remote-sync/validation_errors/output.txt index c4ce29efa0a..467e86d9bcf 100644 --- a/acceptance/bundle/config-remote-sync/validation_errors/output.txt +++ b/acceptance/bundle/config-remote-sync/validation_errors/output.txt @@ -51,7 +51,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/deploy/pipeline-config-dots/output.txt b/acceptance/bundle/deploy/pipeline-config-dots/output.txt index 48b6576f221..1621358d154 100644 --- a/acceptance/bundle/deploy/pipeline-config-dots/output.txt +++ b/acceptance/bundle/deploy/pipeline-config-dots/output.txt @@ -23,7 +23,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.my_schema This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/deploy/snapshot-comparison/output.txt b/acceptance/bundle/deploy/snapshot-comparison/output.txt index b5bb597e264..8978def04ad 100644 --- a/acceptance/bundle/deploy/snapshot-comparison/output.txt +++ b/acceptance/bundle/deploy/snapshot-comparison/output.txt @@ -43,7 +43,7 @@ The following resources will be deleted: delete resources.pipelines.test_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.test_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/snapshot-test-1-[UNIQUE_NAME]/default @@ -58,7 +58,7 @@ The following resources will be deleted: delete resources.pipelines.test_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.test_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/snapshot-test-2-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/destroy/all-resources/output.txt b/acceptance/bundle/destroy/all-resources/output.txt index 5fd8d432a82..093fa6ac4f4 100644 --- a/acceptance/bundle/destroy/all-resources/output.txt +++ b/acceptance/bundle/destroy/all-resources/output.txt @@ -14,7 +14,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.my_schema This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt index ce1530c469a..4f32fbca70a 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt +++ b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt @@ -53,7 +53,7 @@ The following resources will be deleted: delete resources.pipelines.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.bar All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/migrate/runas/output.txt b/acceptance/bundle/migrate/runas/output.txt index f8a9b475bdc..2108deb19ee 100644 --- a/acceptance/bundle/migrate/runas/output.txt +++ b/acceptance/bundle/migrate/runas/output.txt @@ -138,7 +138,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 99595340f02..e6afc1bfd68 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -2273,7 +2273,7 @@ resources.models.*.permissions[*].service_principal_name string ALL resources.models.*.permissions[*].user_name string ALL resources.pipelines.*.allow_duplicate_names bool ALL resources.pipelines.*.budget_policy_id string ALL -resources.pipelines.*.cascade *bool ALL +resources.pipelines.*.cascade_on_destroy *bool ALL resources.pipelines.*.catalog string ALL resources.pipelines.*.cause string REMOTE resources.pipelines.*.channel string ALL diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt index b7eec30702b..ec276143f00 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt +++ b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt @@ -108,7 +108,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resource_deps/remote_app_url/output.txt b/acceptance/bundle/resource_deps/remote_app_url/output.txt index 4c51088f686..22fdee366fd 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/output.txt +++ b/acceptance/bundle/resource_deps/remote_app_url/output.txt @@ -90,7 +90,7 @@ The following resources will be deleted: delete resources.pipelines.mypipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.mypipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt index 2e15e6e4108..e0582581250 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt @@ -30,7 +30,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt index 89f7921475b..15f1750b315 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt @@ -18,7 +18,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt b/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt index 2f3ce99457f..ebf9541f2c1 100644 --- a/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt @@ -13,7 +13,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt index 2e15e6e4108..e0582581250 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt @@ -30,7 +30,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt index 3b82b04aa94..7a408e57076 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt @@ -22,7 +22,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/update/output.txt b/acceptance/bundle/resources/permissions/pipelines/update/output.txt index e5395eeceea..a898beb3387 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/update/output.txt @@ -123,7 +123,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/permissions-test/default diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt index 8cea9634565..cd7c8973d08 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt @@ -47,7 +47,7 @@ The following resources will be deleted: delete resources.pipelines.pipeline_one This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.pipeline_one All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/auto-approve/output.txt b/acceptance/bundle/resources/pipelines/auto-approve/output.txt index 3016a4c248d..165a5f22e32 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/output.txt +++ b/acceptance/bundle/resources/pipelines/auto-approve/output.txt @@ -63,7 +63,7 @@ The following resources will be deleted: delete resources.pipelines.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.bar All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt index 04129e78ce1..18382aa4a80 100644 --- a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt +++ b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt @@ -43,7 +43,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-lakeflow-pipeline-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt index 77bfdc09d32..ade49a90450 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt @@ -150,7 +150,7 @@ The following resources will be deleted: delete resources.pipelines.my This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.my All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt index 74943571027..7c4bde4c4c5 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt @@ -134,7 +134,7 @@ The following resources will be deleted: delete resources.pipelines.my This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.my All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/recreate/output.txt b/acceptance/bundle/resources/pipelines/recreate/output.txt index b6a280d38d5..8f3a276e5ff 100644 --- a/acceptance/bundle/resources/pipelines/recreate/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate/output.txt @@ -65,7 +65,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/pipelines/update/output.txt b/acceptance/bundle/resources/pipelines/update/output.txt index 213e8a0cc64..823d321cdf3 100644 --- a/acceptance/bundle/resources/pipelines/update/output.txt +++ b/acceptance/bundle/resources/pipelines/update/output.txt @@ -72,7 +72,7 @@ The following resources will be deleted: delete resources.pipelines.my This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.my All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/schemas/auto-approve/output.txt b/acceptance/bundle/resources/schemas/auto-approve/output.txt index 375c94fe652..90f2dbebe98 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/output.txt +++ b/acceptance/bundle/resources/schemas/auto-approve/output.txt @@ -72,7 +72,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/summary/modified_status/output.txt b/acceptance/bundle/summary/modified_status/output.txt index 92338e101c9..47ee67480f5 100644 --- a/acceptance/bundle/summary/modified_status/output.txt +++ b/acceptance/bundle/summary/modified_status/output.txt @@ -189,7 +189,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.my_schema This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt index 2206b10cd25..db5e5562b17 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/output.txt +++ b/acceptance/bundle/templates/default-python/integration_classic/output.txt @@ -93,7 +93,7 @@ The following resources will be deleted: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev @@ -491,7 +491,7 @@ The following resources will be deleted: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod diff --git a/acceptance/pipelines/destroy/auto-approve/output.txt b/acceptance/pipelines/destroy/auto-approve/output.txt index 76bf97b4e2f..5cefb4144a5 100644 --- a/acceptance/pipelines/destroy/auto-approve/output.txt +++ b/acceptance/pipelines/destroy/auto-approve/output.txt @@ -19,7 +19,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy/default diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/databricks.yml b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/databricks.yml new file mode 100644 index 00000000000..0c4f487853b --- /dev/null +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-cascade-on-destroy-tf-error +resources: + pipelines: + my_pipeline: + name: test-pipeline-cascade-tf + cascade_on_destroy: false diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/out.test.toml b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/out.test.toml new file mode 100644 index 00000000000..65156e0457c --- /dev/null +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/output.txt b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/output.txt new file mode 100644 index 00000000000..0763f5a218b --- /dev/null +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/output.txt @@ -0,0 +1,8 @@ + +=== pipelines deploy fails with cascade_on_destroy on terraform engine +>>> errcode [CLI] pipelines deploy +Error: cascade_on_destroy is only supported in direct deployment mode + in databricks.yml:7:27 + + +Exit code: 1 diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/script b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/script new file mode 100644 index 00000000000..ba05370f088 --- /dev/null +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/script @@ -0,0 +1,2 @@ +title "pipelines deploy fails with cascade_on_destroy on terraform engine" +trace errcode $CLI pipelines deploy diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/test.toml b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/test.toml new file mode 100644 index 00000000000..ec557241780 --- /dev/null +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/test.toml @@ -0,0 +1,8 @@ +Local = true +RecordRequests = false + +# Static validation only; no need to run on cloud. +Cloud = false + +# cascade_on_destroy is direct-only; assert the terraform engine rejects it. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml index 5e0b4f9b9c1..af159a9e9c1 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml @@ -4,4 +4,4 @@ resources: pipelines: my_pipeline: name: test-pipeline-cascade - cascade: false + cascade_on_destroy: false diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt index d13113ef0f7..8df542873ba 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt @@ -1,5 +1,5 @@ -=== Deploy a pipeline configured with cascade: false +=== Deploy a pipeline configured with cascade_on_destroy: false >>> [CLI] pipelines deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... Deploying resources... @@ -22,7 +22,7 @@ View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] } } -=== Change only cascade (false -> true): expect a state-only update, no pipeline update API call +=== Change only cascade_on_destroy (false -> true): expect a state-only update, no pipeline update API call >>> [CLI] pipelines deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... Deploying resources... @@ -32,13 +32,23 @@ View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] >>> print_requests.py //api/2.0/pipelines -=== Destroy: delete reads the updated cascade=true from persisted state +=== Set cascade_on_destroy back to false and deploy so state reflects it before destroy +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] + +>>> print_requests.py //api/2.0/pipelines + +=== Destroy: approval message notes datasets are retained, and delete sends cascade=false >>> [CLI] pipelines destroy --auto-approve The following resources will be deleted: delete resources.pipelines.my_pipeline -This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines. +The Streaming Tables (STs) and Materialized Views (MVs) managed by them are retained (cascade_on_destroy: false): delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default @@ -51,6 +61,6 @@ Destroy complete! "method": "DELETE", "path": "/api/2.0/pipelines/[UUID]", "q": { - "cascade": "true" + "cascade": "false" } } diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/script b/acceptance/pipelines/destroy/destroy-pipeline-cascade/script index e3124520050..8a8ad36eca4 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline-cascade/script +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/script @@ -1,15 +1,20 @@ -title "Deploy a pipeline configured with cascade: false" +title "Deploy a pipeline configured with cascade_on_destroy: false" trace $CLI pipelines deploy -# Flush the deploy requests: cascade is delete-time-only, so the create request must not carry it. +# Flush the deploy requests: cascade_on_destroy is delete-time-only, so the create request must not carry it. trace print_requests.py //api/2.0/pipelines -title "Change only cascade (false -> true): expect a state-only update, no pipeline update API call" -update_file.py databricks.yml "cascade: false" "cascade: true" +title "Change only cascade_on_destroy (false -> true): expect a state-only update, no pipeline update API call" +update_file.py databricks.yml "cascade_on_destroy: false" "cascade_on_destroy: true" trace $CLI pipelines deploy # No POST/PATCH to /pipelines is expected here; the cascade-only change is persisted to state. trace print_requests.py //api/2.0/pipelines -title "Destroy: delete reads the updated cascade=true from persisted state" +title "Set cascade_on_destroy back to false and deploy so state reflects it before destroy" +update_file.py databricks.yml "cascade_on_destroy: true" "cascade_on_destroy: false" +trace $CLI pipelines deploy +trace print_requests.py //api/2.0/pipelines + +title "Destroy: approval message notes datasets are retained, and delete sends cascade=false" trace $CLI pipelines destroy --auto-approve trace print_requests.py //api/2.0/pipelines diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml index bd95018f1c6..bb784ca3147 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml @@ -1,5 +1,5 @@ -# --cascade only affects the direct engine's delete request; the terraform engine deletes -# pipelines through the provider and ignores it. Restrict to direct so the recorded request -# is meaningful and the single variant does not diverge. +# cascade_on_destroy only affects the direct engine's delete request; the terraform engine +# does not support the attribute yet and rejects it (see the terraform-engine sub-test). +# Restrict this test to direct so the recorded request is meaningful. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] RecordRequests = true diff --git a/acceptance/pipelines/destroy/destroy-pipeline/output.txt b/acceptance/pipelines/destroy/destroy-pipeline/output.txt index 4e5ab020852..3b0e50c9143 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline/output.txt +++ b/acceptance/pipelines/destroy/destroy-pipeline/output.txt @@ -12,7 +12,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy/default diff --git a/acceptance/pipelines/destroy/force-lock/output.txt b/acceptance/pipelines/destroy/force-lock/output.txt index f253d542a91..fbc04f6dbac 100644 --- a/acceptance/pipelines/destroy/force-lock/output.txt +++ b/acceptance/pipelines/destroy/force-lock/output.txt @@ -27,7 +27,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-force-lock/default diff --git a/acceptance/pipelines/e2e/output.txt b/acceptance/pipelines/e2e/output.txt index 8e029977b23..d450268a083 100644 --- a/acceptance/pipelines/e2e/output.txt +++ b/acceptance/pipelines/e2e/output.txt @@ -106,7 +106,7 @@ The following resources will be deleted: delete resources.pipelines.lakeflow_project_etl_2 This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: delete resources.pipelines.lakeflow_project_etl delete resources.pipelines.lakeflow_project_etl_2 diff --git a/bundle/config/mutator/validate_cascade_on_destroy.go b/bundle/config/mutator/validate_cascade_on_destroy.go new file mode 100644 index 00000000000..80c5bcc0588 --- /dev/null +++ b/bundle/config/mutator/validate_cascade_on_destroy.go @@ -0,0 +1,45 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/libs/diag" +) + +type validateCascadeOnDestroy struct { + engine engine.EngineType +} + +// ValidateCascadeOnDestroy returns a mutator that errors when a pipeline sets +// cascade_on_destroy while using the terraform deployment engine. The terraform +// provider does not support the attribute yet, so it is only honored in direct +// deployment mode. +func ValidateCascadeOnDestroy(e engine.EngineType) bundle.Mutator { + return &validateCascadeOnDestroy{engine: e} +} + +func (m *validateCascadeOnDestroy) Name() string { + return "ValidateCascadeOnDestroy" +} + +func (m *validateCascadeOnDestroy) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + if m.engine.IsDirect() { + return nil + } + + var diags diag.Diagnostics + for key, pipeline := range b.Config.Resources.Pipelines { + if pipeline.CascadeOnDestroy == nil { + continue + } + path := "resources.pipelines." + key + ".cascade_on_destroy" + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "cascade_on_destroy is only supported in direct deployment mode", + Locations: b.Config.GetLocations(path), + }) + } + return diags +} diff --git a/bundle/config/mutator/validate_cascade_on_destroy_test.go b/bundle/config/mutator/validate_cascade_on_destroy_test.go new file mode 100644 index 00000000000..6b01ba3987b --- /dev/null +++ b/bundle/config/mutator/validate_cascade_on_destroy_test.go @@ -0,0 +1,68 @@ +package mutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" +) + +func pipelineWithCascade(cascade *bool) *resources.Pipeline { + return &resources.Pipeline{ + CreatePipeline: pipelines.CreatePipeline{Name: "my_pipeline"}, + CascadeOnDestroy: cascade, + } +} + +func TestValidateCascadeOnDestroyDirectEngineReturnsNil(t *testing.T) { + no := false + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "my_pipeline": pipelineWithCascade(&no), + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateCascadeOnDestroy(engine.EngineDirect)) + assert.Empty(t, diags) +} + +func TestValidateCascadeOnDestroyTerraformEngineUnsetReturnsNil(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "my_pipeline": pipelineWithCascade(nil), + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateCascadeOnDestroy(engine.EngineTerraform)) + assert.Empty(t, diags) +} + +func TestValidateCascadeOnDestroyTerraformEngineSetEmitsError(t *testing.T) { + no := false + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "my_pipeline": pipelineWithCascade(&no), + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateCascadeOnDestroy(engine.EngineTerraform)) + assert.Len(t, diags, 1) + assert.Equal(t, "cascade_on_destroy is only supported in direct deployment mode", diags[0].Summary) +} diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index 7fc64dde896..f8f7f63ed4d 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -17,9 +17,11 @@ type Pipeline struct { Permissions []PipelinePermission `json:"permissions,omitempty"` - // Optional parameter that controls whether destroying the pipeline also cascades to datasets. - // By default, the server will cascade the pipeline deletion to its datasets. - Cascade *bool `json:"cascade,omitempty"` + // CascadeOnDestroy controls whether destroying the pipeline also deletes its datasets + // (MVs, STs, Views). Nil (unset) uses the server default, which cascades. Set to false to + // retain the datasets when the pipeline is deleted. Mirrors purge_on_delete on Lakebase + // projects. Delete-time only: never sent on create/update. + CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` } func (p *Pipeline) UnmarshalJSON(b []byte) error { diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline.go b/bundle/deploy/terraform/tfdyn/convert_pipeline.go index 04b652e16c5..e393f69c2c6 100644 --- a/bundle/deploy/terraform/tfdyn/convert_pipeline.go +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline.go @@ -21,8 +21,10 @@ func convertPipelineResource(ctx context.Context, vin dyn.Value) (dyn.Value, err return dyn.InvalidValue, err } - // Current Terraform provider does not support the cascade attribute yet - vout, err = dyn.DropKeys(vout, []string{"dry_run", "cascade"}) + // Current Terraform provider does not support the cascade_on_destroy attribute yet. + // ValidateCascadeOnDestroy rejects it for the terraform engine before we get here; this + // drop is defense-in-depth so we never emit an unknown attribute to the provider schema. + vout, err = dyn.DropKeys(vout, []string{"dry_run", "cascade_on_destroy"}) if err != nil { return dyn.InvalidValue, err } diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index c151acee88c..7d01554258c 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -11,14 +11,14 @@ import ( ) // PipelineState is the state type for Pipeline resources. It extends CreatePipeline with -// the Cascade field, a delete-time setting that is not part of the pipeline spec +// the CascadeOnDestroy field, a delete-time setting that is not part of the pipeline spec type PipelineState struct { pipelines.CreatePipeline - // Cascade controls whether deleting the pipeline also deletes its datasets (MVs, STs, - // Views). Nil means the server default (cascade) applies. Read from persisted state at + // CascadeOnDestroy controls whether deleting the pipeline also deletes its datasets (MVs, + // STs, Views). Nil means the server default (cascade) applies. Read from persisted state at // delete time; never sent on create/update. - Cascade *bool `json:"cascade,omitempty"` + CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` } func (s *PipelineState) UnmarshalJSON(b []byte) error { @@ -34,7 +34,7 @@ func (s PipelineState) MarshalJSON() ([]byte, error) { type PipelineRemote struct { pipelines.CreatePipeline - Cascade *bool `json:"cascade,omitempty"` + CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` // Remote-specific fields from pipelines.GetPipelineResponse Cause string `json:"cause,omitempty"` @@ -72,15 +72,15 @@ func (*ResourcePipeline) New(client *databricks.WorkspaceClient) *ResourcePipeli func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *PipelineState { return &PipelineState{ - CreatePipeline: input.CreatePipeline, - Cascade: input.Cascade, + CreatePipeline: input.CreatePipeline, + CascadeOnDestroy: input.CascadeOnDestroy, } } func (*ResourcePipeline) RemapState(remote *PipelineRemote) *PipelineState { return &PipelineState{ - CreatePipeline: remote.CreatePipeline, - Cascade: remote.Cascade, + CreatePipeline: remote.CreatePipeline, + CascadeOnDestroy: remote.CascadeOnDestroy, } } @@ -137,8 +137,8 @@ func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { } return &PipelineRemote{ CreatePipeline: createPipeline, - // cascade is input-only; the GET response never carries it, so leave it nil. - Cascade: nil, + // cascade_on_destroy is input-only; the GET response never carries it, so leave it nil. + CascadeOnDestroy: nil, Cause: p.Cause, ClusterId: p.ClusterId, CreatorUserName: p.CreatorUserName, @@ -162,10 +162,10 @@ func (r *ResourcePipeline) DoCreate(ctx context.Context, config *PipelineState) } func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *PipelineState, entry *PlanEntry) (*PipelineRemote, error) { - // cascade is a delete-time-only setting with no update API, so a change to it alone must - // persist to state without a pipeline Update call. This mirrors sql_warehouse's handling - // of its non-spec lifecycle field. - if !entry.Changes.HasChangeExcept("cascade") { + // cascade_on_destroy is a delete-time-only setting with no update API, so a change to it + // alone must persist to state without a pipeline Update call. This mirrors sql_warehouse's + // handling of its non-spec lifecycle field. + if !entry.Changes.HasChangeExcept("cascade_on_destroy") { return nil, nil } @@ -211,13 +211,16 @@ func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *Pipe } func (r *ResourcePipeline) DoDelete(ctx context.Context, id string, state *PipelineState) error { - if state.Cascade == nil { - // No explicit cascade in config: preserve the backend default (cascade). + if state.CascadeOnDestroy == nil { + // No explicit cascade_on_destroy in config: preserve the backend default (cascade). return r.client.Pipelines.DeleteByPipelineId(ctx, id) } + // Cascade is marshaled as `url:"cascade,omitempty"`, so a false value would be dropped from + // the query string and the backend would apply its default of true. ForceSendFields makes + // the SDK send cascade=false explicitly so cascade_on_destroy: false is honored. return r.client.Pipelines.Delete(ctx, pipelines.DeletePipelineRequest{ PipelineId: id, - Cascade: *state.Cascade, + Cascade: *state.CascadeOnDestroy, Force: false, ForceSendFields: []string{"Cascade"}, }) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index fdfc94c9de7..dd21ed58ac0 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -171,9 +171,9 @@ resources: reason: "!drop" - field: run_as reason: input_only - # cascade is a delete-time-only setting + # cascade_on_destroy is a delete-time-only setting # it is never returned by GET, so suppress remote drift for it. - - field: cascade + - field: cascade_on_destroy reason: input_only ignore_local_changes: diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 407d7879475..e433dfd51c6 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1229,9 +1229,9 @@ resources: path: ./pipeline.py ``` "$fields": - "cascade": + "cascade_on_destroy": "description": |- - Whether destroying the pipeline also cascades to its datasets. Defaults to true. Set to false to keep the datasets when the pipeline is deleted. Only affects the delete operation. + Whether destroying the pipeline also deletes its datasets (MVs, STs, Views). Defaults to true (the server default). Set to false to retain the datasets when the pipeline is deleted. Only affects the delete operation. "ingestion_definition": "$fields": "objects": diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 2496c7033ad..0084777485d 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -34,7 +34,8 @@ func assertRootPathExists(ctx context.Context, b *bundle.Bundle) (bool, error) { var destroyApprovalGroups = []approvalGroup{ {group: "schemas", message: deleteSchemaMessage}, - {group: "pipelines", message: deletePipelineMessage}, + // Pipelines are handled separately in approvalForDestroy so the message reflects each + // pipeline's cascade_on_destroy setting; see logPipelineDeleteApproval. {group: "volumes", message: deleteVolumeMessage}, {group: "database_instances", message: deleteDatabaseInstanceMessage}, {group: "synced_database_tables", message: deleteSyncedDatabaseTableMessage}, @@ -45,6 +46,38 @@ var destroyApprovalGroups = []approvalGroup{ {group: "genie_spaces", message: deleteGenieSpaceMessage}, } +// logPipelineDeleteApproval prints the pipeline deletions, splitting them by cascade_on_destroy +// so pipelines retaining their datasets are not described as deleting STs/MVs. +func logPipelineDeleteApproval(ctx context.Context, b *bundle.Bundle, actions []deployplan.Action) { + pipelineDeletes := filterGroup(actions, "pipelines", deployplan.Delete) + + var cascading, retaining []deployplan.Action + for _, a := range pipelineDeletes { + if pipelineRetainsDatasets(b, a) { + retaining = append(retaining, a) + } else { + cascading = append(cascading, a) + } + } + + for _, grp := range []struct { + message string + actions []deployplan.Action + }{ + {deletePipelineMessage, cascading}, + {deletePipelineNoCascadeMessage, retaining}, + } { + if len(grp.actions) == 0 { + continue + } + cmdio.LogString(ctx, grp.message) + for _, a := range grp.actions { + cmdio.Log(ctx, a) + } + cmdio.LogString(ctx, "") + } +} + func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (bool, error) { deleteActions := plan.GetActions() @@ -69,6 +102,7 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. } logApprovalGroups(ctx, deleteActions, destroyApprovalGroups, true, deployplan.Delete) + logPipelineDeleteApproval(ctx, b, deleteActions) cmdio.LogString(ctx, "All files and directories at the following location will be deleted: "+b.Config.Workspace.RootPath) cmdio.LogString(ctx, "") diff --git a/bundle/phases/messages.go b/bundle/phases/messages.go index 8f07b1e2429..28bed70c37e 100644 --- a/bundle/phases/messages.go +++ b/bundle/phases/messages.go @@ -60,7 +60,10 @@ const ( deleteSchemaMessage = `This action will result in the deletion of the following UC schemas. Any underlying data may be lost:` deletePipelineMessage = `This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them:` +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets:` + + deletePipelineNoCascadeMessage = `This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines. +The Streaming Tables (STs) and Materialized Views (MVs) managed by them are retained (cascade_on_destroy: false):` deleteVolumeMessage = `This action will result in the deletion of the following volumes. For managed volumes, the files stored in the volume are also deleted from your diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 0af9394f243..b20ae25e8fc 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -26,10 +26,30 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateGitDetails(), mutator.ValidateDirectOnlyResources(engine), mutator.ValidateLifecycleStarted(engine), + mutator.ValidateCascadeOnDestroy(engine), statemgmt.CheckRunningResource(engine), ) } +// pipelineRetainsDatasets reports whether the pipeline referenced by a delete action has +// cascade_on_destroy explicitly set to false, meaning its datasets (MVs, STs, Views) are +// retained. Only pipeline resources carry this field, so a lookup miss returns false. +// Reads from config like checkForPreventDestroy; the field is unset for other resource types. +func pipelineRetainsDatasets(b *bundle.Bundle, action deployplan.Action) bool { + path, err := dyn.NewPathFromString(action.ResourceKey) + if err != nil { + return false + } + path = append(path, dyn.Key("cascade_on_destroy")) + + v, err := dyn.GetByPath(b.Config.Value(), path) + if err != nil { + return false + } + cascade, ok := v.AsBool() + return ok && !cascade +} + // checkForPreventDestroy checks if the resource has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. // If it does, it returns an error. func checkForPreventDestroy(b *bundle.Bundle, actions []deployplan.Action) error { diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index f72d190f3d1..fd23a7e2631 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1357,8 +1357,8 @@ "description": "[Public Preview] Budget policy of this pipeline.", "$ref": "#/$defs/string" }, - "cascade": { - "description": "Whether destroying the pipeline also cascades to its datasets. Defaults to true. Set to false to keep the datasets when the pipeline is deleted. Only affects the delete operation.", + "cascade_on_destroy": { + "description": "Whether destroying the pipeline also deletes its datasets (MVs, STs, Views). Defaults to true (the server default). Set to false to retain the datasets when the pipeline is deleted. Only affects the delete operation.", "$ref": "#/$defs/bool" }, "catalog": { @@ -14871,4 +14871,4 @@ } }, "additionalProperties": {} -} +} \ No newline at end of file From 9a2436c214c1d29cb3da86d8ed1db627f69d651b Mon Sep 17 00:00:00 2001 From: Ronald Zhang Date: Fri, 10 Jul 2026 07:01:03 +0000 Subject: [PATCH 3/3] clean up --- .vscode/settings.json | 3 +++ .../config-remote-sync/cli_defaults/output.txt | 2 +- .../job_pipeline_task/output.txt | 2 +- .../pipeline_fields/output.txt | 2 +- .../resolve_variables/output.txt | 2 +- .../validation_errors/output.txt | 2 +- .../deploy/pipeline-config-dots/output.txt | 2 +- .../deploy/snapshot-comparison/output.txt | 4 ++-- .../bundle/destroy/all-resources/output.txt | 2 +- .../bundle/destroy/jobs-and-pipeline/output.txt | 2 +- acceptance/bundle/migrate/runas/output.txt | 2 +- .../resource_deps/pipelines_recreate/output.txt | 2 +- .../resource_deps/remote_app_url/output.txt | 2 +- .../pipelines/current_can_manage/output.txt | 2 +- .../pipelines/current_is_owner/output.txt | 2 +- .../permissions/pipelines/empty_list/output.txt | 2 +- .../pipelines/other_can_manage/output.txt | 2 +- .../pipelines/other_is_owner/output.txt | 2 +- .../permissions/pipelines/update/output.txt | 2 +- .../pipelines/allow-duplicate-names/output.txt | 2 +- .../resources/pipelines/auto-approve/output.txt | 2 +- .../pipelines/lakeflow-pipeline/output.txt | 2 +- .../change-ingestion-definition/output.txt | 2 +- .../recreate-keys/change-storage/output.txt | 2 +- .../resources/pipelines/recreate/output.txt | 2 +- .../resources/pipelines/update/output.txt | 2 +- .../resources/schemas/auto-approve/output.txt | 2 +- .../bundle/summary/modified_status/output.txt | 2 +- .../integration_classic/output.txt | 4 ++-- .../pipelines/destroy/auto-approve/output.txt | 2 +- .../destroy/destroy-pipeline-cascade/output.txt | 2 +- .../destroy/destroy-pipeline-cascade/test.toml | 3 +-- .../destroy/destroy-pipeline/output.txt | 2 +- .../pipelines/destroy/force-lock/output.txt | 2 +- acceptance/pipelines/e2e/output.txt | 2 +- bundle/config/resources/pipeline.go | 5 ++--- .../deploy/terraform/tfdyn/convert_pipeline.go | 3 +-- bundle/direct/dresources/pipeline.go | 14 ++++++-------- bundle/phases/destroy.go | 12 ++++++------ bundle/phases/messages.go | 6 +++--- bundle/phases/plan.go | 17 +++++++++-------- 41 files changed, 66 insertions(+), 67 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index b393450c80d..4f9fae2d8e2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -28,5 +28,8 @@ "databricks.yml", "databricks.yaml", ] + }, + "files.watcherExclude": { + "**/target": true } } diff --git a/acceptance/bundle/config-remote-sync/cli_defaults/output.txt b/acceptance/bundle/config-remote-sync/cli_defaults/output.txt index bdcf3f6fdce..f3e2372b81a 100644 --- a/acceptance/bundle/config-remote-sync/cli_defaults/output.txt +++ b/acceptance/bundle/config-remote-sync/cli_defaults/output.txt @@ -57,7 +57,7 @@ The following resources will be deleted: delete resources.pipelines.pipeline1 This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.pipeline1 All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt b/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt index 35d6d03d3c5..52e37685043 100644 --- a/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt +++ b/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt @@ -41,7 +41,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt b/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt index 17573e9e043..19b20f8befa 100644 --- a/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt +++ b/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt @@ -60,7 +60,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/resolve_variables/output.txt b/acceptance/bundle/config-remote-sync/resolve_variables/output.txt index b65460cc725..8cb8849da4d 100644 --- a/acceptance/bundle/config-remote-sync/resolve_variables/output.txt +++ b/acceptance/bundle/config-remote-sync/resolve_variables/output.txt @@ -143,7 +143,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/validation_errors/output.txt b/acceptance/bundle/config-remote-sync/validation_errors/output.txt index 467e86d9bcf..372e4fa1ae8 100644 --- a/acceptance/bundle/config-remote-sync/validation_errors/output.txt +++ b/acceptance/bundle/config-remote-sync/validation_errors/output.txt @@ -51,7 +51,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/deploy/pipeline-config-dots/output.txt b/acceptance/bundle/deploy/pipeline-config-dots/output.txt index 1621358d154..8b338fcfbb3 100644 --- a/acceptance/bundle/deploy/pipeline-config-dots/output.txt +++ b/acceptance/bundle/deploy/pipeline-config-dots/output.txt @@ -23,7 +23,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.my_schema This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/deploy/snapshot-comparison/output.txt b/acceptance/bundle/deploy/snapshot-comparison/output.txt index 8978def04ad..c908a5fa61a 100644 --- a/acceptance/bundle/deploy/snapshot-comparison/output.txt +++ b/acceptance/bundle/deploy/snapshot-comparison/output.txt @@ -43,7 +43,7 @@ The following resources will be deleted: delete resources.pipelines.test_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.test_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/snapshot-test-1-[UNIQUE_NAME]/default @@ -58,7 +58,7 @@ The following resources will be deleted: delete resources.pipelines.test_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.test_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/snapshot-test-2-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/destroy/all-resources/output.txt b/acceptance/bundle/destroy/all-resources/output.txt index 093fa6ac4f4..49bed70e906 100644 --- a/acceptance/bundle/destroy/all-resources/output.txt +++ b/acceptance/bundle/destroy/all-resources/output.txt @@ -14,7 +14,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.my_schema This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt index 4f32fbca70a..c5f958ff3cf 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt +++ b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt @@ -53,7 +53,7 @@ The following resources will be deleted: delete resources.pipelines.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.bar All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/migrate/runas/output.txt b/acceptance/bundle/migrate/runas/output.txt index 2108deb19ee..f58deae5c7c 100644 --- a/acceptance/bundle/migrate/runas/output.txt +++ b/acceptance/bundle/migrate/runas/output.txt @@ -138,7 +138,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt index ec276143f00..0d338041e65 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt +++ b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt @@ -108,7 +108,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resource_deps/remote_app_url/output.txt b/acceptance/bundle/resource_deps/remote_app_url/output.txt index 22fdee366fd..7f0ed6a6975 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/output.txt +++ b/acceptance/bundle/resource_deps/remote_app_url/output.txt @@ -90,7 +90,7 @@ The following resources will be deleted: delete resources.pipelines.mypipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.mypipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt index e0582581250..f5b15b7345e 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt @@ -30,7 +30,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt index 15f1750b315..6508649dd4c 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt @@ -18,7 +18,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt b/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt index ebf9541f2c1..cedb5f75f0d 100644 --- a/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt @@ -13,7 +13,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt index e0582581250..f5b15b7345e 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt @@ -30,7 +30,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt index 7a408e57076..e512edbbf53 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt @@ -22,7 +22,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/update/output.txt b/acceptance/bundle/resources/permissions/pipelines/update/output.txt index a898beb3387..4c576ab4d1c 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/update/output.txt @@ -123,7 +123,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/permissions-test/default diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt index cd7c8973d08..2ff2983a465 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt @@ -47,7 +47,7 @@ The following resources will be deleted: delete resources.pipelines.pipeline_one This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.pipeline_one All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/auto-approve/output.txt b/acceptance/bundle/resources/pipelines/auto-approve/output.txt index 165a5f22e32..3f793420787 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/output.txt +++ b/acceptance/bundle/resources/pipelines/auto-approve/output.txt @@ -63,7 +63,7 @@ The following resources will be deleted: delete resources.pipelines.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.bar All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt index 18382aa4a80..be82456d7c1 100644 --- a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt +++ b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt @@ -43,7 +43,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-lakeflow-pipeline-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt index ade49a90450..cb839853340 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt @@ -150,7 +150,7 @@ The following resources will be deleted: delete resources.pipelines.my This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt index 7c4bde4c4c5..43fc3727d26 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt @@ -134,7 +134,7 @@ The following resources will be deleted: delete resources.pipelines.my This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/recreate/output.txt b/acceptance/bundle/resources/pipelines/recreate/output.txt index 8f3a276e5ff..bd1304b9fdf 100644 --- a/acceptance/bundle/resources/pipelines/recreate/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate/output.txt @@ -65,7 +65,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/pipelines/update/output.txt b/acceptance/bundle/resources/pipelines/update/output.txt index 823d321cdf3..3cd4c3a9736 100644 --- a/acceptance/bundle/resources/pipelines/update/output.txt +++ b/acceptance/bundle/resources/pipelines/update/output.txt @@ -72,7 +72,7 @@ The following resources will be deleted: delete resources.pipelines.my This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/schemas/auto-approve/output.txt b/acceptance/bundle/resources/schemas/auto-approve/output.txt index 90f2dbebe98..d27e9a6a6c7 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/output.txt +++ b/acceptance/bundle/resources/schemas/auto-approve/output.txt @@ -72,7 +72,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/summary/modified_status/output.txt b/acceptance/bundle/summary/modified_status/output.txt index 47ee67480f5..9cb3e7b0517 100644 --- a/acceptance/bundle/summary/modified_status/output.txt +++ b/acceptance/bundle/summary/modified_status/output.txt @@ -189,7 +189,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.my_schema This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt index db5e5562b17..cc0c54574dc 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/output.txt +++ b/acceptance/bundle/templates/default-python/integration_classic/output.txt @@ -93,7 +93,7 @@ The following resources will be deleted: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev @@ -491,7 +491,7 @@ The following resources will be deleted: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod diff --git a/acceptance/pipelines/destroy/auto-approve/output.txt b/acceptance/pipelines/destroy/auto-approve/output.txt index 5cefb4144a5..8ab0e6e1eb9 100644 --- a/acceptance/pipelines/destroy/auto-approve/output.txt +++ b/acceptance/pipelines/destroy/auto-approve/output.txt @@ -19,7 +19,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy/default diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt index 8df542873ba..aeeeeba7c8c 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt @@ -48,7 +48,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines. -The Streaming Tables (STs) and Materialized Views (MVs) managed by them are retained (cascade_on_destroy: false): +The Streaming Tables (STs) and Materialized Views (MVs) managed by them are retained, as 'cascade_on_destroy' is set to false: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml index bb784ca3147..18d7f81b515 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml @@ -1,5 +1,4 @@ # cascade_on_destroy only affects the direct engine's delete request; the terraform engine -# does not support the attribute yet and rejects it (see the terraform-engine sub-test). -# Restrict this test to direct so the recorded request is meaningful. +# does not support the attribute yet and rejects it. Restrict this test to direct EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] RecordRequests = true diff --git a/acceptance/pipelines/destroy/destroy-pipeline/output.txt b/acceptance/pipelines/destroy/destroy-pipeline/output.txt index 3b0e50c9143..70cf60cff61 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline/output.txt +++ b/acceptance/pipelines/destroy/destroy-pipeline/output.txt @@ -12,7 +12,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy/default diff --git a/acceptance/pipelines/destroy/force-lock/output.txt b/acceptance/pipelines/destroy/force-lock/output.txt index fbc04f6dbac..f7cf597f67b 100644 --- a/acceptance/pipelines/destroy/force-lock/output.txt +++ b/acceptance/pipelines/destroy/force-lock/output.txt @@ -27,7 +27,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-force-lock/default diff --git a/acceptance/pipelines/e2e/output.txt b/acceptance/pipelines/e2e/output.txt index d450268a083..8148765cd12 100644 --- a/acceptance/pipelines/e2e/output.txt +++ b/acceptance/pipelines/e2e/output.txt @@ -106,7 +106,7 @@ The following resources will be deleted: delete resources.pipelines.lakeflow_project_etl_2 This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.lakeflow_project_etl delete resources.pipelines.lakeflow_project_etl_2 diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index f8f7f63ed4d..7c205f7dc14 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -18,9 +18,8 @@ type Pipeline struct { Permissions []PipelinePermission `json:"permissions,omitempty"` // CascadeOnDestroy controls whether destroying the pipeline also deletes its datasets - // (MVs, STs, Views). Nil (unset) uses the server default, which cascades. Set to false to - // retain the datasets when the pipeline is deleted. Mirrors purge_on_delete on Lakebase - // projects. Delete-time only: never sent on create/update. + // Nil (unset) uses the server default, which cascades. Set to false to retain datasets + // when the pipeline is deleted. Delete-time only: never sent on create/update. CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` } diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline.go b/bundle/deploy/terraform/tfdyn/convert_pipeline.go index e393f69c2c6..2eacbf714af 100644 --- a/bundle/deploy/terraform/tfdyn/convert_pipeline.go +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline.go @@ -22,8 +22,7 @@ func convertPipelineResource(ctx context.Context, vin dyn.Value) (dyn.Value, err } // Current Terraform provider does not support the cascade_on_destroy attribute yet. - // ValidateCascadeOnDestroy rejects it for the terraform engine before we get here; this - // drop is defense-in-depth so we never emit an unknown attribute to the provider schema. + // ValidateCascadeOnDestroy rejects it here and provides an explicit error message. vout, err = dyn.DropKeys(vout, []string{"dry_run", "cascade_on_destroy"}) if err != nil { return dyn.InvalidValue, err diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index 7d01554258c..8b808e203e5 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -163,8 +163,7 @@ func (r *ResourcePipeline) DoCreate(ctx context.Context, config *PipelineState) func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *PipelineState, entry *PlanEntry) (*PipelineRemote, error) { // cascade_on_destroy is a delete-time-only setting with no update API, so a change to it - // alone must persist to state without a pipeline Update call. This mirrors sql_warehouse's - // handling of its non-spec lifecycle field. + // alone must persist to state without a pipeline Update call. if !entry.Changes.HasChangeExcept("cascade_on_destroy") { return nil, nil } @@ -215,13 +214,12 @@ func (r *ResourcePipeline) DoDelete(ctx context.Context, id string, state *Pipel // No explicit cascade_on_destroy in config: preserve the backend default (cascade). return r.client.Pipelines.DeleteByPipelineId(ctx, id) } - // Cascade is marshaled as `url:"cascade,omitempty"`, so a false value would be dropped from - // the query string and the backend would apply its default of true. ForceSendFields makes - // the SDK send cascade=false explicitly so cascade_on_destroy: false is honored. return r.client.Pipelines.Delete(ctx, pipelines.DeletePipelineRequest{ - PipelineId: id, - Cascade: *state.CascadeOnDestroy, - Force: false, + PipelineId: id, + Cascade: *state.CascadeOnDestroy, + Force: false, + // Cascade is marshaled as `url:"cascade,omitempty"`, so a false value would be dropped from + // the query string. We specify `cascade` in ForceSendFields so the SDK send cascade=false explicitly ForceSendFields: []string{"Cascade"}, }) } diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 0084777485d..fb97d38c11b 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -46,17 +46,17 @@ var destroyApprovalGroups = []approvalGroup{ {group: "genie_spaces", message: deleteGenieSpaceMessage}, } -// logPipelineDeleteApproval prints the pipeline deletions, splitting them by cascade_on_destroy -// so pipelines retaining their datasets are not described as deleting STs/MVs. +// logPipelineDeleteApproval prints the pipeline deletions. If cascade_on_destroy is true, we will include +// a note that datasets will be deleted as well. func logPipelineDeleteApproval(ctx context.Context, b *bundle.Bundle, actions []deployplan.Action) { pipelineDeletes := filterGroup(actions, "pipelines", deployplan.Delete) var cascading, retaining []deployplan.Action for _, a := range pipelineDeletes { - if pipelineRetainsDatasets(b, a) { - retaining = append(retaining, a) - } else { + if pipelineDeletionCascades(b, a) { cascading = append(cascading, a) + } else { + retaining = append(retaining, a) } } @@ -64,7 +64,7 @@ func logPipelineDeleteApproval(ctx context.Context, b *bundle.Bundle, actions [] message string actions []deployplan.Action }{ - {deletePipelineMessage, cascading}, + {deletePipelineWithCascadeMessage, cascading}, {deletePipelineNoCascadeMessage, retaining}, } { if len(grp.actions) == 0 { diff --git a/bundle/phases/messages.go b/bundle/phases/messages.go index 28bed70c37e..388c0faa067 100644 --- a/bundle/phases/messages.go +++ b/bundle/phases/messages.go @@ -59,11 +59,11 @@ const DataLossWarning = "Deleting data assets such as schemas, pipelines, or vol const ( deleteSchemaMessage = `This action will result in the deletion of the following UC schemas. Any underlying data may be lost:` - deletePipelineMessage = `This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain its datasets:` + deletePipelineWithCascadeMessage = `This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion:` deletePipelineNoCascadeMessage = `This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines. -The Streaming Tables (STs) and Materialized Views (MVs) managed by them are retained (cascade_on_destroy: false):` +The Streaming Tables (STs) and Materialized Views (MVs) managed by them are retained, as 'cascade_on_destroy' is set to false:` deleteVolumeMessage = `This action will result in the deletion of the following volumes. For managed volumes, the files stored in the volume are also deleted from your diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index b20ae25e8fc..80997f4ebf9 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -31,23 +31,24 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine ) } -// pipelineRetainsDatasets reports whether the pipeline referenced by a delete action has -// cascade_on_destroy explicitly set to false, meaning its datasets (MVs, STs, Views) are -// retained. Only pipeline resources carry this field, so a lookup miss returns false. -// Reads from config like checkForPreventDestroy; the field is unset for other resource types. -func pipelineRetainsDatasets(b *bundle.Bundle, action deployplan.Action) bool { +// pipelineDeletionCascades reports whether deleting the pipeline referenced by a delete action +// also deletes its datasets (MVs, STs, Views). This is the server default (cascade) unless +// cascade_on_destroy is explicitly set to false. A lookup miss (unset field, or a non-pipeline +// resource) means the default applies, so it returns true. +// Reads from config like checkForPreventDestroy. +func pipelineDeletionCascades(b *bundle.Bundle, action deployplan.Action) bool { path, err := dyn.NewPathFromString(action.ResourceKey) if err != nil { - return false + return true } path = append(path, dyn.Key("cascade_on_destroy")) v, err := dyn.GetByPath(b.Config.Value(), path) if err != nil { - return false + return true } cascade, ok := v.AsBool() - return ok && !cascade + return !ok || cascade } // checkForPreventDestroy checks if the resource has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed.