From 8746a37820575c4b3b9da73a91ebb8ff81230522 Mon Sep 17 00:00:00 2001 From: Ronald Zhang Date: Wed, 8 Jul 2026 00:26:33 +0000 Subject: [PATCH] Add cascade field to pipeline resource for destroy Add a `cascade` field to the pipeline bundle resource that controls whether destroying a pipeline also deletes its datasets (materialized views, streaming tables, and views). When unset, the server default applies (cascade); set `cascade: false` to retain the datasets on destroy. The field is delete-time only: it is not part of the pipeline spec, so it is never sent on create/update. The direct engine persists it in state (via a PipelineState wrapper around CreatePipeline, mirroring the sql_warehouse lifecycle pattern) and reads it at delete time, force-sending cascade so an explicit false survives query-string omitempty (honored for query params as of databricks-sdk-go v0.152.0). A cascade-only change is a state-only update with no pipeline API call. cascade is classified input_only so it does not show remote drift. The terraform engine drops the field for now: the pinned provider has no such attribute yet (pending terraform-provider-databricks#5860), so terraform-engine support is a follow-up. Co-authored-by: Isaac --- NEXT_CHANGELOG.md | 1 + acceptance/bundle/refschema/out.fields.txt | 1 + .../destroy-pipeline-cascade/databricks.yml | 7 ++ .../destroy-pipeline-cascade/out.test.toml | 3 + .../destroy-pipeline-cascade/output.txt | 56 ++++++++++++++++ .../destroy/destroy-pipeline-cascade/script | 15 +++++ .../destroy-pipeline-cascade/test.toml | 5 ++ bundle/config/resources/pipeline.go | 4 ++ .../terraform/tfdyn/convert_pipeline.go | 3 +- bundle/direct/dresources/pipeline.go | 65 ++++++++++++++++--- bundle/direct/dresources/resources.yml | 4 ++ bundle/schema/jsonschema.json | 6 +- 12 files changed, 158 insertions(+), 12 deletions(-) create mode 100644 acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml create mode 100644 acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml create mode 100644 acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt create mode 100644 acceptance/pipelines/destroy/destroy-pipeline-cascade/script create mode 100644 acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 023c1d422d5..c1930b242df 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -13,6 +13,7 @@ * Bundle variable references now accept Unicode letters in path segments (e.g. `${var.变量}`). ([#5532](https://github.com/databricks/cli/pull/5532)) * Ignore remote changes for vector search direct_access_index_spec.schema_json to prevent drift when the backend normalizes the schema ([#5481](https://github.com/databricks/cli/pull/5481)). * Remove hidden, never-functional `--existing-dashboard-id`, `--existing-dashboard-path`, `--existing-alert-id`, and `--existing-genie-space-id` alias flags from `bundle generate`; use the documented `--existing-id` / `--existing-path` flags instead ([#5591](https://github.com/databricks/cli/pull/5591)). +* Added a `cascade` field to the pipeline resource to control whether destroying a pipeline also cascades to its datasets. When unset, the server default applies; set `cascade: false` to retain the datasets on destroy. ### Dependency updates diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 80b553561a5..ce22ba07a40 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -2234,6 +2234,7 @@ resources.models.*.permissions[*].service_principal_name string ALL resources.models.*.permissions[*].user_name string ALL resources.pipelines.*.allow_duplicate_names bool ALL resources.pipelines.*.budget_policy_id string ALL +resources.pipelines.*.cascade *bool ALL resources.pipelines.*.catalog string ALL resources.pipelines.*.cause string REMOTE resources.pipelines.*.channel string ALL diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml new file mode 100644 index 00000000000..5e0b4f9b9c1 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-pipeline-destroy-cascade +resources: + pipelines: + my_pipeline: + name: test-pipeline-cascade + cascade: false diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt new file mode 100644 index 00000000000..d13113ef0f7 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt @@ -0,0 +1,56 @@ + +=== Deploy a pipeline configured with cascade: false +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] + +>>> print_requests.py //api/2.0/pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/state/metadata.json" + }, + "edition": "ADVANCED", + "name": "test-pipeline-cascade" + } +} + +=== Change only cascade (false -> true): expect a state-only update, no pipeline update API call +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] + +>>> print_requests.py //api/2.0/pipelines + +=== Destroy: delete reads the updated cascade=true from persisted state +>>> [CLI] pipelines destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.my_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them: + delete resources.pipelines.my_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //api/2.0/pipelines +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[UUID]", + "q": { + "cascade": "true" + } +} diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/script b/acceptance/pipelines/destroy/destroy-pipeline-cascade/script new file mode 100644 index 00000000000..e3124520050 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/script @@ -0,0 +1,15 @@ +title "Deploy a pipeline configured with cascade: false" +trace $CLI pipelines deploy + +# Flush the deploy requests: cascade is delete-time-only, so the create request must not carry it. +trace print_requests.py //api/2.0/pipelines + +title "Change only cascade (false -> true): expect a state-only update, no pipeline update API call" +update_file.py databricks.yml "cascade: false" "cascade: true" +trace $CLI pipelines deploy +# No POST/PATCH to /pipelines is expected here; the cascade-only change is persisted to state. +trace print_requests.py //api/2.0/pipelines + +title "Destroy: delete reads the updated cascade=true from persisted state" +trace $CLI pipelines destroy --auto-approve +trace print_requests.py //api/2.0/pipelines diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml new file mode 100644 index 00000000000..bd95018f1c6 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml @@ -0,0 +1,5 @@ +# --cascade only affects the direct engine's delete request; the terraform engine deletes +# pipelines through the provider and ignores it. Restrict to direct so the recorded request +# is meaningful and the single variant does not diverge. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index c442a01a641..7fc64dde896 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -16,6 +16,10 @@ type Pipeline struct { pipelines.CreatePipeline //nolint CreatePipeline also defines Id field with the same json tag "id" Permissions []PipelinePermission `json:"permissions,omitempty"` + + // Optional parameter that controls whether destroying the pipeline also cascades to datasets. + // By default, the server will cascade the pipeline deletion to its datasets. + Cascade *bool `json:"cascade,omitempty"` } func (p *Pipeline) UnmarshalJSON(b []byte) error { diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline.go b/bundle/deploy/terraform/tfdyn/convert_pipeline.go index 944ce0858c7..04b652e16c5 100644 --- a/bundle/deploy/terraform/tfdyn/convert_pipeline.go +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline.go @@ -21,7 +21,8 @@ func convertPipelineResource(ctx context.Context, vin dyn.Value) (dyn.Value, err return dyn.InvalidValue, err } - vout, err = dyn.DropKeys(vout, []string{"dry_run"}) + // Current Terraform provider does not support the cascade attribute yet + vout, err = dyn.DropKeys(vout, []string{"dry_run", "cascade"}) if err != nil { return dyn.InvalidValue, err } diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index 0861f0fad2f..dea18aeafd3 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -10,11 +10,32 @@ import ( "github.com/databricks/databricks-sdk-go/service/pipelines" ) +// PipelineState is the state type for Pipeline resources. It extends CreatePipeline with +// the Cascade field, a delete-time setting that is not part of the pipeline spec +type PipelineState struct { + pipelines.CreatePipeline + + // Cascade controls whether deleting the pipeline also deletes its datasets (MVs, STs, + // Views). Nil means the server default (cascade) applies. Read from persisted state at + // delete time; never sent on create/update. + Cascade *bool `json:"cascade,omitempty"` +} + +func (s *PipelineState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s PipelineState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + // PipelineRemote is the return type for DoRead. It embeds CreatePipeline so that all // paths in StateType are valid paths in RemoteType. type PipelineRemote struct { pipelines.CreatePipeline + Cascade *bool `json:"cascade,omitempty"` + // Remote-specific fields from pipelines.GetPipelineResponse Cause string `json:"cause,omitempty"` ClusterId string `json:"cluster_id,omitempty"` @@ -49,12 +70,18 @@ func (*ResourcePipeline) New(client *databricks.WorkspaceClient) *ResourcePipeli } } -func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *pipelines.CreatePipeline { - return &input.CreatePipeline +func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *PipelineState { + return &PipelineState{ + CreatePipeline: input.CreatePipeline, + Cascade: input.Cascade, + } } -func (*ResourcePipeline) RemapState(remote *PipelineRemote) *pipelines.CreatePipeline { - return &remote.CreatePipeline +func (*ResourcePipeline) RemapState(remote *PipelineRemote) *PipelineState { + return &PipelineState{ + CreatePipeline: remote.CreatePipeline, + Cascade: remote.Cascade, + } } func (r *ResourcePipeline) DoRead(ctx context.Context, id string) (*PipelineRemote, error) { @@ -108,7 +135,9 @@ func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { } } return &PipelineRemote{ - CreatePipeline: createPipeline, + CreatePipeline: createPipeline, + // cascade is input-only; the GET response never carries it, so leave it nil. + Cascade: nil, Cause: p.Cause, ClusterId: p.ClusterId, CreatorUserName: p.CreatorUserName, @@ -123,15 +152,22 @@ func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { } } -func (r *ResourcePipeline) DoCreate(ctx context.Context, config *pipelines.CreatePipeline) (string, *PipelineRemote, error) { - response, err := r.client.Pipelines.Create(ctx, *config) +func (r *ResourcePipeline) DoCreate(ctx context.Context, config *PipelineState) (string, *PipelineRemote, error) { + response, err := r.client.Pipelines.Create(ctx, config.CreatePipeline) if err != nil { return "", nil, err } return response.PipelineId, nil, nil } -func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *pipelines.CreatePipeline, _ *PlanEntry) (*PipelineRemote, error) { +func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *PipelineState, entry *PlanEntry) (*PipelineRemote, error) { + // cascade is a delete-time-only setting with no update API, so a change to it alone must + // persist to state without a pipeline Update call. This mirrors sql_warehouse's handling + // of its non-spec lifecycle field. + if !entry.Changes.HasChangeExcept("cascade") { + return nil, nil + } + request := pipelines.EditPipeline{ AllowDuplicateNames: config.AllowDuplicateNames, BudgetPolicyId: config.BudgetPolicyId, @@ -172,8 +208,17 @@ func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *pipe return nil, r.client.Pipelines.Update(ctx, request) } -func (r *ResourcePipeline) DoDelete(ctx context.Context, id string, _ *pipelines.CreatePipeline) error { - return r.client.Pipelines.DeleteByPipelineId(ctx, id) +func (r *ResourcePipeline) DoDelete(ctx context.Context, id string, state *PipelineState) error { + if state.Cascade == nil { + // No explicit cascade in config: preserve the backend default (cascade). + return r.client.Pipelines.DeleteByPipelineId(ctx, id) + } + return r.client.Pipelines.Delete(ctx, pipelines.DeletePipelineRequest{ + PipelineId: id, + Cascade: *state.Cascade, + Force: false, + ForceSendFields: []string{"Cascade"}, + }) } // Note, terraform provider either diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 9c12da9f39b..9df68f0d4e8 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -126,6 +126,10 @@ resources: reason: "!drop" - field: run_as reason: input_only + # cascade is a delete-time-only setting + # it is never returned by GET, so suppress remote drift for it. + - field: cascade + reason: input_only ignore_local_changes: # "id" is output-only, providing it in config would be a mistake diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index f88a9389348..81209c8fa76 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1272,6 +1272,10 @@ "description": "[Public Preview] Budget policy of this pipeline.", "$ref": "#/$defs/string" }, + "cascade": { + "description": "Whether destroying the pipeline also cascades to its datasets. Defaults to true. Set to false to keep the datasets when the pipeline is deleted. Only affects the delete operation.", + "$ref": "#/$defs/bool" + }, "catalog": { "description": "A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog.", "$ref": "#/$defs/string" @@ -14062,4 +14066,4 @@ } }, "additionalProperties": {} -} \ No newline at end of file +}