diff --git a/.vscode/settings.json b/.vscode/settings.json index b393450c80d..4f9fae2d8e2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -28,5 +28,8 @@ "databricks.yml", "databricks.yaml", ] + }, + "files.watcherExclude": { + "**/target": true } } diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 30a64c54100..d67d0387ab6 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -7,6 +7,7 @@ ### CLI ### Bundles +* Added a `cascade_on_destroy` field to the pipeline resource to control whether destroying a pipeline also deletes its datasets (MVs, STs, Views). When unset, the server default applies; set `cascade_on_destroy: false` to retain the datasets on destroy. Supported with the direct deployment engine. ### Dependency updates diff --git a/acceptance/bundle/config-remote-sync/cli_defaults/output.txt b/acceptance/bundle/config-remote-sync/cli_defaults/output.txt index 307a945ea93..f3e2372b81a 100644 --- a/acceptance/bundle/config-remote-sync/cli_defaults/output.txt +++ b/acceptance/bundle/config-remote-sync/cli_defaults/output.txt @@ -57,7 +57,7 @@ The following resources will be deleted: delete resources.pipelines.pipeline1 This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.pipeline1 All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt b/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt index 54a1e342788..52e37685043 100644 --- a/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt +++ b/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt @@ -41,7 +41,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt b/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt index e14b94f76f5..19b20f8befa 100644 --- a/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt +++ b/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt @@ -60,7 +60,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/resolve_variables/output.txt b/acceptance/bundle/config-remote-sync/resolve_variables/output.txt index 388f7ab3bbf..8cb8849da4d 100644 --- a/acceptance/bundle/config-remote-sync/resolve_variables/output.txt +++ b/acceptance/bundle/config-remote-sync/resolve_variables/output.txt @@ -143,7 +143,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/validation_errors/output.txt b/acceptance/bundle/config-remote-sync/validation_errors/output.txt index c4ce29efa0a..372e4fa1ae8 100644 --- a/acceptance/bundle/config-remote-sync/validation_errors/output.txt +++ b/acceptance/bundle/config-remote-sync/validation_errors/output.txt @@ -51,7 +51,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/deploy/pipeline-config-dots/output.txt b/acceptance/bundle/deploy/pipeline-config-dots/output.txt index 48b6576f221..8b338fcfbb3 100644 --- a/acceptance/bundle/deploy/pipeline-config-dots/output.txt +++ b/acceptance/bundle/deploy/pipeline-config-dots/output.txt @@ -23,7 +23,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.my_schema This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/deploy/snapshot-comparison/output.txt b/acceptance/bundle/deploy/snapshot-comparison/output.txt index b5bb597e264..c908a5fa61a 100644 --- a/acceptance/bundle/deploy/snapshot-comparison/output.txt +++ b/acceptance/bundle/deploy/snapshot-comparison/output.txt @@ -43,7 +43,7 @@ The following resources will be deleted: delete resources.pipelines.test_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.test_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/snapshot-test-1-[UNIQUE_NAME]/default @@ -58,7 +58,7 @@ The following resources will be deleted: delete resources.pipelines.test_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.test_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/snapshot-test-2-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/destroy/all-resources/output.txt b/acceptance/bundle/destroy/all-resources/output.txt index 5fd8d432a82..49bed70e906 100644 --- a/acceptance/bundle/destroy/all-resources/output.txt +++ b/acceptance/bundle/destroy/all-resources/output.txt @@ -14,7 +14,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.my_schema This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt index ce1530c469a..c5f958ff3cf 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt +++ b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt @@ -53,7 +53,7 @@ The following resources will be deleted: delete resources.pipelines.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.bar All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/migrate/runas/output.txt b/acceptance/bundle/migrate/runas/output.txt index f8a9b475bdc..f58deae5c7c 100644 --- a/acceptance/bundle/migrate/runas/output.txt +++ b/acceptance/bundle/migrate/runas/output.txt @@ -138,7 +138,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 7f9cf9c94f5..e6afc1bfd68 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -2273,6 +2273,7 @@ resources.models.*.permissions[*].service_principal_name string ALL resources.models.*.permissions[*].user_name string ALL resources.pipelines.*.allow_duplicate_names bool ALL resources.pipelines.*.budget_policy_id string ALL +resources.pipelines.*.cascade_on_destroy *bool ALL resources.pipelines.*.catalog string ALL resources.pipelines.*.cause string REMOTE resources.pipelines.*.channel string ALL diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt index b7eec30702b..0d338041e65 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt +++ b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt @@ -108,7 +108,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resource_deps/remote_app_url/output.txt b/acceptance/bundle/resource_deps/remote_app_url/output.txt index 4c51088f686..7f0ed6a6975 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/output.txt +++ b/acceptance/bundle/resource_deps/remote_app_url/output.txt @@ -90,7 +90,7 @@ The following resources will be deleted: delete resources.pipelines.mypipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.mypipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt index 2e15e6e4108..f5b15b7345e 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt @@ -30,7 +30,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt index 89f7921475b..6508649dd4c 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt @@ -18,7 +18,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt b/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt index 2f3ce99457f..cedb5f75f0d 100644 --- a/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt @@ -13,7 +13,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt index 2e15e6e4108..f5b15b7345e 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt @@ -30,7 +30,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt index 3b82b04aa94..e512edbbf53 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt @@ -22,7 +22,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/update/output.txt b/acceptance/bundle/resources/permissions/pipelines/update/output.txt index e5395eeceea..4c576ab4d1c 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/update/output.txt @@ -123,7 +123,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/permissions-test/default diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt index 8cea9634565..2ff2983a465 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt @@ -47,7 +47,7 @@ The following resources will be deleted: delete resources.pipelines.pipeline_one This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.pipeline_one All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/auto-approve/output.txt b/acceptance/bundle/resources/pipelines/auto-approve/output.txt index 3016a4c248d..3f793420787 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/output.txt +++ b/acceptance/bundle/resources/pipelines/auto-approve/output.txt @@ -63,7 +63,7 @@ The following resources will be deleted: delete resources.pipelines.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.bar All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt index 04129e78ce1..be82456d7c1 100644 --- a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt +++ b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt @@ -43,7 +43,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-lakeflow-pipeline-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt index 77bfdc09d32..cb839853340 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt @@ -150,7 +150,7 @@ The following resources will be deleted: delete resources.pipelines.my This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt index 74943571027..43fc3727d26 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt @@ -134,7 +134,7 @@ The following resources will be deleted: delete resources.pipelines.my This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/recreate/output.txt b/acceptance/bundle/resources/pipelines/recreate/output.txt index b6a280d38d5..bd1304b9fdf 100644 --- a/acceptance/bundle/resources/pipelines/recreate/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate/output.txt @@ -65,7 +65,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/pipelines/update/output.txt b/acceptance/bundle/resources/pipelines/update/output.txt index 213e8a0cc64..3cd4c3a9736 100644 --- a/acceptance/bundle/resources/pipelines/update/output.txt +++ b/acceptance/bundle/resources/pipelines/update/output.txt @@ -72,7 +72,7 @@ The following resources will be deleted: delete resources.pipelines.my This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/schemas/auto-approve/output.txt b/acceptance/bundle/resources/schemas/auto-approve/output.txt index 375c94fe652..d27e9a6a6c7 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/output.txt +++ b/acceptance/bundle/resources/schemas/auto-approve/output.txt @@ -72,7 +72,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/summary/modified_status/output.txt b/acceptance/bundle/summary/modified_status/output.txt index 92338e101c9..9cb3e7b0517 100644 --- a/acceptance/bundle/summary/modified_status/output.txt +++ b/acceptance/bundle/summary/modified_status/output.txt @@ -189,7 +189,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.my_schema This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt index 2206b10cd25..cc0c54574dc 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/output.txt +++ b/acceptance/bundle/templates/default-python/integration_classic/output.txt @@ -93,7 +93,7 @@ The following resources will be deleted: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev @@ -491,7 +491,7 @@ The following resources will be deleted: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod diff --git a/acceptance/pipelines/destroy/auto-approve/output.txt b/acceptance/pipelines/destroy/auto-approve/output.txt index 76bf97b4e2f..8ab0e6e1eb9 100644 --- a/acceptance/pipelines/destroy/auto-approve/output.txt +++ b/acceptance/pipelines/destroy/auto-approve/output.txt @@ -19,7 +19,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy/default diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/databricks.yml b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/databricks.yml new file mode 100644 index 00000000000..0c4f487853b --- /dev/null +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-cascade-on-destroy-tf-error +resources: + pipelines: + my_pipeline: + name: test-pipeline-cascade-tf + cascade_on_destroy: false diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/out.test.toml b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/out.test.toml new file mode 100644 index 00000000000..65156e0457c --- /dev/null +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/output.txt b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/output.txt new file mode 100644 index 00000000000..0763f5a218b --- /dev/null +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/output.txt @@ -0,0 +1,8 @@ + +=== pipelines deploy fails with cascade_on_destroy on terraform engine +>>> errcode [CLI] pipelines deploy +Error: cascade_on_destroy is only supported in direct deployment mode + in databricks.yml:7:27 + + +Exit code: 1 diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/script b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/script new file mode 100644 index 00000000000..ba05370f088 --- /dev/null +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/script @@ -0,0 +1,2 @@ +title "pipelines deploy fails with cascade_on_destroy on terraform engine" +trace errcode $CLI pipelines deploy diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/test.toml b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/test.toml new file mode 100644 index 00000000000..ec557241780 --- /dev/null +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/test.toml @@ -0,0 +1,8 @@ +Local = true +RecordRequests = false + +# Static validation only; no need to run on cloud. +Cloud = false + +# cascade_on_destroy is direct-only; assert the terraform engine rejects it. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml new file mode 100644 index 00000000000..af159a9e9c1 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-pipeline-destroy-cascade +resources: + pipelines: + my_pipeline: + name: test-pipeline-cascade + cascade_on_destroy: false diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt new file mode 100644 index 00000000000..aeeeeba7c8c --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt @@ -0,0 +1,66 @@ + +=== Deploy a pipeline configured with cascade_on_destroy: false +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] + +>>> print_requests.py //api/2.0/pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/state/metadata.json" + }, + "edition": "ADVANCED", + "name": "test-pipeline-cascade" + } +} + +=== Change only cascade_on_destroy (false -> true): expect a state-only update, no pipeline update API call +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] + +>>> print_requests.py //api/2.0/pipelines + +=== Set cascade_on_destroy back to false and deploy so state reflects it before destroy +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] + +>>> print_requests.py //api/2.0/pipelines + +=== Destroy: approval message notes datasets are retained, and delete sends cascade=false +>>> [CLI] pipelines destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.my_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines. +The Streaming Tables (STs) and Materialized Views (MVs) managed by them are retained, as 'cascade_on_destroy' is set to false: + delete resources.pipelines.my_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //api/2.0/pipelines +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[UUID]", + "q": { + "cascade": "false" + } +} diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/script b/acceptance/pipelines/destroy/destroy-pipeline-cascade/script new file mode 100644 index 00000000000..8a8ad36eca4 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/script @@ -0,0 +1,20 @@ +title "Deploy a pipeline configured with cascade_on_destroy: false" +trace $CLI pipelines deploy + +# Flush the deploy requests: cascade_on_destroy is delete-time-only, so the create request must not carry it. +trace print_requests.py //api/2.0/pipelines + +title "Change only cascade_on_destroy (false -> true): expect a state-only update, no pipeline update API call" +update_file.py databricks.yml "cascade_on_destroy: false" "cascade_on_destroy: true" +trace $CLI pipelines deploy +# No POST/PATCH to /pipelines is expected here; the cascade-only change is persisted to state. +trace print_requests.py //api/2.0/pipelines + +title "Set cascade_on_destroy back to false and deploy so state reflects it before destroy" +update_file.py databricks.yml "cascade_on_destroy: true" "cascade_on_destroy: false" +trace $CLI pipelines deploy +trace print_requests.py //api/2.0/pipelines + +title "Destroy: approval message notes datasets are retained, and delete sends cascade=false" +trace $CLI pipelines destroy --auto-approve +trace print_requests.py //api/2.0/pipelines diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml new file mode 100644 index 00000000000..18d7f81b515 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml @@ -0,0 +1,4 @@ +# cascade_on_destroy only affects the direct engine's delete request; the terraform engine +# does not support the attribute yet and rejects it. Restrict this test to direct +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true diff --git a/acceptance/pipelines/destroy/destroy-pipeline/output.txt b/acceptance/pipelines/destroy/destroy-pipeline/output.txt index 4e5ab020852..70cf60cff61 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline/output.txt +++ b/acceptance/pipelines/destroy/destroy-pipeline/output.txt @@ -12,7 +12,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy/default diff --git a/acceptance/pipelines/destroy/force-lock/output.txt b/acceptance/pipelines/destroy/force-lock/output.txt index f253d542a91..f7cf597f67b 100644 --- a/acceptance/pipelines/destroy/force-lock/output.txt +++ b/acceptance/pipelines/destroy/force-lock/output.txt @@ -27,7 +27,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-force-lock/default diff --git a/acceptance/pipelines/e2e/output.txt b/acceptance/pipelines/e2e/output.txt index 8e029977b23..8148765cd12 100644 --- a/acceptance/pipelines/e2e/output.txt +++ b/acceptance/pipelines/e2e/output.txt @@ -106,7 +106,7 @@ The following resources will be deleted: delete resources.pipelines.lakeflow_project_etl_2 This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.lakeflow_project_etl delete resources.pipelines.lakeflow_project_etl_2 diff --git a/bundle/config/mutator/validate_cascade_on_destroy.go b/bundle/config/mutator/validate_cascade_on_destroy.go new file mode 100644 index 00000000000..80c5bcc0588 --- /dev/null +++ b/bundle/config/mutator/validate_cascade_on_destroy.go @@ -0,0 +1,45 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/libs/diag" +) + +type validateCascadeOnDestroy struct { + engine engine.EngineType +} + +// ValidateCascadeOnDestroy returns a mutator that errors when a pipeline sets +// cascade_on_destroy while using the terraform deployment engine. The terraform +// provider does not support the attribute yet, so it is only honored in direct +// deployment mode. +func ValidateCascadeOnDestroy(e engine.EngineType) bundle.Mutator { + return &validateCascadeOnDestroy{engine: e} +} + +func (m *validateCascadeOnDestroy) Name() string { + return "ValidateCascadeOnDestroy" +} + +func (m *validateCascadeOnDestroy) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + if m.engine.IsDirect() { + return nil + } + + var diags diag.Diagnostics + for key, pipeline := range b.Config.Resources.Pipelines { + if pipeline.CascadeOnDestroy == nil { + continue + } + path := "resources.pipelines." + key + ".cascade_on_destroy" + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "cascade_on_destroy is only supported in direct deployment mode", + Locations: b.Config.GetLocations(path), + }) + } + return diags +} diff --git a/bundle/config/mutator/validate_cascade_on_destroy_test.go b/bundle/config/mutator/validate_cascade_on_destroy_test.go new file mode 100644 index 00000000000..6b01ba3987b --- /dev/null +++ b/bundle/config/mutator/validate_cascade_on_destroy_test.go @@ -0,0 +1,68 @@ +package mutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" +) + +func pipelineWithCascade(cascade *bool) *resources.Pipeline { + return &resources.Pipeline{ + CreatePipeline: pipelines.CreatePipeline{Name: "my_pipeline"}, + CascadeOnDestroy: cascade, + } +} + +func TestValidateCascadeOnDestroyDirectEngineReturnsNil(t *testing.T) { + no := false + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "my_pipeline": pipelineWithCascade(&no), + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateCascadeOnDestroy(engine.EngineDirect)) + assert.Empty(t, diags) +} + +func TestValidateCascadeOnDestroyTerraformEngineUnsetReturnsNil(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "my_pipeline": pipelineWithCascade(nil), + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateCascadeOnDestroy(engine.EngineTerraform)) + assert.Empty(t, diags) +} + +func TestValidateCascadeOnDestroyTerraformEngineSetEmitsError(t *testing.T) { + no := false + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "my_pipeline": pipelineWithCascade(&no), + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateCascadeOnDestroy(engine.EngineTerraform)) + assert.Len(t, diags, 1) + assert.Equal(t, "cascade_on_destroy is only supported in direct deployment mode", diags[0].Summary) +} diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index c442a01a641..7c205f7dc14 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -16,6 +16,11 @@ type Pipeline struct { pipelines.CreatePipeline //nolint CreatePipeline also defines Id field with the same json tag "id" Permissions []PipelinePermission `json:"permissions,omitempty"` + + // CascadeOnDestroy controls whether destroying the pipeline also deletes its datasets + // Nil (unset) uses the server default, which cascades. Set to false to retain datasets + // when the pipeline is deleted. Delete-time only: never sent on create/update. + CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` } func (p *Pipeline) UnmarshalJSON(b []byte) error { diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline.go b/bundle/deploy/terraform/tfdyn/convert_pipeline.go index 944ce0858c7..2eacbf714af 100644 --- a/bundle/deploy/terraform/tfdyn/convert_pipeline.go +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline.go @@ -21,7 +21,9 @@ func convertPipelineResource(ctx context.Context, vin dyn.Value) (dyn.Value, err return dyn.InvalidValue, err } - vout, err = dyn.DropKeys(vout, []string{"dry_run"}) + // Current Terraform provider does not support the cascade_on_destroy attribute yet. + // ValidateCascadeOnDestroy rejects it here and provides an explicit error message. + vout, err = dyn.DropKeys(vout, []string{"dry_run", "cascade_on_destroy"}) if err != nil { return dyn.InvalidValue, err } diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index 20d2eb08245..8b808e203e5 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -10,11 +10,32 @@ import ( "github.com/databricks/databricks-sdk-go/service/pipelines" ) +// PipelineState is the state type for Pipeline resources. It extends CreatePipeline with +// the CascadeOnDestroy field, a delete-time setting that is not part of the pipeline spec +type PipelineState struct { + pipelines.CreatePipeline + + // CascadeOnDestroy controls whether deleting the pipeline also deletes its datasets (MVs, + // STs, Views). Nil means the server default (cascade) applies. Read from persisted state at + // delete time; never sent on create/update. + CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` +} + +func (s *PipelineState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s PipelineState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + // PipelineRemote is the return type for DoRead. It embeds CreatePipeline so that all // paths in StateType are valid paths in RemoteType. type PipelineRemote struct { pipelines.CreatePipeline + CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` + // Remote-specific fields from pipelines.GetPipelineResponse Cause string `json:"cause,omitempty"` ClusterId string `json:"cluster_id,omitempty"` @@ -49,12 +70,18 @@ func (*ResourcePipeline) New(client *databricks.WorkspaceClient) *ResourcePipeli } } -func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *pipelines.CreatePipeline { - return &input.CreatePipeline +func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *PipelineState { + return &PipelineState{ + CreatePipeline: input.CreatePipeline, + CascadeOnDestroy: input.CascadeOnDestroy, + } } -func (*ResourcePipeline) RemapState(remote *PipelineRemote) *pipelines.CreatePipeline { - return &remote.CreatePipeline +func (*ResourcePipeline) RemapState(remote *PipelineRemote) *PipelineState { + return &PipelineState{ + CreatePipeline: remote.CreatePipeline, + CascadeOnDestroy: remote.CascadeOnDestroy, + } } func (r *ResourcePipeline) DoRead(ctx context.Context, id string) (*PipelineRemote, error) { @@ -109,7 +136,9 @@ func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { } } return &PipelineRemote{ - CreatePipeline: createPipeline, + CreatePipeline: createPipeline, + // cascade_on_destroy is input-only; the GET response never carries it, so leave it nil. + CascadeOnDestroy: nil, Cause: p.Cause, ClusterId: p.ClusterId, CreatorUserName: p.CreatorUserName, @@ -124,15 +153,21 @@ func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { } } -func (r *ResourcePipeline) DoCreate(ctx context.Context, config *pipelines.CreatePipeline) (string, *PipelineRemote, error) { - response, err := r.client.Pipelines.Create(ctx, *config) +func (r *ResourcePipeline) DoCreate(ctx context.Context, config *PipelineState) (string, *PipelineRemote, error) { + response, err := r.client.Pipelines.Create(ctx, config.CreatePipeline) if err != nil { return "", nil, err } return response.PipelineId, nil, nil } -func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *pipelines.CreatePipeline, _ *PlanEntry) (*PipelineRemote, error) { +func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *PipelineState, entry *PlanEntry) (*PipelineRemote, error) { + // cascade_on_destroy is a delete-time-only setting with no update API, so a change to it + // alone must persist to state without a pipeline Update call. + if !entry.Changes.HasChangeExcept("cascade_on_destroy") { + return nil, nil + } + request := pipelines.EditPipeline{ AllowDuplicateNames: config.AllowDuplicateNames, BudgetPolicyId: config.BudgetPolicyId, @@ -174,8 +209,19 @@ func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *pipe return nil, r.client.Pipelines.Update(ctx, request) } -func (r *ResourcePipeline) DoDelete(ctx context.Context, id string, _ *pipelines.CreatePipeline) error { - return r.client.Pipelines.DeleteByPipelineId(ctx, id) +func (r *ResourcePipeline) DoDelete(ctx context.Context, id string, state *PipelineState) error { + if state.CascadeOnDestroy == nil { + // No explicit cascade_on_destroy in config: preserve the backend default (cascade). + return r.client.Pipelines.DeleteByPipelineId(ctx, id) + } + return r.client.Pipelines.Delete(ctx, pipelines.DeletePipelineRequest{ + PipelineId: id, + Cascade: *state.CascadeOnDestroy, + Force: false, + // Cascade is marshaled as `url:"cascade,omitempty"`, so a false value would be dropped from + // the query string. We specify `cascade` in ForceSendFields so the SDK send cascade=false explicitly + ForceSendFields: []string{"Cascade"}, + }) } // Note, terraform provider either diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 6c8cd3973ed..dd21ed58ac0 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -171,6 +171,10 @@ resources: reason: "!drop" - field: run_as reason: input_only + # cascade_on_destroy is a delete-time-only setting + # it is never returned by GET, so suppress remote drift for it. + - field: cascade_on_destroy + reason: input_only ignore_local_changes: - field: deployment.version_id diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index a868bdc98f7..e433dfd51c6 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1229,6 +1229,9 @@ resources: path: ./pipeline.py ``` "$fields": + "cascade_on_destroy": + "description": |- + Whether destroying the pipeline also deletes its datasets (MVs, STs, Views). Defaults to true (the server default). Set to false to retain the datasets when the pipeline is deleted. Only affects the delete operation. "ingestion_definition": "$fields": "objects": diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 2496c7033ad..fb97d38c11b 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -34,7 +34,8 @@ func assertRootPathExists(ctx context.Context, b *bundle.Bundle) (bool, error) { var destroyApprovalGroups = []approvalGroup{ {group: "schemas", message: deleteSchemaMessage}, - {group: "pipelines", message: deletePipelineMessage}, + // Pipelines are handled separately in approvalForDestroy so the message reflects each + // pipeline's cascade_on_destroy setting; see logPipelineDeleteApproval. {group: "volumes", message: deleteVolumeMessage}, {group: "database_instances", message: deleteDatabaseInstanceMessage}, {group: "synced_database_tables", message: deleteSyncedDatabaseTableMessage}, @@ -45,6 +46,38 @@ var destroyApprovalGroups = []approvalGroup{ {group: "genie_spaces", message: deleteGenieSpaceMessage}, } +// logPipelineDeleteApproval prints the pipeline deletions. If cascade_on_destroy is true, we will include +// a note that datasets will be deleted as well. +func logPipelineDeleteApproval(ctx context.Context, b *bundle.Bundle, actions []deployplan.Action) { + pipelineDeletes := filterGroup(actions, "pipelines", deployplan.Delete) + + var cascading, retaining []deployplan.Action + for _, a := range pipelineDeletes { + if pipelineDeletionCascades(b, a) { + cascading = append(cascading, a) + } else { + retaining = append(retaining, a) + } + } + + for _, grp := range []struct { + message string + actions []deployplan.Action + }{ + {deletePipelineWithCascadeMessage, cascading}, + {deletePipelineNoCascadeMessage, retaining}, + } { + if len(grp.actions) == 0 { + continue + } + cmdio.LogString(ctx, grp.message) + for _, a := range grp.actions { + cmdio.Log(ctx, a) + } + cmdio.LogString(ctx, "") + } +} + func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (bool, error) { deleteActions := plan.GetActions() @@ -69,6 +102,7 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. } logApprovalGroups(ctx, deleteActions, destroyApprovalGroups, true, deployplan.Delete) + logPipelineDeleteApproval(ctx, b, deleteActions) cmdio.LogString(ctx, "All files and directories at the following location will be deleted: "+b.Config.Workspace.RootPath) cmdio.LogString(ctx, "") diff --git a/bundle/phases/messages.go b/bundle/phases/messages.go index 8f07b1e2429..388c0faa067 100644 --- a/bundle/phases/messages.go +++ b/bundle/phases/messages.go @@ -59,8 +59,11 @@ const DataLossWarning = "Deleting data assets such as schemas, pipelines, or vol const ( deleteSchemaMessage = `This action will result in the deletion of the following UC schemas. Any underlying data may be lost:` - deletePipelineMessage = `This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them:` + deletePipelineWithCascadeMessage = `This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion:` + + deletePipelineNoCascadeMessage = `This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines. +The Streaming Tables (STs) and Materialized Views (MVs) managed by them are retained, as 'cascade_on_destroy' is set to false:` deleteVolumeMessage = `This action will result in the deletion of the following volumes. For managed volumes, the files stored in the volume are also deleted from your diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 0af9394f243..80997f4ebf9 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -26,10 +26,31 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateGitDetails(), mutator.ValidateDirectOnlyResources(engine), mutator.ValidateLifecycleStarted(engine), + mutator.ValidateCascadeOnDestroy(engine), statemgmt.CheckRunningResource(engine), ) } +// pipelineDeletionCascades reports whether deleting the pipeline referenced by a delete action +// also deletes its datasets (MVs, STs, Views). This is the server default (cascade) unless +// cascade_on_destroy is explicitly set to false. A lookup miss (unset field, or a non-pipeline +// resource) means the default applies, so it returns true. +// Reads from config like checkForPreventDestroy. +func pipelineDeletionCascades(b *bundle.Bundle, action deployplan.Action) bool { + path, err := dyn.NewPathFromString(action.ResourceKey) + if err != nil { + return true + } + path = append(path, dyn.Key("cascade_on_destroy")) + + v, err := dyn.GetByPath(b.Config.Value(), path) + if err != nil { + return true + } + cascade, ok := v.AsBool() + return !ok || cascade +} + // checkForPreventDestroy checks if the resource has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. // If it does, it returns an error. func checkForPreventDestroy(b *bundle.Bundle, actions []deployplan.Action) error { diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 87154f7945f..fd23a7e2631 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1357,6 +1357,10 @@ "description": "[Public Preview] Budget policy of this pipeline.", "$ref": "#/$defs/string" }, + "cascade_on_destroy": { + "description": "Whether destroying the pipeline also deletes its datasets (MVs, STs, Views). Defaults to true (the server default). Set to false to retain the datasets when the pipeline is deleted. Only affects the delete operation.", + "$ref": "#/$defs/bool" + }, "catalog": { "description": "A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog.", "$ref": "#/$defs/string"