diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/databricks.yml b/acceptance/bundle/deployment/bind/job/already-managed-different/databricks.yml new file mode 100644 index 0000000000..7366df05bc --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: my_project + +resources: + jobs: + foo: + name: My Job diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/out.bind.terraform.txt b/acceptance/bundle/deployment/bind/job/already-managed-different/out.bind.terraform.txt new file mode 100644 index 0000000000..0415fee529 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/out.bind.terraform.txt @@ -0,0 +1,12 @@ + +>>> musterr [CLI] bundle deployment bind foo [NEW_JOB_ID] +Error: terraform import: exit status 1 + +Error: Resource already managed by Terraform + +Terraform is already managing a remote object for databricks_job.foo. To +import to this address you must first remove the existing object from the +state. + + + diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml new file mode 100644 index 0000000000..90061dedb1 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/output.txt b/acceptance/bundle/deployment/bind/job/already-managed-different/output.txt new file mode 100644 index 0000000000..9f0d1005cd --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/output.txt @@ -0,0 +1,8 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_project/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] jobs create --json {"name": "My Job"} diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/script b/acceptance/bundle/deployment/bind/job/already-managed-different/script new file mode 100644 index 0000000000..3508a57b8a --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/script @@ -0,0 +1,7 @@ +# Bind job that is already bound to another ID +trace $CLI bundle deploy + +new_job_id=$(trace $CLI jobs create --json '{"name": "My Job"}' | jq -r '.job_id') +add_repl.py $new_job_id NEW_JOB_ID + +trace musterr $CLI bundle deployment bind foo $new_job_id &> out.bind.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/test.toml b/acceptance/bundle/deployment/bind/job/already-managed-different/test.toml new file mode 100644 index 0000000000..18b1a88417 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/bundle/deployment/bind/job/already-managed-same/databricks.yml b/acceptance/bundle/deployment/bind/job/already-managed-same/databricks.yml new file mode 100644 index 0000000000..7366df05bc --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/already-managed-same/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: my_project + +resources: + jobs: + foo: + name: My Job diff --git a/acceptance/bundle/deployment/bind/job/already-managed-same/out.bind.terraform.txt b/acceptance/bundle/deployment/bind/job/already-managed-same/out.bind.terraform.txt new file mode 100644 index 0000000000..a3be6f2ee3 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/already-managed-same/out.bind.terraform.txt @@ -0,0 +1,12 @@ + +>>> musterr [CLI] bundle deployment bind foo [FOO_ID] +Error: terraform import: exit status 1 + +Error: Resource already managed by Terraform + +Terraform is already managing a remote object for databricks_job.foo. To +import to this address you must first remove the existing object from the +state. + + + diff --git a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml new file mode 100644 index 0000000000..90061dedb1 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-same/output.txt b/acceptance/bundle/deployment/bind/job/already-managed-same/output.txt new file mode 100644 index 0000000000..29757521ac --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/already-managed-same/output.txt @@ -0,0 +1,9 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_project/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/deployment/bind/job/already-managed-same/script b/acceptance/bundle/deployment/bind/job/already-managed-same/script new file mode 100644 index 0000000000..a63358c112 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/already-managed-same/script @@ -0,0 +1,8 @@ +# Bind job that is already bound to the same ID. +# This is a no-op, but terraform complains anyway. +trace $CLI bundle deploy + +job_id=$(read_id.py foo) + +trace musterr $CLI bundle deployment bind foo $job_id &> out.bind.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan | contains.py "1 unchanged" diff --git a/acceptance/bundle/deployment/bind/job/already-managed-same/test.toml b/acceptance/bundle/deployment/bind/job/already-managed-same/test.toml new file mode 100644 index 0000000000..18b1a88417 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/already-managed-same/test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/databricks.yml b/acceptance/bundle/deployment/bind/pipelines/recreate/databricks.yml new file mode 100644 index 0000000000..152942975f --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-pipeline-recreate + +resources: + pipelines: + foo: + name: test-pipeline + libraries: + - notebook: + path: ./nb.sql + catalog: main diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/nb.sql b/acceptance/bundle/deployment/bind/pipelines/recreate/nb.sql new file mode 100644 index 0000000000..199ff50788 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/nb.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +select 1 diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-fail.terraform.txt b/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-fail.terraform.txt new file mode 100644 index 0000000000..667ffd2fd6 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-fail.terraform.txt @@ -0,0 +1,59 @@ + +>>> musterr [CLI] bundle deployment bind foo [NEW_PIPELINE_ID] +databricks_pipeline.foo: Refreshing state... [id=[NEW_PIPELINE_ID]] + +Terraform used the selected providers to generate the following execution +plan. Resource actions are indicated with the following symbols: +-/+ destroy and then create replacement + +Terraform will perform the following actions: + + # databricks_pipeline.foo must be replaced +-/+ resource "databricks_pipeline" "foo" { + - allow_duplicate_names = false -> null + ~ catalog = "old_catalog" -> "main" # forces replacement + + cause = (known after apply) + + channel = "CURRENT" + + cluster_id = (known after apply) + - continuous = false -> null + ~ creator_user_name = "[USERNAME]" -> (known after apply) + - development = false -> null + + edition = "ADVANCED" + - expected_last_modified = 0 -> null + + health = (known after apply) + ~ id = "[NEW_PIPELINE_ID]" -> (known after apply) + ~ last_modified = [UNIX_TIME_MILLIS] -> (known after apply) + ~ name = "lakeflow-pipeline" -> "test-pipeline" + - photon = false -> null + - root_path = "/Workspace/Users/[USERNAME]/lakeflow_pipeline" -> null + ~ run_as_user_name = "[USERNAME]" -> (known after apply) + - serverless = false -> null + ~ state = "IDLE" -> (known after apply) + - storage = "old_storage" -> null # forces replacement + ~ url = "[DATABRICKS_URL]/#joblist/pipelines/[NEW_PIPELINE_ID]" -> (known after apply) + + + deployment { + + kind = "BUNDLE" + + metadata_file_path = "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state/metadata.json" + } + + ~ library { + - glob { + - include = "/Workspace/Users/[USERNAME]/lakeflow_pipeline/transformations/**" -> null + } + + notebook { + + path = "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files/nb" + } + } + - library { + - glob { + - include = "/Workspace/Users/foo@databricks.com/another/**" -> null + } + } + } + +Plan: 1 to add, 0 to change, 1 to destroy. + + +Error: This bind operation requires user confirmation, but the current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed. + diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-success.terraform.txt b/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-success.terraform.txt new file mode 100644 index 0000000000..6c88a3134b --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-success.terraform.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle deployment bind foo [NEW_PIPELINE_ID] --auto-approve +Updating deployment state... +Successfully bound pipeline with an id '[NEW_PIPELINE_ID]' +Run 'bundle deploy' to deploy changes to your workspace diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.deploy.requests.terraform.json b/acceptance/bundle/deployment/bind/pipelines/recreate/out.deploy.requests.terraform.json new file mode 100644 index 0000000000..ce3ee109e0 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.deploy.requests.terraform.json @@ -0,0 +1,32 @@ +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/artifacts/.internal" + } +} +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[NEW_PIPELINE_ID]" +} +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "notebook": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files/nb" + } + } + ], + "name": "test-pipeline" + } +} diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.summary.terraform.json b/acceptance/bundle/deployment/bind/pipelines/recreate/out.summary.terraform.json new file mode 100644 index 0000000000..d4d9b72a85 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.summary.terraform.json @@ -0,0 +1,51 @@ +{ + "bundle": { + "environment": "default", + "git": { + "bundle_root_path": "." + }, + "name": "test-pipeline-recreate", + "target": "default" + }, + "resources": { + "pipelines": { + "foo": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state/metadata.json" + }, + "edition": "ADVANCED", + "id": "[NEW_PIPELINE_ID]", + "libraries": [ + { + "notebook": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files/nb" + } + } + ], + "name": "test-pipeline", + "url": "[DATABRICKS_URL]/pipelines/[NEW_PIPELINE_ID]?o=[NUMID]" + } + } + }, + "sync": { + "paths": [ + "." + ] + }, + "workspace": { + "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/artifacts", + "current_user": { + "domain_friendly_name": "[USERNAME]", + "id": "[USERID]", + "short_name": "[USERNAME]", + "userName": "[USERNAME]" + }, + "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files", + "resource_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/resources", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default", + "state_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state" + } +} diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml new file mode 100644 index 0000000000..90061dedb1 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt b/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt new file mode 100644 index 0000000000..e95281632b --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt @@ -0,0 +1,32 @@ + +>>> [CLI] bundle summary -o json + +>>> [CLI] bundle plan +recreate pipelines.foo + +Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files... + +This action will result in the deletion or recreation of the following Lakeflow Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the pipelines will +restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline +properties such as the 'catalog' or 'storage' are changed: + recreate resources.pipelines.foo +Error: the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed + + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files... + +This action will result in the deletion or recreation of the following Lakeflow Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the pipelines will +restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline +properties such as the 'catalog' or 'storage' are changed: + recreate resources.pipelines.foo +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py ^//import-file/ ^//workspace/delete ^//telemetry-ext diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/pipeline.json b/acceptance/bundle/deployment/bind/pipelines/recreate/pipeline.json new file mode 100644 index 0000000000..d022085ea9 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/pipeline.json @@ -0,0 +1,18 @@ +{ + "name": "lakeflow-pipeline", + "catalog": "old_catalog", + "storage": "old_storage", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline/transformations/**" + } + }, + { + "glob": { + "include": "/Workspace/Users/foo@databricks.com/another/**" + } + } + ], + "root_path": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline" +} diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/script b/acceptance/bundle/deployment/bind/pipelines/recreate/script new file mode 100644 index 0000000000..542c3182d0 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/script @@ -0,0 +1,19 @@ +NEW_PIPELINE_ID=$($CLI pipelines create --json @pipeline.json | jq -r .pipeline_id) +add_repl.py $NEW_PIPELINE_ID NEW_PIPELINE_ID + +rm -f out.requests.txt +trace musterr $CLI bundle deployment bind foo $NEW_PIPELINE_ID &> out.bind-fail.$DATABRICKS_BUNDLE_ENGINE.txt +print_requests.py '^//import-file/' '^//workspace/delete' + +rm -f out.requests.txt +trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.$DATABRICKS_BUNDLE_ENGINE.txt +print_requests.py '^//import-file/' '^//workspace/delete' + +trace $CLI bundle summary -o json > out.summary.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan + +trace musterr $CLI bundle deploy + +rm -f out.requests.txt +trace $CLI bundle deploy --auto-approve +trace print_requests.py '^//import-file/' '^//workspace/delete' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/test.toml b/acceptance/bundle/deployment/bind/pipelines/recreate/test.toml new file mode 100644 index 0000000000..eb67288687 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/test.toml @@ -0,0 +1,2 @@ +RecordRequests = true +Ignore = [".databricks"] diff --git a/acceptance/bundle/deployment/bind/pipelines/test.toml b/acceptance/bundle/deployment/bind/pipelines/test.toml new file mode 100644 index 0000000000..18b1a88417 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/bundle/deployment/bind/pipelines/update/databricks.yml b/acceptance/bundle/deployment/bind/pipelines/update/databricks.yml new file mode 100644 index 0000000000..152942975f --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/update/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-pipeline-recreate + +resources: + pipelines: + foo: + name: test-pipeline + libraries: + - notebook: + path: ./nb.sql + catalog: main diff --git a/acceptance/bundle/deployment/bind/pipelines/update/nb.sql b/acceptance/bundle/deployment/bind/pipelines/update/nb.sql new file mode 100644 index 0000000000..199ff50788 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/update/nb.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +select 1 diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.terraform.txt b/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.terraform.txt new file mode 100644 index 0000000000..2ff60fce2b --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.terraform.txt @@ -0,0 +1,45 @@ + +>>> musterr [CLI] bundle deployment bind foo [NEW_PIPELINE_ID] +databricks_pipeline.foo: Refreshing state... [id=[NEW_PIPELINE_ID]] + +Terraform used the selected providers to generate the following execution +plan. Resource actions are indicated with the following symbols: + ~ update in-place + +Terraform will perform the following actions: + + # databricks_pipeline.foo will be updated in-place + ~ resource "databricks_pipeline" "foo" { + + catalog = "main" + + channel = "CURRENT" + + edition = "ADVANCED" + id = "[NEW_PIPELINE_ID]" + ~ name = "lakeflow-pipeline" -> "test-pipeline" + - root_path = "/Workspace/Users/[USERNAME]/lakeflow_pipeline" -> null + # (12 unchanged attributes hidden) + + + deployment { + + kind = "BUNDLE" + + metadata_file_path = "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state/metadata.json" + } + + ~ library { + - glob { + - include = "/Workspace/Users/[USERNAME]/lakeflow_pipeline/transformations/**" -> null + } + + notebook { + + path = "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files/nb" + } + } + - library { + - glob { + - include = "/Workspace/Users/foo@databricks.com/another/**" -> null + } + } + } + +Plan: 0 to add, 1 to change, 0 to destroy. + + +Error: This bind operation requires user confirmation, but the current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed. + diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.bind-success.terraform.txt b/acceptance/bundle/deployment/bind/pipelines/update/out.bind-success.terraform.txt new file mode 100644 index 0000000000..6c88a3134b --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.bind-success.terraform.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle deployment bind foo [NEW_PIPELINE_ID] --auto-approve +Updating deployment state... +Successfully bound pipeline with an id '[NEW_PIPELINE_ID]' +Run 'bundle deploy' to deploy changes to your workspace diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.deploy.requests.terraform.json b/acceptance/bundle/deployment/bind/pipelines/update/out.deploy.requests.terraform.json new file mode 100644 index 0000000000..3c8d1946b8 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.deploy.requests.terraform.json @@ -0,0 +1,37 @@ +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/artifacts/.internal" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files" + } +} +{ + "method": "PUT", + "path": "/api/2.0/pipelines/[NEW_PIPELINE_ID]", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state/metadata.json" + }, + "edition": "ADVANCED", + "id": "[NEW_PIPELINE_ID]", + "libraries": [ + { + "notebook": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files/nb" + } + } + ], + "name": "test-pipeline", + "storage": "dbfs:/pipelines/[NEW_PIPELINE_ID]" + } +} diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.summary.terraform.json b/acceptance/bundle/deployment/bind/pipelines/update/out.summary.terraform.json new file mode 100644 index 0000000000..d4d9b72a85 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.summary.terraform.json @@ -0,0 +1,51 @@ +{ + "bundle": { + "environment": "default", + "git": { + "bundle_root_path": "." + }, + "name": "test-pipeline-recreate", + "target": "default" + }, + "resources": { + "pipelines": { + "foo": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state/metadata.json" + }, + "edition": "ADVANCED", + "id": "[NEW_PIPELINE_ID]", + "libraries": [ + { + "notebook": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files/nb" + } + } + ], + "name": "test-pipeline", + "url": "[DATABRICKS_URL]/pipelines/[NEW_PIPELINE_ID]?o=[NUMID]" + } + } + }, + "sync": { + "paths": [ + "." + ] + }, + "workspace": { + "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/artifacts", + "current_user": { + "domain_friendly_name": "[USERNAME]", + "id": "[USERID]", + "short_name": "[USERNAME]", + "userName": "[USERNAME]" + }, + "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files", + "resource_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/resources", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default", + "state_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state" + } +} diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml new file mode 100644 index 0000000000..90061dedb1 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deployment/bind/pipelines/update/output.txt b/acceptance/bundle/deployment/bind/pipelines/update/output.txt new file mode 100644 index 0000000000..f88ed70d69 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/update/output.txt @@ -0,0 +1,15 @@ + +>>> [CLI] bundle summary -o json + +>>> [CLI] bundle plan +update pipelines.foo + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py ^//import-file/ ^//workspace/delete ^//telemetry-ext diff --git a/acceptance/bundle/deployment/bind/pipelines/update/pipeline.json b/acceptance/bundle/deployment/bind/pipelines/update/pipeline.json new file mode 100644 index 0000000000..0254d7b549 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/update/pipeline.json @@ -0,0 +1,16 @@ +{ + "name": "lakeflow-pipeline", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline/transformations/**" + } + }, + { + "glob": { + "include": "/Workspace/Users/foo@databricks.com/another/**" + } + } + ], + "root_path": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline" +} diff --git a/acceptance/bundle/deployment/bind/pipelines/update/script b/acceptance/bundle/deployment/bind/pipelines/update/script new file mode 100644 index 0000000000..1235b895d6 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/update/script @@ -0,0 +1,17 @@ +NEW_PIPELINE_ID=$($CLI pipelines create --json @pipeline.json | jq -r .pipeline_id) +add_repl.py $NEW_PIPELINE_ID NEW_PIPELINE_ID + +rm -f out.requests.txt +trace musterr $CLI bundle deployment bind foo $NEW_PIPELINE_ID &> out.bind-fail.$DATABRICKS_BUNDLE_ENGINE.txt +print_requests.py '^//import-file/' '^//workspace/delete' + +rm -f out.requests.txt +trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.$DATABRICKS_BUNDLE_ENGINE.txt +print_requests.py '^//import-file/' '^//workspace/delete' + +trace $CLI bundle summary -o json > out.summary.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan + +rm -f out.requests.txt +trace $CLI bundle deploy --auto-approve +trace print_requests.py '^//import-file/' '^//workspace/delete' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/deployment/bind/pipelines/update/test.toml b/acceptance/bundle/deployment/bind/pipelines/update/test.toml new file mode 100644 index 0000000000..eb67288687 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/update/test.toml @@ -0,0 +1,2 @@ +RecordRequests = true +Ignore = [".databricks"]