Skip to content

Commit f4edb94

Browse files
anton-107anton-107
authored
Convert TestFilesAreSyncedCorrectlyWhenNoSnapshot to an acceptance test (#2924)
## Why <!-- Why are these changes needed? Provide the context that the reviewer might be missing. For example, were there any decisions behind the change that are not reflected in the code itself? --> One change in the series of changes for converting integration tests into acceptance tests. This will allow for easier testing of various backing solutions for bundle deployment
1 parent 88b8a17 commit f4edb94

10 files changed

Lines changed: 142 additions & 107 deletions

File tree

‎acceptance/bundle/deploy/files/no-snapshot-sync/databricks.yml.tmpl‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
bundle:
2+
name: test-sync-with-no-snapshot
3+
4+
workspace:
5+
root_path: ~/.bundle/$UNIQUE_NAME
6+
7+
resources:
8+
jobs:
9+
foo:
10+
name: test-job-basic-$UNIQUE_NAME
11+
tasks:
12+
- task_key: my_notebook_task
13+
new_cluster:
14+
num_workers: 1
15+
spark_version: $DEFAULT_SPARK_VERSION
16+
node_type_id: $NODE_TYPE_ID
17+
spark_python_task:
18+
python_file: ./hello_world.py

‎acceptance/bundle/deploy/files/no-snapshot-sync/hello_world.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
print("Hello World!")

‎acceptance/bundle/deploy/files/no-snapshot-sync/notebook.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Databricks notebook source
2+
print("Hello, World!")

‎acceptance/bundle/deploy/files/no-snapshot-sync/output.txt‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
2+
>>> [CLI] bundle deploy
3+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
4+
Deploying resources...
5+
Updating deployment state...
6+
Deployment complete!
7+
8+
=== Check that test file is in workspace
9+
>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/test.py
10+
{
11+
"object_type": "FILE",
12+
"path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/test.py"
13+
}
14+
15+
>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/test_to_modify.py
16+
{
17+
"object_type": "FILE",
18+
"path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/test_to_modify.py"
19+
}
20+
21+
=== Check that notebook is in workspace
22+
>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/notebook
23+
{
24+
"object_type": "NOTEBOOK",
25+
"path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/notebook",
26+
"language": "PYTHON"
27+
}
28+
29+
=== Check that deployment.json is synced correctly
30+
>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json
31+
{
32+
"object_type": "FILE",
33+
"path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json"
34+
}
35+
36+
=== Remove .databricks directory to simulate a fresh deployment like in CI/CD environment
37+
>>> rm -rf .databricks
38+
39+
>>> rm test.py
40+
41+
>>> rm notebook.py
42+
43+
>>> echo print('Modified!')
44+
45+
>>> [CLI] bundle deploy
46+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
47+
Deploying resources...
48+
Updating deployment state...
49+
Deployment complete!
50+
51+
=== Check that removed files are not in the workspace anymore
52+
>>> errcode [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/test.py
53+
Error: Path (/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/test.py) doesn't exist.
54+
55+
Exit code: 1
56+
57+
>>> errcode [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/notebook
58+
Error: Path (/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/notebook) doesn't exist.
59+
60+
Exit code: 1
61+
62+
=== Check the content of modified file
63+
>>> [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/test_to_modify.py
64+
print('Modified!')
65+
66+
>>> [CLI] bundle destroy --auto-approve
67+
The following resources will be deleted:
68+
delete job foo
69+
70+
All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
71+
72+
Deleting files...
73+
Destroy complete!

‎acceptance/bundle/deploy/files/no-snapshot-sync/script‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
envsubst < databricks.yml.tmpl > databricks.yml
2+
3+
cleanup() {
4+
trace $CLI bundle destroy --auto-approve
5+
}
6+
trap cleanup EXIT
7+
8+
trace $CLI bundle deploy
9+
10+
title "Check that test file is in workspace"
11+
trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/${UNIQUE_NAME}/files/test.py" | jq '{object_type,path}'
12+
trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/${UNIQUE_NAME}/files/test_to_modify.py" | jq '{object_type,path}'
13+
14+
title "Check that notebook is in workspace"
15+
trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/${UNIQUE_NAME}/files/notebook" | jq '{object_type,path,language}'
16+
17+
title "Check that deployment.json is synced correctly"
18+
trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/${UNIQUE_NAME}/state/deployment.json" | jq '{object_type,path}'
19+
20+
title "Remove .databricks directory to simulate a fresh deployment like in CI/CD environment"
21+
trace rm -rf .databricks
22+
23+
trace rm test.py
24+
trace rm notebook.py
25+
trace echo "print('Modified!')" > test_to_modify.py
26+
27+
trace $CLI bundle deploy
28+
29+
title "Check that removed files are not in the workspace anymore"
30+
trace errcode $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/${UNIQUE_NAME}/files/test.py"
31+
trace errcode $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/${UNIQUE_NAME}/files/notebook"
32+
33+
title "Check the content of modified file"
34+
trace $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/${UNIQUE_NAME}/files/test_to_modify.py"

‎acceptance/bundle/deploy/files/no-snapshot-sync/test.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
print("Hello, World!")

‎acceptance/bundle/deploy/files/no-snapshot-sync/test.toml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
Local = false
2+
Cloud = true
3+
4+
Ignore = [
5+
"databricks.yml",
6+
]
7+
8+
[Env]
9+
# MSYS2 automatically converts absolute paths like /Users/$username/$UNIQUE_NAME to
10+
# C:/Program Files/Git/Users/$username/UNIQUE_NAME before passing it to the CLI
11+
# Setting this environment variable prevents that conversion on windows.
12+
MSYS_NO_PATHCONV = "1"

‎acceptance/bundle/deploy/files/no-snapshot-sync/test_to_modify.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
print("Hello, World!")

‎integration/bundle/deployment_state_test.go‎

Lines changed: 0 additions & 97 deletions
This file was deleted.

‎integration/bundle/helpers_test.go‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package bundle_test
33
import (
44
"context"
55
"encoding/json"
6-
"fmt"
76
"os"
87
"path/filepath"
98
"strings"
@@ -15,7 +14,6 @@ import (
1514
"github.com/databricks/cli/libs/env"
1615
"github.com/databricks/cli/libs/flags"
1716
"github.com/databricks/cli/libs/template"
18-
"github.com/databricks/databricks-sdk-go"
1917
"github.com/stretchr/testify/require"
2018
)
2119

@@ -104,11 +102,3 @@ func destroyBundle(t testutil.TestingT, ctx context.Context, path string) {
104102
_, _, err := c.Run()
105103
require.NoError(t, err)
106104
}
107-
108-
func getBundleRemoteRootPath(w *databricks.WorkspaceClient, t testutil.TestingT, uniqueId string) string {
109-
// Compute root path for the bundle deployment
110-
me, err := w.CurrentUser.Me(context.Background())
111-
require.NoError(t, err)
112-
root := fmt.Sprintf("/Workspace/Users/%s/.bundle/%s", me.UserName, uniqueId)
113-
return root
114-
}

0 commit comments

Comments
 (0)