Skip to content

Commit 59a4da6

Browse files
authored
Read dashboard definition from file_path via filer interface (#2738)
## Changes <!-- Brief summary of your changes that is easy to understand --> 1. When dashboard resource contains a `file_path` field, that file is read and its contents are set to the `serialized_dashboard` field. This is done in a newly introduced `ConfigureDashboardSerializedDashboard` mutator 2. Old behavior of wrapping file_path into terraform's `file` directive is removed from `dashboardConverter` (unit tests for this are also removed) 3. `ConfigureWSFS` mutator call is moved to be executed before `PythonMutator` so that the latter has access to the Workspace File System client ## Why <!-- Why are these changes needed? Provide the context that the reviewer might be missing. For example, were there any decisions behind the change that are not reflected in the code itself? --> This change allows `bundle deploy` to correctly deploy dashboard resource when running in Databricks Workspace / Runtime. Deployment does not work currently, because reading a dashboard file via file system interface throws an error in the Workspace. ## Tests <!-- How have you tested the changes? --> - Added a new dashboard deployment acceptance test - Performed a manual test to deploy a dashboard from a workspace <!-- If your PR needs to be included in the release notes for next release, add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->
1 parent b5f9b1e commit 59a4da6

15 files changed

Lines changed: 179 additions & 104 deletions

acceptance/bundle/debug/out.stderr.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
10:07:59 Debug: Apply pid=12345 mutator=ResolveVariableReferences
4141
10:07:59 Debug: Apply pid=12345 mutator=validate:volume-path
4242
10:07:59 Debug: Apply pid=12345 mutator=ApplyTargetMode
43+
10:07:59 Debug: Apply pid=12345 mutator=ConfigureWSFS
4344
10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources
4445
10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=ResolveVariableReferences(resources)
4546
10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=NormalizePaths
@@ -48,7 +49,6 @@
4849
10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(load_resources)
4950
10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(apply_mutators)
5051
10:07:59 Debug: Apply pid=12345 mutator=CheckPermissions
51-
10:07:59 Debug: Apply pid=12345 mutator=ConfigureWSFS
5252
10:07:59 Debug: Apply pid=12345 mutator=TranslatePaths
5353
10:07:59 Debug: Apply pid=12345 mutator=PythonWrapperWarning
5454
10:07:59 Debug: Apply pid=12345 mutator=artifacts.Prepare
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
bundle:
2+
name: deploy-dashboard-test-$UNIQUE_NAME
3+
4+
resources:
5+
dashboards:
6+
dashboard1:
7+
display_name: $DASHBOARD_DISPLAY_NAME
8+
warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID
9+
embed_credentials: true
10+
file_path: "sample-dashboard.lvdash.json"
11+
parent_path: /Users/$CURRENT_USER_NAME
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
2+
>>> [CLI] bundle deploy
3+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-dashboard-test-[UNIQUE_NAME]/default/files...
4+
Deploying resources...
5+
Updating deployment state...
6+
Deployment complete!
7+
8+
>>> [CLI] lakeview get [DASHBOARD_ID]
9+
{
10+
"lifecycle_state": "ACTIVE",
11+
"parent_path": "/Users/[USERNAME]",
12+
"path": "/Users/[USERNAME]/test bundle-deploy-dashboard [UUID].lvdash.json",
13+
"serialized_dashboard": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}"
14+
}
15+
16+
>>> [CLI] bundle destroy --auto-approve
17+
The following resources will be deleted:
18+
delete dashboard dashboard1
19+
20+
All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-dashboard-test-[UNIQUE_NAME]/default
21+
22+
Deleting files...
23+
Destroy complete!
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"pages":[{"name":"02724bf2","displayName":"Dashboard test bundle-deploy-dashboard"}]}
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
DASHBOARD_DISPLAY_NAME="test bundle-deploy-dashboard $(uuid)"
2+
if [ -z "$CLOUD_ENV" ]; then
3+
DASHBOARD_DISPLAY_NAME="test bundle/deploy/ 6260d50f-e8ff-4905-8f28-812345678903" # use hard-coded uuid when running locally
4+
export TEST_DEFAULT_WAREHOUSE_ID="warehouse-1234"
5+
fi
6+
7+
export DASHBOARD_DISPLAY_NAME
8+
envsubst < databricks.yml.tmpl > databricks.yml
9+
10+
cleanup() {
11+
trace $CLI bundle destroy --auto-approve
12+
}
13+
trap cleanup EXIT
14+
15+
trace $CLI bundle deploy
16+
DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id')
17+
trace $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard}'
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
Local = true
2+
Cloud = true
3+
RequiresWarehouse = true
4+
5+
Ignore = [
6+
"databricks.yml",
7+
]
8+
9+
[[Repls]]
10+
Old = "[0-9a-f]{32}"
11+
New = "[DASHBOARD_ID]"
12+
13+
[[Server]]
14+
Pattern = "POST /api/2.0/lakeview/dashboards"
15+
Response.Body = '''
16+
{
17+
"dashboard_id":"1234567890abcdef1234567890abcdef"
18+
}
19+
'''
20+
21+
[[Server]]
22+
Pattern = "POST /api/2.0/lakeview/dashboards/{dashboard_id}/published"
23+
24+
[[Server]]
25+
Pattern = "GET /api/2.0/lakeview/dashboards/{dashboard_id}"
26+
Response.Body = '''
27+
{
28+
"dashboard_id":"1234567890abcdef1234567890abcdef",
29+
"display_name": "test dashboard 6260d50f-e8ff-4905-8f28-812345678903",
30+
"lifecycle_state": "ACTIVE",
31+
"path": "/Users/[USERNAME]/test bundle-deploy-dashboard 6260d50f-e8ff-4905-8f28-812345678903.lvdash.json",
32+
"parent_path": "/Users/tester@databricks.com",
33+
"serialized_dashboard": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}"
34+
}
35+
'''
36+
37+
[[Server]]
38+
Pattern = "DELETE /api/2.0/lakeview/dashboards/{dashboard_id}"

bundle/config/mutator/paths/dashboard_paths_visitor.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,6 @@ func VisitDashboardPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) {
1313
)
1414

1515
return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) {
16-
return fn(path, TranslateModeLocalAbsoluteFile, value)
16+
return fn(path, TranslateModeLocalRelative, value)
1717
})
1818
}

bundle/config/mutator/paths/translation_mode.go

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,6 @@ const (
1313
// TranslateModeDirectory translates a path to a remote directory.
1414
TranslateModeDirectory
1515

16-
// TranslateModeLocalAbsoluteFile translates a path to the local absolute file path.
17-
// It returns an error if the path does not exist or is a directory.
18-
TranslateModeLocalAbsoluteFile
19-
2016
// TranslateModeLocalAbsoluteDirectory translates a path to the local absolute directory path.
2117
// It returns an error if the path does not exist or is not a directory.
2218
TranslateModeLocalAbsoluteDirectory
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
package resourcemutator
2+
3+
import (
4+
"context"
5+
"fmt"
6+
7+
"github.com/databricks/cli/bundle"
8+
"github.com/databricks/cli/libs/diag"
9+
"github.com/databricks/cli/libs/dyn"
10+
)
11+
12+
const (
13+
filePathFieldName = "file_path"
14+
serializedDashboardFieldName = "serialized_dashboard"
15+
)
16+
17+
type configureDashboardSerializedDashboard struct{}
18+
19+
func ConfigureDashboardSerializedDashboard() bundle.Mutator {
20+
return &configureDashboardSerializedDashboard{}
21+
}
22+
23+
func (c configureDashboardSerializedDashboard) Name() string {
24+
return "ConfigureDashboardSerializedDashboard"
25+
}
26+
27+
func (c configureDashboardSerializedDashboard) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
28+
var diags diag.Diagnostics
29+
30+
pattern := dyn.NewPattern(
31+
dyn.Key("resources"),
32+
dyn.Key("dashboards"),
33+
dyn.AnyKey(),
34+
)
35+
36+
// Configure serialized_dashboard field for all dashboards.
37+
err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
38+
return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
39+
// Include "serialized_dashboard" field if "file_path" is set.
40+
// Note: the Terraform resource supports "file_path" natively, but we read the contents of the dashboard here
41+
// to be able to read file contents in Databricks Workspace (reading a dashboard file via file system fails there)
42+
path, ok := v.Get(filePathFieldName).AsString()
43+
if !ok {
44+
return v, nil
45+
}
46+
47+
contents, err := b.SyncRoot.ReadFile(path)
48+
if err != nil {
49+
return dyn.InvalidValue, fmt.Errorf("failed to read serialized dashboard from file_path %s: %w", path, err)
50+
}
51+
52+
v, err = dyn.Set(v, serializedDashboardFieldName, dyn.V(string(contents)))
53+
if err != nil {
54+
return dyn.InvalidValue, fmt.Errorf("failed to set serialized_dashboard: %w", err)
55+
}
56+
57+
// Drop the "file_path" field. It is mutually exclusive with "serialized_dashboard".
58+
return dyn.Walk(v, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
59+
switch len(p) {
60+
case 0:
61+
return v, nil
62+
case 1:
63+
if p[0] == dyn.Key(filePathFieldName) {
64+
return v, dyn.ErrDrop
65+
}
66+
}
67+
68+
// Skip everything else.
69+
return v, dyn.ErrSkip
70+
})
71+
})
72+
})
73+
74+
diags = diags.Extend(diag.FromErr(err))
75+
return diags
76+
}

bundle/config/mutator/resourcemutator/resource_mutator.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,11 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) diag.Diagnost
146146
// Updates (typed): resources.pipelines.*.{schema,target}, resources.volumes.*.schema_name (converts implicit schema references to explicit ${resources.schemas.<schema_key>.name} syntax)
147147
// Translates implicit schema references in DLT pipelines or UC Volumes to explicit syntax to capture dependencies
148148
CaptureSchemaDependency(),
149+
150+
// Reads (dynamic): resources.dashboards.*.file_path
151+
// Updates (dynamic): resources.dashboards.*.serialized_dashboard
152+
// Drops (dynamic): resources.dashboards.*.file_path
153+
ConfigureDashboardSerializedDashboard(),
149154
)
150155
}
151156

0 commit comments

Comments
 (0)