Skip to content

Commit 25cc7d9

Browse files
andrewnesterdenik
andrewnester
and
denik
authored
Add metrics for compute type used (#2879)
## Changes Add metrics for compute type used ## Why This allows us to track which compute (serverless or classic and which type) is used in bundle tasks. ## Tests Added acceptance test <!-- If your PR needs to be included in the release notes for next release, add a separate entry in NEXT_CHANGELOG.md as part of your PR. --> --------- Co-authored-by: Denis Bilenko <denis.bilenko@databricks.com>
1 parent bc68b15 commit 25cc7d9

17 files changed

Lines changed: 406 additions & 0 deletions

File tree

‎acceptance/bundle/paths/fallback_metric/output.txt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,18 @@ Deployment complete!
1717
{
1818
"key": "python_wheel_wrapper_is_set",
1919
"value": false
20+
},
21+
{
22+
"key": "has_serverless_compute",
23+
"value": false
24+
},
25+
{
26+
"key": "has_classic_job_compute",
27+
"value": true
28+
},
29+
{
30+
"key": "has_classic_interactive_compute",
31+
"value": false
2032
}
2133
]
2234
[
@@ -27,5 +39,17 @@ Deployment complete!
2739
{
2840
"key": "python_wheel_wrapper_is_set",
2941
"value": false
42+
},
43+
{
44+
"key": "has_serverless_compute",
45+
"value": false
46+
},
47+
{
48+
"key": "has_classic_job_compute",
49+
"value": true
50+
},
51+
{
52+
"key": "has_classic_interactive_compute",
53+
"value": false
3054
}
3155
]

‎acceptance/bundle/telemetry/deploy-artifact-path-type/output.txt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,18 @@ Deployment complete!
3636
{
3737
"key": "python_wheel_wrapper_is_set",
3838
"value": false
39+
},
40+
{
41+
"key": "has_serverless_compute",
42+
"value": false
43+
},
44+
{
45+
"key": "has_classic_job_compute",
46+
"value": false
47+
},
48+
{
49+
"key": "has_classic_interactive_compute",
50+
"value": false
3951
}
4052
],
4153
"bundle_mode": "TYPE_UNSPECIFIED",
@@ -68,6 +80,18 @@ Deployment complete!
6880
{
6981
"key": "python_wheel_wrapper_is_set",
7082
"value": false
83+
},
84+
{
85+
"key": "has_serverless_compute",
86+
"value": false
87+
},
88+
{
89+
"key": "has_classic_job_compute",
90+
"value": false
91+
},
92+
{
93+
"key": "has_classic_interactive_compute",
94+
"value": false
7195
}
7296
],
7397
"bundle_mode": "TYPE_UNSPECIFIED",

‎acceptance/bundle/telemetry/deploy-compute-type/databricks.yml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
bundle:
2+
name: deploy-compute-type
3+
4+
resources:
5+
jobs:
6+
my_job:
7+
environments:
8+
- environment_key: "env"
9+
spec:
10+
client: "1"
11+
dependencies:
12+
- "test_package"
13+
14+
targets:
15+
one:
16+
resources:
17+
jobs:
18+
my_job:
19+
tasks:
20+
- task_key: task1
21+
notebook_task:
22+
notebook_path: ./notebook.py
23+
24+
two:
25+
resources:
26+
jobs:
27+
my_job:
28+
tasks:
29+
- task_key: task1
30+
environment_key: env
31+
spark_python_task:
32+
python_file: ./test.py
33+
34+
three:
35+
resources:
36+
jobs:
37+
my_job:
38+
tasks:
39+
- task_key: task1
40+
new_cluster:
41+
spark_version: 15.4.x-scala2.12
42+
node_type_id: i3.xlarge
43+
data_security_mode: SINGLE_USER
44+
num_workers: 0
45+
spark_conf:
46+
spark.master: "local[*, 4]"
47+
spark.databricks.cluster.profile: singleNode
48+
custom_tags:
49+
ResourceClass: SingleNode
50+
51+
spark_python_task:
52+
python_file: ./test.py
53+
54+
four:
55+
resources:
56+
jobs:
57+
my_job:
58+
tasks:
59+
- task_key: task1
60+
existing_cluster_id: "1234"
61+
spark_python_task:
62+
python_file: ./test.py
63+
- task_key: task2
64+
notebook_task:
65+
notebook_path: ./notebook.py

‎acceptance/bundle/telemetry/deploy-compute-type/notebook.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Databricks notebook source
2+
1 + 1

‎acceptance/bundle/telemetry/deploy-compute-type/output.txt‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
2+
>>> [CLI] bundle deploy -t one
3+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-compute-type/one/files...
4+
Deploying resources...
5+
Updating deployment state...
6+
Deployment complete!
7+
8+
>>> [CLI] bundle deploy -t two
9+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-compute-type/two/files...
10+
Deploying resources...
11+
Updating deployment state...
12+
Deployment complete!
13+
14+
>>> [CLI] bundle deploy -t three
15+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-compute-type/three/files...
16+
Deploying resources...
17+
Updating deployment state...
18+
Deployment complete!
19+
20+
>>> [CLI] bundle deploy -t four
21+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-compute-type/four/files...
22+
Deploying resources...
23+
Updating deployment state...
24+
Deployment complete!
25+
26+
>>> cat out.requests.txt
27+
[
28+
{
29+
"key": "python_wheel_wrapper_is_set",
30+
"value": false
31+
},
32+
{
33+
"key": "has_serverless_compute",
34+
"value": true
35+
},
36+
{
37+
"key": "has_classic_job_compute",
38+
"value": false
39+
},
40+
{
41+
"key": "has_classic_interactive_compute",
42+
"value": false
43+
}
44+
]
45+
[
46+
{
47+
"key": "python_wheel_wrapper_is_set",
48+
"value": false
49+
},
50+
{
51+
"key": "has_serverless_compute",
52+
"value": true
53+
},
54+
{
55+
"key": "has_classic_job_compute",
56+
"value": false
57+
},
58+
{
59+
"key": "has_classic_interactive_compute",
60+
"value": false
61+
}
62+
]
63+
[
64+
{
65+
"key": "python_wheel_wrapper_is_set",
66+
"value": false
67+
},
68+
{
69+
"key": "has_serverless_compute",
70+
"value": false
71+
},
72+
{
73+
"key": "has_classic_job_compute",
74+
"value": true
75+
},
76+
{
77+
"key": "has_classic_interactive_compute",
78+
"value": false
79+
}
80+
]
81+
[
82+
{
83+
"key": "python_wheel_wrapper_is_set",
84+
"value": false
85+
},
86+
{
87+
"key": "has_serverless_compute",
88+
"value": true
89+
},
90+
{
91+
"key": "has_classic_job_compute",
92+
"value": false
93+
},
94+
{
95+
"key": "has_classic_interactive_compute",
96+
"value": true
97+
}
98+
]

‎acceptance/bundle/telemetry/deploy-compute-type/script‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
trace $CLI bundle deploy -t one
2+
trace $CLI bundle deploy -t two
3+
trace $CLI bundle deploy -t three
4+
trace $CLI bundle deploy -t four
5+
6+
trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values'
7+
8+
rm out.requests.txt

‎acceptance/bundle/telemetry/deploy-compute-type/test.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
print("Hello world!")

‎acceptance/bundle/telemetry/deploy-compute-type/test.toml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
[[Server]]
2+
Pattern = "GET /api/2.1/unity-catalog/volumes/main.a.b"
3+
Response.Body = '''
4+
{
5+
"catalog_name": "main",
6+
"schema_name": "a",
7+
"name": "b",
8+
"volume_type": "MANAGED",
9+
"full_name": "main.a.b"
10+
}
11+
'''
12+
13+
[[Server]]
14+
Pattern = "PUT /api/2.0/fs/directories/Volumes/main/a/b/c/.internal"
15+
Response.StatusCode = 200

‎acceptance/bundle/telemetry/deploy-config-file-count/out.telemetry.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,18 @@
3636
{
3737
"key": "python_wheel_wrapper_is_set",
3838
"value": false
39+
},
40+
{
41+
"key": "has_serverless_compute",
42+
"value": false
43+
},
44+
{
45+
"key": "has_classic_job_compute",
46+
"value": false
47+
},
48+
{
49+
"key": "has_classic_interactive_compute",
50+
"value": false
3951
}
4052
],
4153
"bundle_mode": "TYPE_UNSPECIFIED",

‎acceptance/bundle/telemetry/deploy-mode/output.txt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,18 @@ A common practice is to use a username or principal name in this path, i.e. use
4242
{
4343
"key": "python_wheel_wrapper_is_set",
4444
"value": false
45+
},
46+
{
47+
"key": "has_serverless_compute",
48+
"value": false
49+
},
50+
{
51+
"key": "has_classic_job_compute",
52+
"value": false
53+
},
54+
{
55+
"key": "has_classic_interactive_compute",
56+
"value": false
4557
}
4658
],
4759
"bundle_mode": "DEVELOPMENT",
@@ -74,6 +86,18 @@ A common practice is to use a username or principal name in this path, i.e. use
7486
{
7587
"key": "python_wheel_wrapper_is_set",
7688
"value": false
89+
},
90+
{
91+
"key": "has_serverless_compute",
92+
"value": false
93+
},
94+
{
95+
"key": "has_classic_job_compute",
96+
"value": false
97+
},
98+
{
99+
"key": "has_classic_interactive_compute",
100+
"value": false
77101
}
78102
],
79103
"bundle_mode": "PRODUCTION",

0 commit comments

Comments
 (0)