Bump default serverless environment version to 5

.nextchanges/bundles/serverless-environment-version-v5.md

@@ -0,0 +1 @@
+Bundle templates are upgraded from serverless environment version 4 to [version 5](https://docs.databricks.com/aws/en/release-notes/serverless/environment-version/five) for generated serverless jobs and notebooks (`default-python`, `default-scala`, `dbt-sql`, `pydabs`, and `lakeflow-pipelines`). The Python templates now pin `databricks-connect>=18.0,<18.1` and Python `>=3.12,<3.13` (previously `databricks-connect>=15.4,<15.5` and Python `>=3.10,<3.13`) to match environment version 5's DB Connect 18 and Python 3.12 runtime.

acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml

@@ -29,6 +29,6 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "4"
+            environment_version: "5"
             dependencies:
               - dbt-databricks>=1.8.0,<2.0.0

acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/pyproject.toml

@@ -2,7 +2,7 @@
 name = "my_default_minimal"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<3.13"
+requires-python = ">=3.12,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies
@@ -16,7 +16,7 @@ dev = [
     "pytest",
     "ruff",
     "databricks-dlt",
-    "databricks-connect>=15.4,<15.5",
+    "databricks-connect>=18.0,<18.1",
     "ipykernel",
 ]
 

acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff

@@ -51,7 +51,7 @@
 -      environments:
 -        - environment_key: default
 -          spec:
--            environment_version: "4"
+-            environment_version: "5"
 -            dependencies:
 -              # By default we just include the .whl file generated for the my_default_python package.
 -              # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html

acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml

@@ -2,7 +2,7 @@
 name = "my_default_python"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<3.13"
+requires-python = ">=3.12,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies
@@ -16,7 +16,7 @@ dev = [
     "pytest",
     "ruff",
     "databricks-dlt",
-    "databricks-connect>=15.4,<15.5",
+    "databricks-connect>=18.0,<18.1",
     "ipykernel",
 ]
 

acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb

@@ -72,7 +72,7 @@
     "dependencies": [
      "--editable .."
     ],
-    "environment_version": "4"
+    "environment_version": "5"
    },
    "language": "python",
    "notebookMetadata": {

acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json

@@ -28,7 +28,7 @@
               "dependencies": [
                 "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl"
               ],
-              "environment_version": "4"
+              "environment_version": "5"
             }
           }
         ],

acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json

@@ -28,7 +28,7 @@
               "dependencies": [
                 "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl"
               ],
-              "environment_version": "4"
+              "environment_version": "5"
             }
           }
         ],

acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json

@@ -24,7 +24,7 @@
                 "dependencies": [
                   "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl"
                 ],
-                "environment_version": "4"
+                "environment_version": "5"
               }
             }
           ],

acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json

@@ -24,7 +24,7 @@
                 "dependencies": [
                   "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl"
                 ],
-                "environment_version": "4"
+                "environment_version": "5"
               }
             }
           ],

acceptance/bundle/templates/default-python/serverless/out.requests.dev.direct.txt

@@ -111,7 +111,7 @@
           "dependencies": [
             "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl"
           ],
-          "environment_version": "4"
+          "environment_version": "5"
         }
       }
     ],

acceptance/bundle/templates/default-python/serverless/out.requests.dev.terraform.txt

@@ -111,7 +111,7 @@
           "dependencies": [
             "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl"
           ],
-          "environment_version": "4"
+          "environment_version": "5"
         }
       }
     ],

acceptance/bundle/templates/default-python/serverless/out.requests.prod.direct.txt

@@ -114,7 +114,7 @@
           "dependencies": [
             "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl"
           ],
-          "environment_version": "4"
+          "environment_version": "5"
         }
       }
     ],

acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt

@@ -114,7 +114,7 @@
           "dependencies": [
             "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl"
           ],
-          "environment_version": "4"
+          "environment_version": "5"
         }
       }
     ],

acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml

@@ -2,7 +2,7 @@
 name = "my_default_python"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<3.13"
+requires-python = ">=3.12,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies
@@ -16,7 +16,7 @@ dev = [
     "pytest",
     "ruff",
     "databricks-dlt",
-    "databricks-connect>=15.4,<15.5",
+    "databricks-connect>=18.0,<18.1",
     "ipykernel",
 ]
 

acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml

@@ -46,7 +46,7 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "4"
+            environment_version: "5"
             dependencies:
               # By default we just include the .whl file generated for the my_default_python package.
               # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html

acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb

@@ -72,7 +72,7 @@
     "dependencies": [
      "--editable .."
     ],
-    "environment_version": "4"
+    "environment_version": "5"
    },
    "language": "python",
    "notebookMetadata": {

acceptance/bundle/templates/default-scala/output/my_default_scala/resources/my_default_scala.job.yml

@@ -22,6 +22,6 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "4"
+            environment_version: "5"
             java_dependencies:
               - ${workspace.artifact_path}/.internal/my_default_scala-assembly-0.1.jar

acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml

@@ -2,7 +2,7 @@
 name = "my_lakeflow_pipelines"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<3.13"
+requires-python = ">=3.12,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies
@@ -16,7 +16,7 @@ dev = [
     "pytest",
     "ruff",
     "databricks-dlt",
-    "databricks-connect>=15.4,<15.5",
+    "databricks-connect>=18.0,<18.1",
     "ipykernel",
 ]
 

acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml

@@ -29,4 +29,4 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "4"
+            environment_version: "5"

acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml

@@ -29,4 +29,4 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "4"
+            environment_version: "5"

acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml

@@ -2,7 +2,7 @@
 name = "my_pydabs"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<3.13"
+requires-python = ">=3.12,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies
@@ -16,7 +16,7 @@ dev = [
     "pytest",
     "ruff",
     "databricks-dlt",
-    "databricks-connect>=15.4,<15.5",
+    "databricks-connect>=18.0,<18.1",
     "ipykernel",
     "databricks-bundles==0.0.0",
 ]

acceptance/pipelines/e2e/output/lakeflow_project/pyproject.toml

@@ -2,7 +2,7 @@
 name = "lakeflow_project"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<3.13"
+requires-python = ">=3.12,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies
@@ -16,7 +16,7 @@ dev = [
     "pytest",
     "ruff",
     "databricks-dlt",
-    "databricks-connect>=15.4,<15.5",
+    "databricks-connect>=18.0,<18.1",
     "ipykernel",
 ]
 

acceptance/pipelines/e2e/output/lakeflow_project/resources/sample_job.job.yml

@@ -29,4 +29,4 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "4"
+            environment_version: "5"

acceptance/pipelines/init/python/output/my_python_project/pyproject.toml

@@ -2,7 +2,7 @@
 name = "my_python_project"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<3.13"
+requires-python = ">=3.12,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies
@@ -16,7 +16,7 @@ dev = [
     "pytest",
     "ruff",
     "databricks-dlt",
-    "databricks-connect>=15.4,<15.5",
+    "databricks-connect>=18.0,<18.1",
     "ipykernel",
 ]
 

acceptance/pipelines/init/python/output/my_python_project/resources/sample_job.job.yml

@@ -29,4 +29,4 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "4"
+            environment_version: "5"

acceptance/pipelines/init/sql/output/my_sql_project/resources/sample_job.job.yml

@@ -29,4 +29,4 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "4"
+            environment_version: "5"

libs/template/templates/dbt-sql/library/versions.tmpl

@@ -2,13 +2,14 @@
   15.4.x-scala2.12
 {{- end}}
 
+{{/* TODO: unused — no dbt-sql template references this macro. Remove it. */}}
 {{define "latest_lts_db_connect_version_spec" -}}
-  >=15.4,<15.5
+  >=18.0,<18.1
 {{- end}}
 
 {{/* The serverless environment version to use.
    * See https://docs.databricks.com/aws/en/release-notes/serverless/environment-version/
    */}}
 {{define "serverless_environment_version" -}}
-  4
+  5
 {{- end}}

libs/template/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl

@@ -31,7 +31,7 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "4"
+            environment_version: "5"
             java_dependencies:
               - ${workspace.artifact_path}/.internal/{{.project_name}}-assembly-{{template `version` .}}.jar
 {{- else}}

libs/template/templates/default/library/versions.tmpl

@@ -5,40 +5,39 @@
   16.4.x-scala2.12
 {{- end}}
 
-{{/* A safe version of DB Connect that is compatible with at least half the
-   * clusters running in production.
+{{/* The DB Connect version for local development, matched to the serverless
+   * environment version selected above.
    *
-   * We need to be very conservative in updating this, since a newer version can
-   * only connect to compute of that same version and higher. If the version is
-   * deemed too old, customers can update the version themselves after initializing
-   * the template.
+   * Environment version 5 ships DB Connect 18 (Python 3.12), so we pin to 18.0.x.
+   * DB Connect is forward-compatible (a client connects to compute of the same
+   * version and higher), so a client newer than the environment's version would
+   * not connect; matching the environment version keeps local development
+   * client-server compatible with the serverless runtime. Customers can change
+   * this after initializing the template.
    *
    * See https://docs.databricks.com/dev-tools/databricks-connect/python/index.html#requirements
    * for DB Connect release notes and version compatibility.
    */}}
 {{define "conservative_db_connect_version_spec" -}}
-  >=15.4,<15.5
+  >=18.0,<18.1
 {{- end}}
 
 {{/* The serverless environment version to use.
    * See https://docs.databricks.com/release-notes/runtime/index.html for available versions.
    */}}
 {{define "serverless_environment_version" -}}
-  4
+  5
 {{- end}}
 
 {{/* Python version specification for the template.
    * This needs to be compatible with the version of DB Connect and the serverless_environment_version.
    *
-   * Serverless environment version 4 (and upcoming v5), DBR 16, and DBR 17 all use Python 3.12.
-   * DB Connect 15 supports all of these and is compatible with Python 3.10-3.12.
-   *
-   * We use >=3.10,<3.13 rather than pinning to 3.12 specifically because overly narrow pinning
-   * means users always need to use uv or similar tools to install the exact version. A broader
-   * range improves usability while maintaining compatibility with both DB Connect and serverless.
+   * Serverless environment versions 4 and 5, DBR 16, and DBR 17 all use Python 3.12.
+   * DB Connect 16.4 requires Python 3.12, so we pin the project to 3.12 to match both
+   * DB Connect and the serverless runtime.
    */}}
 {{define "python_version_spec" -}}
-  >=3.10,<3.13
+  >=3.12,<3.13
 {{- end}}
 
 {{/* The default Python version used in notebook metadata.