Correctly translate paths to local requirements.txt file in environment dependencies (#2736)

andrewnester · Copilot · web-flow · commit 526a70abdc33 · 2025-04-22T14:33:56.000Z
## Changes
Correctly translate paths to local requirement.txt file in environment
dependencies

## Why
At the moment we don't translate paths to requirements.txt files which
makes it impossible to reference local path to such files in environment
dependencies. This PR fixes this

```
          dependencies:
              - -r ../requirements.txt
```

## Tests
Added acceptance test

&lt;!-- If your PR needs to be included in the release notes for next
release,
add a separate entry in NEXT_CHANGELOG.md as part of your PR. --&gt;

---------

Co-authored-by: Copilot &lt;175728472+Copilot@users.noreply.github.com&gt;
diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md
@@ -9,6 +9,7 @@
 ### CLI
 
 ### Bundles
+* Correctly translate paths to local requirements.txt file in environment dependencies ([#2736](https://github.com/databricks/cli/pull/2736))
 * Check for running resources with --fail-on-active-runs before any mutative operation during deploy ([#2743](https://github.com/databricks/cli/pull/2743))
 
 ### API Changes
diff --git a/acceptance/bundle/environments/dependencies/databricks.yml b/acceptance/bundle/environments/dependencies/databricks.yml
@@ -0,0 +1,29 @@
+bundle:
+  name: "dependencies"
+
+
+resources:
+  jobs:
+    test_job:
+      name: "Test Job"
+      tasks:
+        - task_key: "main"
+          python_wheel_task:
+            package_name: "test_package"
+            entry_point: "main"
+          environment_key: "test_env"
+
+      environments:
+        - environment_key: "test_env"
+          spec:
+            client: "1"
+            dependencies:
+              - "-r ./requirements.txt"
+              - "test_package"
+              - "test_package==2.0.1"
+              - "test_package>=2.0.1"
+              - "dist/*.whl"
+              - "/Workspace/Users/test@databricks.com/test-package.whl"
+              - "beautifulsoup4>=1.0.0,~=1.2.0,<2.0.0"
+              - "beautifulsoup4[security, tests] ~= 4.12.3"
+              - "requests[security] @ https://github.com/psf/requests/archive/refs/heads/main.zip"
diff --git a/acceptance/bundle/environments/dependencies/dist/test.whl b/acceptance/bundle/environments/dependencies/dist/test.whl
diff --git a/acceptance/bundle/environments/dependencies/output.txt b/acceptance/bundle/environments/dependencies/output.txt
@@ -0,0 +1,50 @@
+
+>>> [CLI] bundle validate
+Name: dependencies
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/dependencies/default
+
+Validation OK!
+
+>>> [CLI] bundle deploy
+Uploading dist/test.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments out.requests.txt
+[
+  {
+    "environment_key": "test_env",
+    "spec": {
+      "client": "1",
+      "dependencies": [
+        "-r /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/requirements.txt",
+        "test_package",
+        "test_package==2.0.1",
+        "test_package>=2.0.1",
+        "/Workspace/Users/[USERNAME]/.bundle/dependencies/default/artifacts/.internal/test.whl",
+        "/Workspace/Users/test@databricks.com/test-package.whl",
+        "beautifulsoup4>=1.0.0,~=1.2.0,<2.0.0",
+        "beautifulsoup4[security, tests] ~= 4.12.3",
+        "requests[security] @ https://github.com/psf/requests/archive/refs/heads/main.zip"
+      ]
+    }
+  }
+]
+
+>>> [CLI] bundle validate -o json
+[
+  "-r /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/requirements.txt",
+  "test_package",
+  "test_package==2.0.1",
+  "test_package>=2.0.1",
+  "./dist/*.whl",
+  "/Workspace/Users/test@databricks.com/test-package.whl",
+  "beautifulsoup4>=1.0.0,~=1.2.0,<2.0.0",
+  "beautifulsoup4[security, tests] ~= 4.12.3",
+  "requests[security] @ https://github.com/psf/requests/archive/refs/heads/main.zip"
+]
diff --git a/acceptance/bundle/environments/dependencies/requirements.txt b/acceptance/bundle/environments/dependencies/requirements.txt
@@ -0,0 +1 @@
+Flask==2.0.1
diff --git a/acceptance/bundle/environments/dependencies/script b/acceptance/bundle/environments/dependencies/script
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+trace $CLI bundle validate
+
+trace $CLI bundle deploy
+trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt
+trace $CLI bundle validate -o json | jq '.resources.jobs.test_job.environments[0].spec.dependencies'
+rm out.requests.txt
diff --git a/acceptance/bundle/environments/dependencies/test.toml b/acceptance/bundle/environments/dependencies/test.toml
@@ -0,0 +1,2 @@
+Cloud = false
+RecordRequests = true
diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go
@@ -85,9 +85,30 @@ func jobRewritePatterns() []jobRewritePattern {
 		},
 	}
 
+	jobEnvironmentsWithRequirementsPatterns := []jobRewritePattern{
+		{
+			dyn.NewPattern(
+				dyn.Key("resources"),
+				dyn.Key("jobs"),
+				dyn.AnyKey(),
+				dyn.Key("environments"),
+				dyn.AnyIndex(),
+				dyn.Key("spec"),
+				dyn.Key("dependencies"),
+				dyn.AnyIndex(),
+			),
+			TranslateModeEnvironmentRequirements,
+			func(s string) bool {
+				_, ok := libraries.IsLocalRequirementsFile(s)
+				return !ok
+			},
+		},
+	}
+
 	taskPatterns := jobTaskRewritePatterns(base)
 	forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task")))
 	allPatterns := append(taskPatterns, jobEnvironmentsPatterns...)
+	allPatterns = append(allPatterns, jobEnvironmentsWithRequirementsPatterns...)
 	allPatterns = append(allPatterns, forEachPatterns...)
 	return allPatterns
 }
diff --git a/bundle/config/mutator/paths/translation_mode.go b/bundle/config/mutator/paths/translation_mode.go
@@ -29,4 +29,7 @@ const (
 	// It a "./" prefix to the path if it does not already have one.
 	// This allows for disambiguating between paths and PyPI package names.
 	TranslateModeLocalRelativeWithPrefix
+
+	// TranslateModeEnvironmentRequirements translates a local requirements file path to be absolute.
+	TranslateModeEnvironmentRequirements
 )
diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go
@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"github.com/databricks/cli/bundle/config/mutator/paths"
+	"github.com/databricks/cli/bundle/libraries"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
@@ -91,6 +92,11 @@ func (t *translateContext) rewritePath(
 	input string,
 	opts translateOptions,
 ) (string, error) {
+	// If the input is a local requirements file, we need to translate it to an absolute path.
+	if reqPath, ok := libraries.IsLocalRequirementsFile(input); ok {
+		input = reqPath
+	}
+
 	// We assume absolute paths point to a location in the workspace
 	if path.IsAbs(input) {
 		return "", nil
@@ -143,6 +149,10 @@ func (t *translateContext) rewritePath(
 		interp, err = t.translateLocalRelativePath(ctx, input, localPath, localRelPath)
 	case paths.TranslateModeLocalRelativeWithPrefix:
 		interp, err = t.translateLocalRelativeWithPrefixPath(ctx, input, localPath, localRelPath)
+	case paths.TranslateModeEnvironmentRequirements:
+		interp, err = t.translateFilePath(ctx, input, localPath, localRelPath)
+		// Add the -r flag to the path to indicate it's a requirements file used for environment dependencies.
+		interp = "-r " + interp
 	default:
 		return "", fmt.Errorf("unsupported translate mode: %d", opts.Mode)
 	}
diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go
@@ -64,7 +64,10 @@ func IsLibraryLocal(dep string) bool {
 		return false
 	}
 
-	// If the dependency is a requirements file, it's not a valid local path
+	// If the dependency is a requirements file, it can either be a local path or a remote path.
+	// Even though the path to requirements.txt can be local we don't return true in this function anyway
+	// and don't treat such path as a local library path.
+	// Instead we handle translation of these paths in a separate code path in TranslatePath mutator.
 	if strings.HasPrefix(dep, "-r") {
 		return false
 	}
@@ -77,6 +80,16 @@ func IsLibraryLocal(dep string) bool {
 	return IsLocalPath(dep)
 }
 
+func IsLocalRequirementsFile(dep string) (string, bool) {
+	dep, ok := strings.CutPrefix(dep, "-r ")
+	if !ok {
+		return "", false
+	}
+
+	dep = strings.TrimSpace(dep)
+	return dep, IsLocalPath(dep)
+}
+
 func containsPipFlag(input string) bool {
 	re := regexp.MustCompile(`--[a-zA-Z0-9-]+`)
 	return re.MatchString(input)
diff --git a/bundle/libraries/local_path_test.go b/bundle/libraries/local_path_test.go
@@ -52,6 +52,7 @@ func TestIsLibraryLocal(t *testing.T) {
 		{path: "pypipackage", expected: false},
 		{path: "/Volumes/catalog/schema/volume/path.whl", expected: false},
 		{path: "/Workspace/my_project/dist.whl", expected: false},
+		{path: "-r ../requirements.txt", expected: false},
 		{path: "-r /Workspace/my_project/requirements.txt", expected: false},
 		{path: "s3://mybucket/path/to/package", expected: false},
 		{path: "dbfs:/mnt/path/to/package", expected: false},
@@ -86,3 +87,51 @@ func TestIsLibraryLocal(t *testing.T) {
 		require.Equalf(t, tc.expected, IsLibraryLocal(tc.path), "failed case: %d, path: %s", i, tc.path)
 	}
 }
+
+func TestIsLocalRequirementsFile(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+		isLocal  bool
+	}{
+		{
+			name:     "valid requirements file with space",
+			input:    "-r requirements.txt",
+			expected: "requirements.txt",
+			isLocal:  true,
+		},
+		{
+			name:     "remote requirements file",
+			input:    "-r /Workspace/Users/requirements.txt",
+			expected: "/Workspace/Users/requirements.txt",
+			isLocal:  false,
+		},
+		{
+			name:     "not a requirements file",
+			input:    "some.txt",
+			expected: "",
+			isLocal:  false,
+		},
+		{
+			name:     "-r with no space",
+			input:    "-rrequirements.txt",
+			expected: "",
+			isLocal:  false,
+		},
+		{
+			name:     "empty string",
+			input:    "",
+			expected: "",
+			isLocal:  false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, isLocal := IsLocalRequirementsFile(tt.input)
+			require.Equal(t, tt.expected, got)
+			require.Equal(t, tt.isLocal, isLocal)
+		})
+	}
+}

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+Cloud = false`
	`2`	`+RecordRequests = true`
Original file line number	Diff line number	Diff line change
`@@ -29,4 +29,7 @@ const (`
`29`	`29`	`// It a "./" prefix to the path if it does not already have one.`
`30`	`30`	`// This allows for disambiguating between paths and PyPI package names.`
`31`	`31`	`TranslateModeLocalRelativeWithPrefix`
	`32`	`+`
	`33`	`+ // TranslateModeEnvironmentRequirements translates a local requirements file path to be absolute.`
	`34`	`+ TranslateModeEnvironmentRequirements`
`32`	`35`	`)`