Fix pipeline globs (#2723)

kanterov · shreyas-goenka · web-flow · commit 700683b432cb · 2025-04-15T15:01:06.000Z
## Changes Fix regression where combination of pipeline globs and relative paths didn't work. The regression has happened because we translated paths to paths relative to bundle root after we have expanded globs that have assumed that paths are already normalized against bundle root. Existing tests didn't cover scenario where pipeline is defined outside of bundle root path. Fixes #2721 ## Tests Add acceptance tests.  --------- Co-authored-by: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md
@@ -3,6 +3,7 @@
 ## Release v0.248.0
 
 ### Notable Changes
+* Fixed a regression with pipeline library globs introduced in 0.247.0 ([#2723](https://github.com/databricks/cli/pull/2723)). The issue caused glob patterns to fail when using paths relative to a directory that is not the bundle root.
 
 ### Dependency updates
 * Updated Go SDK to 0.63.0 ([#2716](https://github.com/databricks/cli/pull/2716))
diff --git a/acceptance/bundle/debug/out.stderr.txt b/acceptance/bundle/debug/out.stderr.txt
@@ -40,7 +40,8 @@
 10:07:59 Debug: Apply pid=12345 mutator=ResolveVariableReferences
 10:07:59 Debug: Apply pid=12345 mutator=ApplyTargetMode
 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources
-10:07:59 Debug: Apply pid=12345 mutator=NormalizePaths
+10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=ResolveVariableReferences(resources)
+10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=NormalizePaths
 10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(load)
 10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(init)
 10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(load_resources)
diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml b/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml
@@ -0,0 +1,13 @@
+bundle:
+  name: invalid_pipeline_glob_paths
+
+resources:
+  pipelines:
+    nyc_taxi_pipeline:
+      libraries:
+        - notebook: { path: "${var.notebook_dir}/*.ipynb" }
+
+variables:
+  notebook_dir:
+    description: Directory with DLT notebooks
+    default: non-existent
diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/output.txt b/acceptance/bundle/paths/invalid_pipeline_globs/output.txt
@@ -0,0 +1,22 @@
+
+>>> [CLI] bundle validate --output json
+Error: notebook non-existent/*.ipynb not found
+
+
+Exit code: 1
+{
+  "resources": {
+    "pipelines": {
+      "nyc_taxi_pipeline": {
+        "libraries": [
+          {
+            "notebook": {
+              "path": "non-existent/*.ipynb"
+            }
+          }
+        ],
+        "permissions": []
+      }
+    }
+  }
+}
diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/script b/acceptance/bundle/paths/invalid_pipeline_globs/script
@@ -0,0 +1 @@
+errcode trace $CLI bundle validate --output json | jq 'pick(.resources)'
diff --git a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/databricks.yml b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/databricks.yml
@@ -0,0 +1,10 @@
+bundle:
+  name: pipeline_expected_file_got_notebook
+
+include:
+  - resources/pipeline.yml
+
+variables:
+  notebook_dir:
+    description: Directory with DLT notebooks
+    default: notebooks
diff --git a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/notebooks/nyc_taxi_loader.py b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/notebooks/nyc_taxi_loader.py
diff --git a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/output.txt b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/output.txt
@@ -0,0 +1,22 @@
+
+>>> [CLI] bundle validate --output json
+Error: expected a file for "resources.pipelines.nyc_taxi_pipeline.libraries[0].file.path" but got a notebook: file at [TEST_TMP_DIR]/notebooks/nyc_taxi_loader.py is a notebook
+
+
+Exit code: 1
+{
+  "resources": {
+    "pipelines": {
+      "nyc_taxi_pipeline": {
+        "libraries": [
+          {
+            "file": {
+              "path": "notebooks/nyc_taxi_loader.py"
+            }
+          }
+        ],
+        "permissions": []
+      }
+    }
+  }
+}
diff --git a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/resources/pipeline.yml b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/resources/pipeline.yml
@@ -0,0 +1,6 @@
+resources:
+  pipelines:
+    nyc_taxi_pipeline:
+      libraries:
+        # path points to a notebook, not a file, it should error out
+        - file: { path: "../${var.notebook_dir}/*.py" }
diff --git a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/script b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/script
@@ -0,0 +1 @@
+errcode trace $CLI bundle validate --output json | jq 'pick(.resources)'
diff --git a/acceptance/bundle/paths/pipeline_globs/files/wrong_file.py b/acceptance/bundle/paths/pipeline_globs/files/wrong_file.py
@@ -0,0 +1 @@
+# this file should never be used, because glob points to a different file
diff --git a/acceptance/bundle/paths/pipeline_globs/notebooks/wrong_file.py b/acceptance/bundle/paths/pipeline_globs/notebooks/wrong_file.py
@@ -0,0 +1,2 @@
+# Databricks notebook source
+# this file should never be used, because glob points to a different file
diff --git a/acceptance/bundle/paths/pipeline_globs/output.txt b/acceptance/bundle/paths/pipeline_globs/output.txt
@@ -0,0 +1,60 @@
+
+>>> [CLI] bundle validate --output json
+{
+  "resources": {
+    "pipelines": {
+      "nyc_taxi_pipeline": {
+        "deployment": {
+          "kind": "BUNDLE",
+          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/state/metadata.json"
+        },
+        "libraries": [
+          {
+            "notebook": {
+              "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/notebooks/nyc_taxi_loader"
+            }
+          },
+          {
+            "file": {
+              "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/files/nyc_taxi_loader.py"
+            }
+          },
+          {
+            "notebook": {
+              "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/notebooks/nyc_taxi_loader"
+            }
+          },
+          {
+            "file": {
+              "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/files/nyc_taxi_loader.py"
+            }
+          },
+          {
+            "notebook": {
+              "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/notebooks/nyc_taxi_loader"
+            }
+          },
+          {
+            "maven": {
+              "coordinates": "org.jsoup:jsoup:1.7.2"
+            }
+          },
+          {
+            "jar": "*/*.jar"
+          },
+          {
+            "notebook": {
+              "path": "/Workspace/Users/me@company.com/*.ipynb"
+            }
+          },
+          {
+            "notebook": {
+              "path": "s3://notebooks/*.ipynb"
+            }
+          }
+        ],
+        "permissions": []
+      }
+    }
+  }
+}
diff --git a/acceptance/bundle/paths/pipeline_globs/root/databricks.yml b/acceptance/bundle/paths/pipeline_globs/root/databricks.yml
@@ -0,0 +1,13 @@
+bundle:
+  name: pipeline_glob_paths
+
+include:
+  - resources/pipeline.yml
+
+variables:
+  notebook_dir:
+    description: Directory with DLT notebooks
+    default: notebooks
+  file_dir:
+    description: Directory with DLT files
+    default: files
diff --git a/acceptance/bundle/paths/pipeline_globs/root/files/nyc_taxi_loader.py b/acceptance/bundle/paths/pipeline_globs/root/files/nyc_taxi_loader.py
@@ -0,0 +1 @@
+print("Hello from file!")
diff --git a/acceptance/bundle/paths/pipeline_globs/root/notebooks/nyc_taxi_loader.py b/acceptance/bundle/paths/pipeline_globs/root/notebooks/nyc_taxi_loader.py
@@ -0,0 +1,3 @@
+# Databricks notebook source
+
+print("Hello from notebook!")
diff --git a/acceptance/bundle/paths/pipeline_globs/root/resources/files/wrong_file.py b/acceptance/bundle/paths/pipeline_globs/root/resources/files/wrong_file.py
@@ -0,0 +1 @@
+# this file should never be used, because glob points to a different file
diff --git a/acceptance/bundle/paths/pipeline_globs/root/resources/notebooks/wrong_file.py b/acceptance/bundle/paths/pipeline_globs/root/resources/notebooks/wrong_file.py
@@ -0,0 +1,2 @@
+# Databricks notebook source
+# this file should never be used, because glob points to a different file
diff --git a/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml b/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml
@@ -0,0 +1,18 @@
+resources:
+  pipelines:
+    nyc_taxi_pipeline:
+      libraries:
+        # globs for notebooks and files are expanded
+        - notebook: { path: "../${var.notebook_dir}/*" }
+        - file: { path: "../${var.file_dir}/*" }
+        # globs can include file extensions
+        - notebook: { path: "../${var.notebook_dir}/*.py" }
+        - file: { path: "../${var.file_dir}/*.py" }
+        # non-glob files work
+        - notebook: { path: "../${var.notebook_dir}/nyc_taxi_loader.py" }
+        # maven libraries and jars remain as-is
+        - maven: { coordinates: "org.jsoup:jsoup:1.7.2" }
+        - jar: "*/*.jar"
+        # absolute paths and paths using URLs remain as-is
+        - notebook: { path: "/Workspace/Users/me@company.com/*.ipynb" }
+        - notebook: { path: "s3://notebooks/*.ipynb" }
diff --git a/acceptance/bundle/paths/pipeline_globs/script b/acceptance/bundle/paths/pipeline_globs/script
@@ -0,0 +1,3 @@
+cd root
+trace $CLI bundle validate --output json | jq 'pick(.resources)'
+rm -rf .databricks
diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/mutator"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
 )
@@ -32,6 +33,27 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia
 		return diag.FromErr(err)
 	}
 
+	// only YAML resources need to have paths normalized, before normalizing paths
+	// we need to resolve variables because they can change path values:
+	// - variable can be used a prefix
+	// - path can be part of a complex variable value
+	diags := bundle.ApplySeq(
+		ctx,
+		b,
+		// Reads (dynamic): * (strings) (searches for variable references in string values)
+		// Updates (dynamic): resources.* (strings) (resolves variable references to their actual values)
+		// Resolves variable references in 'resources' using bundle, workspace, and variables prefixes
+		mutator.ResolveVariableReferencesOnlyResources(
+			"bundle",
+			"workspace",
+			"variables",
+		),
+		mutator.NormalizePaths(),
+	)
+	if diags.HasError() {
+		return diags
+	}
+
 	return NormalizeAndInitializeResources(ctx, b, addedResources)
 }
 
diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go
@@ -121,7 +121,6 @@ func Initialize(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 
 		// Static resources (e.g. YAML) are already loaded, we initialize and normalize them before Python
 		resourcemutator.ProcessStaticResources(),
-		mutator.NormalizePaths(),
 
 		pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoad),
 		pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseInit),
diff --git a/bundle/tests/loader.go b/bundle/tests/loader.go
@@ -9,12 +9,7 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config/mutator"
 	"github.com/databricks/cli/bundle/phases"
-	"github.com/databricks/cli/libs/dbr"
 	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/databricks-sdk-go/config"
-	"github.com/databricks/databricks-sdk-go/experimental/mocks"
-	"github.com/databricks/databricks-sdk-go/service/iam"
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )
 
@@ -54,26 +49,3 @@ func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) {
 	))
 	return b, diags
 }
-
-func configureMock(t *testing.T, b *bundle.Bundle) {
-	// Configure mock workspace client
-	m := mocks.NewMockWorkspaceClient(t)
-	m.WorkspaceClient.Config = &config.Config{
-		Host: "https://mock.databricks.workspace.com",
-	}
-	m.GetMockCurrentUserAPI().EXPECT().Me(mock.Anything).Return(&iam.User{
-		UserName: "user@domain.com",
-	}, nil)
-	b.SetWorkpaceClient(m.WorkspaceClient)
-}
-
-func initializeTarget(t *testing.T, path, env string) (*bundle.Bundle, diag.Diagnostics) {
-	b := load(t, path)
-	configureMock(t, b)
-
-	ctx := dbr.MockRuntime(context.Background(), dbr.Environment{})
-	diags := bundle.Apply(ctx, b, mutator.SelectTarget(env))
-	diags = diags.Extend(phases.Initialize(ctx, b))
-
-	return b, diags
-}
diff --git a/bundle/tests/pipeline_glob_paths/databricks.yml b/bundle/tests/pipeline_glob_paths/databricks.yml
diff --git a/bundle/tests/pipeline_glob_paths_test.go b/bundle/tests/pipeline_glob_paths_test.go

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+errcode trace $CLI bundle validate --output json \| jq 'pick(.resources)'`