diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 2c07236abf0..76267ac144d 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -12,6 +12,7 @@ * `bundle generate job` now downloads workspace files referenced by `spark_python_task`, rewriting them to a relative path like it already does for notebooks. Git-sourced files and cloud URIs are left untouched ([#5799](https://github.com/databricks/cli/pull/5799)). * Fix permissions added to a job or pipeline by a Python (PyDABs) mutator failing to deploy with "must have exactly one owner"; the deploying identity is now set as owner, matching resources whose permissions are declared in YAML ([#5821](https://github.com/databricks/cli/pull/5821)). +* `bundle generate` now warns when the generated configuration file is not matched by any pattern in the `include` section of `databricks.yml` ([#XXXX](https://github.com/databricks/cli/pull/XXXX)). ### Dependency updates diff --git a/acceptance/bundle/generate/alert/output.txt b/acceptance/bundle/generate/alert/output.txt index d6482446e33..8503daa3568 100644 --- a/acceptance/bundle/generate/alert/output.txt +++ b/acceptance/bundle/generate/alert/output.txt @@ -4,3 +4,11 @@ >>> [CLI] bundle generate alert --existing-id [ALERT_ID] --source-dir out/alert --config-dir out/resource Alert configuration successfully saved to [TEST_TMP_DIR]/out/resource/test_alert.alert.yml Serialized alert definition to [TEST_TMP_DIR]/out/alert/test_alert.dbalert.json +Warning: Generated configuration is not included in the bundle + +The file out/resource/test_alert.alert.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - out/resource/*.yml + diff --git a/acceptance/bundle/generate/app_not_yet_deployed/output.txt b/acceptance/bundle/generate/app_not_yet_deployed/output.txt index bbfb51b291c..5742aab1b58 100644 --- a/acceptance/bundle/generate/app_not_yet_deployed/output.txt +++ b/acceptance/bundle/generate/app_not_yet_deployed/output.txt @@ -21,3 +21,11 @@ >>> [CLI] bundle generate app --existing-app-name my-app --config-dir . --key out Loading app 'my-app' configuration App configuration successfully saved to out.app.yml +Warning: Generated configuration is not included in the bundle + +The file out.app.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - *.yml + diff --git a/acceptance/bundle/generate/app_subfolders/output.txt b/acceptance/bundle/generate/app_subfolders/output.txt index 654d38ed1d6..40c2705cae3 100644 --- a/acceptance/bundle/generate/app_subfolders/output.txt +++ b/acceptance/bundle/generate/app_subfolders/output.txt @@ -1,3 +1,11 @@ Loading app 'my_app' configuration File successfully saved to out/sub/folder/1.py App configuration successfully saved to out.app.yml +Warning: Generated configuration is not included in the bundle + +The file out.app.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - *.yml + diff --git a/acceptance/bundle/generate/dashboard/output.txt b/acceptance/bundle/generate/dashboard/output.txt index 07a77e1097e..4ff194d800f 100644 --- a/acceptance/bundle/generate/dashboard/output.txt +++ b/acceptance/bundle/generate/dashboard/output.txt @@ -4,3 +4,11 @@ >>> [CLI] bundle generate dashboard --existing-id [DASHBOARD_ID] --dashboard-dir out/dashboard --resource-dir out/resource Writing dashboard to out/dashboard/test_dashboard.lvdash.json Writing configuration to out/resource/test_dashboard.dashboard.yml +Warning: Generated configuration is not included in the bundle + +The file out/resource/test_dashboard.dashboard.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - out/resource/*.yml + diff --git a/acceptance/bundle/generate/dashboard_existing_path_nominal/output.txt b/acceptance/bundle/generate/dashboard_existing_path_nominal/output.txt index dcef0efdf55..3b76a51c84b 100644 --- a/acceptance/bundle/generate/dashboard_existing_path_nominal/output.txt +++ b/acceptance/bundle/generate/dashboard_existing_path_nominal/output.txt @@ -2,7 +2,23 @@ >>> [CLI] bundle generate dashboard --existing-path /path/to/dashboard --dashboard-dir out/dashboard --resource-dir out/resource Writing dashboard to out/dashboard/this_is_a_test_dashboard.lvdash.json Writing configuration to out/resource/this_is_a_test_dashboard.dashboard.yml +Warning: Generated configuration is not included in the bundle + +The file out/resource/this_is_a_test_dashboard.dashboard.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - out/resource/*.yml + >>> [CLI] bundle generate dashboard --existing-path /path/to/dashboard --key custom_key --dashboard-dir out/dashboard --resource-dir out/resource Writing dashboard to out/dashboard/custom_key.lvdash.json Writing configuration to out/resource/custom_key.dashboard.yml +Warning: Generated configuration is not included in the bundle + +The file out/resource/custom_key.dashboard.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - out/resource/*.yml + diff --git a/acceptance/bundle/generate/designer_job/output.txt b/acceptance/bundle/generate/designer_job/output.txt index d0009533565..44c322537b7 100644 --- a/acceptance/bundle/generate/designer_job/output.txt +++ b/acceptance/bundle/generate/designer_job/output.txt @@ -1,2 +1,10 @@ File successfully saved to outnotebook.designer.ipynb Job configuration successfully saved to out.job.yml +Warning: Generated configuration is not included in the bundle + +The file out.job.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - *.yml + diff --git a/acceptance/bundle/generate/genie_space/output.txt b/acceptance/bundle/generate/genie_space/output.txt index a313a51adc3..68334c9c1d1 100644 --- a/acceptance/bundle/generate/genie_space/output.txt +++ b/acceptance/bundle/generate/genie_space/output.txt @@ -4,3 +4,11 @@ >>> [CLI] bundle generate genie-space --existing-id [GENIE_SPACE_ID] --genie-space-dir out/genie_space --resource-dir out/resource Writing genie space to out/genie_space/test_genie_space.geniespace.json Writing configuration to out/resource/test_genie_space.genie_space.yml +Warning: Generated configuration is not included in the bundle + +The file out/resource/test_genie_space.genie_space.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - out/resource/*.yml + diff --git a/acceptance/bundle/generate/git_job/output.txt b/acceptance/bundle/generate/git_job/output.txt index 680c92ff94a..864f53e7dc1 100644 --- a/acceptance/bundle/generate/git_job/output.txt +++ b/acceptance/bundle/generate/git_job/output.txt @@ -1,2 +1,10 @@ Job is using Git source, skipping downloading files Job configuration successfully saved to out.job.yml +Warning: Generated configuration is not included in the bundle + +The file out.job.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - *.yml + diff --git a/acceptance/bundle/generate/include_warning/databricks.yml b/acceptance/bundle/generate/include_warning/databricks.yml new file mode 100644 index 00000000000..4057cf2dd10 --- /dev/null +++ b/acceptance/bundle/generate/include_warning/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: include_warning + +include: + - resources/*.yml diff --git a/acceptance/bundle/generate/include_warning/out.test.toml b/acceptance/bundle/generate/include_warning/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/generate/include_warning/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/include_warning/output.txt b/acceptance/bundle/generate/include_warning/output.txt new file mode 100644 index 00000000000..e9869179df8 --- /dev/null +++ b/acceptance/bundle/generate/include_warning/output.txt @@ -0,0 +1,16 @@ + +>>> [CLI] bundle generate job --existing-job-id 1234 --config-dir resources --key included +Job is using Git source, skipping downloading files +Job configuration successfully saved to resources/included.job.yml + +>>> [CLI] bundle generate job --existing-job-id 1234 --config-dir subdir --key excluded +Job is using Git source, skipping downloading files +Job configuration successfully saved to subdir/excluded.job.yml +Warning: Generated configuration is not included in the bundle + +The file subdir/excluded.job.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - subdir/*.yml + diff --git a/acceptance/bundle/generate/include_warning/script b/acceptance/bundle/generate/include_warning/script new file mode 100644 index 00000000000..87f7afeb37a --- /dev/null +++ b/acceptance/bundle/generate/include_warning/script @@ -0,0 +1,11 @@ +# A resource generated into a directory matched by the 'include' section +# is deployed, so no warning is emitted. +trace $CLI bundle generate job --existing-job-id 1234 --config-dir resources --key included + +# A resource generated into a directory that no 'include' pattern matches is +# written to disk but silently ignored during deploy, so we warn about it. +trace $CLI bundle generate job --existing-job-id 1234 --config-dir subdir --key excluded + +# The generated config files are not asserted on here; remove them so they are +# not reported as unexpected leftover files. +rm -r resources subdir diff --git a/acceptance/bundle/generate/include_warning/test.toml b/acceptance/bundle/generate/include_warning/test.toml new file mode 100644 index 00000000000..3a137df700c --- /dev/null +++ b/acceptance/bundle/generate/include_warning/test.toml @@ -0,0 +1,24 @@ +[[Server]] +Pattern = "GET /api/2.2/jobs/get" +Response.Body = ''' +{ + "job_id": 11223344, + "settings": { + "name": "gitjob", + "git_source": { + "git_url": "https://git.databricks.com", + "git_provider": "github", + "git_branch": "main", + "git_commit": "abcdef" + }, + "tasks": [ + { + "task_key": "test_task", + "notebook_task": { + "notebook_path": "some/test/notebook.py" + } + } + ] + } +} +''' diff --git a/acceptance/bundle/generate/ipynb_job/output.txt b/acceptance/bundle/generate/ipynb_job/output.txt index 61273cbf63e..e88d09508dc 100644 --- a/acceptance/bundle/generate/ipynb_job/output.txt +++ b/acceptance/bundle/generate/ipynb_job/output.txt @@ -1,2 +1,10 @@ File successfully saved to outnotebook.ipynb Job configuration successfully saved to out.job.yml +Warning: Generated configuration is not included in the bundle + +The file out.job.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - *.yml + diff --git a/acceptance/bundle/generate/job_nested_notebooks/output.txt b/acceptance/bundle/generate/job_nested_notebooks/output.txt index f0b8326de09..c0615715c5c 100644 --- a/acceptance/bundle/generate/job_nested_notebooks/output.txt +++ b/acceptance/bundle/generate/job_nested_notebooks/output.txt @@ -1,6 +1,14 @@ File successfully saved to src/my_folder/my_notebook.py File successfully saved to src/other_folder/other_notebook.py Job configuration successfully saved to out.job.yml +Warning: Generated configuration is not included in the bundle + +The file out.job.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - *.yml + === old flattened files should be gone === src/my_notebook.py removed src/other_notebook.py removed diff --git a/acceptance/bundle/generate/job_nested_notebooks/script b/acceptance/bundle/generate/job_nested_notebooks/script index 04805db6389..a52154efbb0 100644 --- a/acceptance/bundle/generate/job_nested_notebooks/script +++ b/acceptance/bundle/generate/job_nested_notebooks/script @@ -2,7 +2,13 @@ mkdir -p src echo "old" > src/my_notebook.py echo "old" > src/other_notebook.py -$CLI bundle generate job --existing-job-id 1234 --config-dir . --key out --force --source-dir src 2>&1 | sort +# The concurrent downloader emits the "File successfully saved" lines in +# nondeterministic order, so sort just those; the remaining lines (including the +# multi-line include warning) are deterministic and kept in their natural order. +$CLI bundle generate job --existing-job-id 1234 --config-dir . --key out --force --source-dir src > out.gen.txt 2>&1 +grep '^File successfully saved to' out.gen.txt | sort +grep -v '^File successfully saved to' out.gen.txt +rm out.gen.txt echo "=== old flattened files should be gone ===" test ! -f src/my_notebook.py && echo "src/my_notebook.py removed" || echo "src/my_notebook.py still exists" diff --git a/acceptance/bundle/generate/lakeflow_pipelines/output.txt b/acceptance/bundle/generate/lakeflow_pipelines/output.txt index b4c4e3d60aa..fbef31fbbe5 100644 --- a/acceptance/bundle/generate/lakeflow_pipelines/output.txt +++ b/acceptance/bundle/generate/lakeflow_pipelines/output.txt @@ -1,4 +1,12 @@ Pipeline configuration successfully saved to out/config/out.pipeline.yml +Warning: Generated configuration is not included in the bundle + +The file out/config/out.pipeline.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - out/config/*.yml + File successfully saved to out/pipeline/explorations/1.py File successfully saved to out/pipeline/transformations/1.py File successfully saved to out/pipeline/transformations/2.py diff --git a/acceptance/bundle/generate/pipeline/output.txt b/acceptance/bundle/generate/pipeline/output.txt index e6cf41871c1..bd1d6d9031c 100644 --- a/acceptance/bundle/generate/pipeline/output.txt +++ b/acceptance/bundle/generate/pipeline/output.txt @@ -1,2 +1,10 @@ File successfully saved to outnotebook.py Pipeline configuration successfully saved to out.pipeline.yml +Warning: Generated configuration is not included in the bundle + +The file out.pipeline.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - *.yml + diff --git a/acceptance/bundle/generate/pipeline_and_deploy/output.txt b/acceptance/bundle/generate/pipeline_and_deploy/output.txt index 41c92d9ac10..02f5f7ad198 100644 --- a/acceptance/bundle/generate/pipeline_and_deploy/output.txt +++ b/acceptance/bundle/generate/pipeline_and_deploy/output.txt @@ -9,6 +9,14 @@ === Generate bundle config from the pipelineFile successfully saved to src/notebook.py File successfully saved to src/test.py Pipeline configuration successfully saved to resources/out.pipeline.yml +Warning: Generated configuration is not included in the bundle + +The file resources/out.pipeline.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - resources/*.yml + === Verify generated yaml has expected fields === Deploy the generated bundle diff --git a/acceptance/bundle/generate/pipeline_and_deploy/script b/acceptance/bundle/generate/pipeline_and_deploy/script index 85d9e044d5f..ab6b01e6764 100644 --- a/acceptance/bundle/generate/pipeline_and_deploy/script +++ b/acceptance/bundle/generate/pipeline_and_deploy/script @@ -32,7 +32,13 @@ cleanup() { trap cleanup EXIT title "Generate bundle config from the pipeline" -$CLI bundle generate pipeline --existing-pipeline-id "$PIPELINE_ID" --key out --config-dir resources --source-dir src --force 2>&1 | sort +# The concurrent downloader emits the "File successfully saved" lines in +# nondeterministic order, so sort just those; the remaining lines (including the +# multi-line include warning) are deterministic and kept in their natural order. +$CLI bundle generate pipeline --existing-pipeline-id "$PIPELINE_ID" --key out --config-dir resources --source-dir src --force > out.gen.txt 2>&1 +grep '^File successfully saved to' out.gen.txt | sort +grep -v '^File successfully saved to' out.gen.txt +rm out.gen.txt title "Verify generated yaml has expected fields" cat resources/out.pipeline.yml | env -u MSYS_NO_PATHCONV contains.py "libraries:" "- notebook:" "path: ../src/notebook.py" "- file:" "path: ../src/test.py" > /dev/null diff --git a/acceptance/bundle/generate/pipeline_with_glob/output.txt b/acceptance/bundle/generate/pipeline_with_glob/output.txt index fbb2259bd26..fa9a5817593 100644 --- a/acceptance/bundle/generate/pipeline_with_glob/output.txt +++ b/acceptance/bundle/generate/pipeline_with_glob/output.txt @@ -1 +1,9 @@ Pipeline configuration successfully saved to out.pipeline.yml +Warning: Generated configuration is not included in the bundle + +The file out.pipeline.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - *.yml + diff --git a/acceptance/bundle/generate/pipeline_with_sql/output.txt b/acceptance/bundle/generate/pipeline_with_sql/output.txt index 3818255b6f9..230c4791b95 100644 --- a/acceptance/bundle/generate/pipeline_with_sql/output.txt +++ b/acceptance/bundle/generate/pipeline_with_sql/output.txt @@ -1,2 +1,10 @@ File successfully saved to outsql.sql Pipeline configuration successfully saved to out.pipeline.yml +Warning: Generated configuration is not included in the bundle + +The file out.pipeline.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - *.yml + diff --git a/acceptance/bundle/generate/python_job/output.txt b/acceptance/bundle/generate/python_job/output.txt index ad44285e3a9..28adf652f1d 100644 --- a/acceptance/bundle/generate/python_job/output.txt +++ b/acceptance/bundle/generate/python_job/output.txt @@ -1,2 +1,10 @@ File successfully saved to outnotebook.py Job configuration successfully saved to out.job.yml +Warning: Generated configuration is not included in the bundle + +The file out.job.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - *.yml + diff --git a/acceptance/bundle/generate/python_job_and_deploy/output.txt b/acceptance/bundle/generate/python_job_and_deploy/output.txt index aaa39b42448..8297c41e964 100644 --- a/acceptance/bundle/generate/python_job_and_deploy/output.txt +++ b/acceptance/bundle/generate/python_job_and_deploy/output.txt @@ -8,6 +8,14 @@ >>> [CLI] bundle generate job --existing-job-id [JOB_ID] --key out --config-dir resources --source-dir src --force File successfully saved to src/test_notebook.py Job configuration successfully saved to resources/out.job.yml +Warning: Generated configuration is not included in the bundle + +The file resources/out.job.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - resources/*.yml + === Verify generated yaml has expected fields === Deploy the generated bundle diff --git a/acceptance/bundle/generate/spark_python_task_job/output.txt b/acceptance/bundle/generate/spark_python_task_job/output.txt index e160a45625f..7bff1e49496 100644 --- a/acceptance/bundle/generate/spark_python_task_job/output.txt +++ b/acceptance/bundle/generate/spark_python_task_job/output.txt @@ -1,2 +1,10 @@ File successfully saved to outjob.py Job configuration successfully saved to out.job.yml +Warning: Generated configuration is not included in the bundle + +The file out.job.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - *.yml + diff --git a/cmd/bundle/generate/alert.go b/cmd/bundle/generate/alert.go index e80e559f319..cb78b24de1e 100644 --- a/cmd/bundle/generate/alert.go +++ b/cmd/bundle/generate/alert.go @@ -168,6 +168,8 @@ After generation, you can deploy this alert to other targets using: cmdio.LogString(ctx, "Alert configuration successfully saved to "+filepath.ToSlash(configPath)) cmdio.LogString(ctx, "Serialized alert definition to "+filepath.ToSlash(alertPath)) + warnIfNotIncluded(ctx, b, configPath) + return nil } diff --git a/cmd/bundle/generate/app.go b/cmd/bundle/generate/app.go index 120c405e793..845853adbdf 100644 --- a/cmd/bundle/generate/app.go +++ b/cmd/bundle/generate/app.go @@ -128,6 +128,8 @@ per target environment.`, cmdio.LogString(ctx, "App configuration successfully saved to "+filename) + warnIfNotIncluded(ctx, b, filename) + if bind { return deployment.BindResource(cmd, appKey, app.Name, true, false, true) } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 920d1064f01..086ec1d600a 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -251,6 +251,8 @@ func (d *dashboard) saveConfiguration(ctx context.Context, b *bundle.Bundle, das return err } + warnIfNotIncluded(ctx, b, resourcePath) + return nil } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 649f0cacc2c..6d938c5e03d 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -166,6 +166,8 @@ func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, ge return err } + warnIfNotIncluded(ctx, b, resourcePath) + return nil } diff --git a/cmd/bundle/generate/include.go b/cmd/bundle/generate/include.go new file mode 100644 index 00000000000..9801601ce23 --- /dev/null +++ b/cmd/bundle/generate/include.go @@ -0,0 +1,95 @@ +package generate + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" +) + +// warnIfNotIncluded warns when the freshly generated configuration file is not +// matched by any pattern in the root databricks.yml 'include' section. Such a +// file is written to disk but silently ignored during deploy, so without a +// matching 'include' entry the generate command is effectively a no-op. +func warnIfNotIncluded(ctx context.Context, b *bundle.Bundle, filename string) { + // b.Config.Include has been overwritten with the expanded list of already + // loaded files by ProcessRootIncludes, so re-read the root configuration + // file to recover the raw (unexpanded) include patterns. + rootFile, err := config.FileNames.FindInPath(b.BundleRootPath) + if err != nil { + log.Debugf(ctx, "Skipping include check: %v", err) + return + } + + root, diags := config.Load(rootFile) + if diags.HasError() { + log.Debugf(ctx, "Skipping include check: failed to load %s: %v", rootFile, diags.Error()) + return + } + + absFilename, err := filepath.Abs(filename) + if err != nil { + log.Debugf(ctx, "Skipping include check: %v", err) + return + } + + if isIncluded(b.BundleRootPath, root.Include, absFilename) { + return + } + + rel, err := filepath.Rel(b.BundleRootPath, absFilename) + if err != nil { + rel = absFilename + } + rel = filepath.ToSlash(rel) + + // Suggest a glob that covers the directory the file was written to. + suggestion := filepath.ToSlash(filepath.Join(filepath.Dir(rel), "*.yml")) + + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "Generated configuration is not included in the bundle", + Detail: fmt.Sprintf(""+ + "The file %s is not matched by any pattern in the 'include' section of databricks.yml,\n"+ + "so it will not be deployed. Add a matching entry to the 'include' section, for example:\n"+ + "\n"+ + "include:\n"+ + " - %s", + rel, suggestion), + }) +} + +// isIncluded reports whether absFilename is matched by any of the include +// patterns, resolved the same way as ProcessRootIncludes: each pattern is +// anchored to the bundle root and expanded with filepath.Glob against the +// files on disk. +func isIncluded(bundleRootPath string, include []string, absFilename string) bool { + for _, entry := range include { + // Includes must be relative paths; the loader errors on absolute ones. + if filepath.IsAbs(entry) { + continue + } + + matches, err := filepath.Glob(filepath.Join(bundleRootPath, entry)) + if err != nil { + continue + } + + for _, match := range matches { + absMatch, err := filepath.Abs(match) + if err != nil { + continue + } + if absMatch == absFilename { + return true + } + } + } + + return false +} diff --git a/cmd/bundle/generate/include_test.go b/cmd/bundle/generate/include_test.go new file mode 100644 index 00000000000..74bdd02bcc2 --- /dev/null +++ b/cmd/bundle/generate/include_test.go @@ -0,0 +1,66 @@ +package generate + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsIncluded(t *testing.T) { + // Lay out a bundle root with a couple of resource files on disk, because + // isIncluded resolves the include patterns with filepath.Glob against the + // files that actually exist. + root := t.TempDir() + included := filepath.Join(root, "resources", "my.job.yml") + nested := filepath.Join(root, "resources", "sub", "other.job.yml") + for _, f := range []string{included, nested} { + require.NoError(t, os.MkdirAll(filepath.Dir(f), 0o755)) + require.NoError(t, os.WriteFile(f, []byte("{}"), 0o644)) + } + + tests := []struct { + name string + include []string + filename string + want bool + }{ + { + name: "matched by glob", + include: []string{"resources/*.yml"}, + filename: included, + want: true, + }, + { + name: "not matched, empty include", + include: nil, + filename: included, + want: false, + }, + { + name: "glob is not recursive", + include: []string{"resources/*.yml"}, + filename: nested, + want: false, + }, + { + name: "matched by exact path", + include: []string{"resources/my.job.yml"}, + filename: included, + want: true, + }, + { + name: "absolute include pattern is ignored", + include: []string{included}, + filename: included, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, isIncluded(root, tt.include, tt.filename)) + }) + } +} diff --git a/cmd/bundle/generate/job.go b/cmd/bundle/generate/job.go index c3aba49c5f2..6bb10d30ede 100644 --- a/cmd/bundle/generate/job.go +++ b/cmd/bundle/generate/job.go @@ -147,6 +147,8 @@ After generation, you can deploy this job to other targets using: cmdio.LogString(ctx, "Job configuration successfully saved to "+filepath.ToSlash(filename)) + warnIfNotIncluded(ctx, b, filename) + if bind { return deployment.BindResource(cmd, jobKey, strconv.FormatInt(jobId, 10), true, false, true) } diff --git a/cmd/bundle/generate/pipeline.go b/cmd/bundle/generate/pipeline.go index 35fb073cadd..cdde5984d40 100644 --- a/cmd/bundle/generate/pipeline.go +++ b/cmd/bundle/generate/pipeline.go @@ -151,6 +151,8 @@ like catalogs, schemas, and compute configurations per target.`, cmdio.LogString(ctx, "Pipeline configuration successfully saved to "+filepath.ToSlash(filename)) + warnIfNotIncluded(ctx, b, filename) + if bind { return deployment.BindResource(cmd, pipelineKey, pipelineId, true, false, true) }