Simplify & fix default configuration for whl artifacts (#2987)

## Changes
- Insert regular glob into default settings instead of custom function
for finding wheels.
- Remove code to explicitly fail the build if wheels are not found.

## Why
- Helps in #2982 where we modify
glob expansion to filter out same-name wheels.
- Failed build will stop deploy regardless, because there is return code
check and glob expansion check, no need for custom code there.
- Simpler & more consistent behaviour: whl behave like those without
'type: whl'.

## Tests
Existing tests.

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>

Author: denik

acceptance/bundle/artifacts/build_and_files_whl/databricks.yml

@@ -1,7 +1,5 @@
 artifacts:
   artifact_with_custom_dist:
-    # commenting out 'type: whl' here actually makes 'bundle deploy' to build and upload correct wheel,
-    # because for non-whl types it does respect 'files' section.
     type: whl
     build: python setup.py bdist_wheel --dist-dir mydist
     files:

acceptance/bundle/artifacts/build_and_files_whl/output.txt

@@ -10,7 +10,7 @@ Validation OK!
 
 >>> errcode [CLI] bundle deploy
 Building artifact_with_custom_dist...
-Error: cannot find built wheel in [TEST_TMP_DIR] for package artifact_with_custom_dist
-
-
-Exit code: 1
+Uploading mydist/my_test_code-0.0.1-py3-none-any.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Deployment complete!

acceptance/bundle/artifacts/build_and_files_whl/test.toml

@@ -1,2 +1 @@
-Badness = "artifacts of type 'whl' hard-code dist directory even if explicit files section is present"
 RecordRequests = false

acceptance/bundle/artifacts/whl_dynamic/output.txt

@@ -14,6 +14,11 @@
   "my_test_code": {
     "build": "python setup.py bdist_wheel",
     "dynamic_version": true,
+    "files": [
+      {
+        "source": "[TEST_TMP_DIR]/my_test_code/dist/*.whl"
+      }
+    ],
     "path": "[TEST_TMP_DIR]/my_test_code",
     "type": "whl"
   }

acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json

@@ -2,6 +2,11 @@
   "artifacts": {
     "python_artifact": {
       "build": "python3 setup.py bdist_wheel",
+      "files": [
+        {
+          "source": "[TEST_TMP_DIR]/project_name_[UNIQUE_NAME]/dist/*.whl"
+        }
+      ],
       "path": "[TEST_TMP_DIR]/project_name_[UNIQUE_NAME]",
       "type": "whl"
     }

acceptance/bundle/templates/default-python/integration_classic/output.txt

@@ -54,39 +54,46 @@ Resources:
 >>> diff.py ../out.validate.dev.json ../out.summary.dev.json
 --- ../out.validate.dev.json
 +++ ../out.summary.dev.json
-@@ -45,4 +45,5 @@
+@@ -5,4 +5,6 @@
+       "files": [
+         {
++          "patched": "",
++          "remote_path": "",
+           "source": "[TEST_TMP_DIR]/project_name_[UNIQUE_NAME]/dist/*.whl"
+         }
+@@ -50,4 +52,5 @@
          "edit_mode": "UI_LOCKED",
          "format": "MULTI_TASK",
 +        "id": "[NUMID]",
          "job_clusters": [
            {
-@@ -55,5 +56,4 @@
+@@ -60,5 +63,4 @@
                "data_security_mode": "SINGLE_USER",
                "node_type_id": "[NODE_TYPE_ID]",
 -              "num_workers": 0,
                "spark_version": "15.4.x-scala2.12"
              }
-@@ -62,5 +62,4 @@
+@@ -67,5 +69,4 @@
          "max_concurrent_runs": 4,
          "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_job",
 -        "permissions": [],
          "queue": {
            "enabled": true
-@@ -113,5 +112,6 @@
+@@ -118,5 +119,6 @@
              "unit": "DAYS"
            }
 -        }
 +        },
 +        "url": "[DATABRICKS_URL]/jobs/[NUMID]"
        }
      },
-@@ -128,4 +128,5 @@
+@@ -133,4 +135,5 @@
          "development": true,
          "edition": "ADVANCED",
 +        "id": "[UUID]",
          "libraries": [
            {
-@@ -136,6 +137,6 @@
+@@ -141,6 +144,6 @@
          ],
          "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_pipeline",
 -        "permissions": [],
@@ -95,7 +102,7 @@ Resources:
 +        "url": "[DATABRICKS_URL]/pipelines/[UUID]"
        }
      }
-@@ -146,5 +147,4 @@
+@@ -151,5 +154,4 @@
      ]
    },
 -  "targets": null,
@@ -127,7 +134,7 @@ Validation OK!
 >>> diff.py ../out.validate.dev.json ../out.validate.prod.json
 --- ../out.validate.dev.json
 +++ ../out.validate.prod.json
-@@ -8,16 +8,11 @@
+@@ -13,16 +13,11 @@
    },
    "bundle": {
 -    "deployment": {
@@ -147,7 +154,7 @@ Validation OK!
 +    "target": "prod",
      "terraform": {},
      "uuid": "[UUID]"
-@@ -27,13 +22,10 @@
+@@ -32,13 +27,10 @@
      "resources/project_name_[UNIQUE_NAME].pipeline.yml"
    ],
 -  "presets": {
@@ -167,14 +174,14 @@ Validation OK!
 +  ],
    "resources": {
      "jobs": {
-@@ -41,5 +33,5 @@
+@@ -46,5 +38,5 @@
          "deployment": {
            "kind": "BUNDLE",
 -          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json"
 +          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/state/metadata.json"
          },
          "edit_mode": "UI_LOCKED",
-@@ -60,12 +52,9 @@
+@@ -65,12 +57,9 @@
            }
          ],
 -        "max_concurrent_runs": 4,
@@ -189,21 +196,21 @@ Validation OK!
 -          "dev": "[USERNAME]"
          },
          "tasks": [
-@@ -73,5 +62,5 @@
+@@ -78,5 +67,5 @@
              "job_cluster_key": "job_cluster",
              "notebook_task": {
 -              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook"
 +              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/notebook"
              },
              "task_key": "notebook_task"
-@@ -108,5 +97,5 @@
+@@ -113,5 +102,5 @@
          ],
          "trigger": {
 -          "pause_status": "PAUSED",
 +          "pause_status": "UNPAUSED",
            "periodic": {
              "interval": 1,
-@@ -120,22 +109,21 @@
+@@ -125,22 +114,21 @@
          "channel": "CURRENT",
          "configuration": {
 -          "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src"
@@ -231,7 +238,7 @@ Validation OK!
 +        "schema": "project_name_[UNIQUE_NAME]_prod"
        }
      }
-@@ -148,10 +136,10 @@
+@@ -153,10 +141,10 @@
    "targets": null,
    "workspace": {
 -    "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts",
@@ -278,7 +285,7 @@ Resources:
 >>> diff.py ../out.summary.dev.json ../out.summary.prod.json
 --- ../out.summary.dev.json
 +++ ../out.summary.prod.json
-@@ -9,15 +9,13 @@
+@@ -16,15 +16,13 @@
    "bundle": {
      "deployment": {
 -      "lock": {
@@ -298,7 +305,7 @@ Resources:
 +    "target": "prod",
      "terraform": {},
      "uuid": "[UUID]"
-@@ -27,13 +25,11 @@
+@@ -34,13 +32,11 @@
      "resources/project_name_[UNIQUE_NAME].pipeline.yml"
    ],
 -  "presets": {
@@ -319,14 +326,14 @@ Resources:
 +  "presets": {},
    "resources": {
      "jobs": {
-@@ -41,5 +37,5 @@
+@@ -48,5 +44,5 @@
          "deployment": {
            "kind": "BUNDLE",
 -          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json"
 +          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/state/metadata.json"
          },
          "edit_mode": "UI_LOCKED",
-@@ -60,11 +56,8 @@
+@@ -67,11 +63,8 @@
            }
          ],
 -        "max_concurrent_runs": 4,
@@ -340,21 +347,21 @@ Resources:
 -          "dev": "[USERNAME]"
          },
          "tasks": [
-@@ -72,5 +65,5 @@
+@@ -79,5 +72,5 @@
              "job_cluster_key": "job_cluster",
              "notebook_task": {
 -              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook"
 +              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/notebook"
              },
              "task_key": "notebook_task"
-@@ -107,5 +100,5 @@
+@@ -114,5 +107,5 @@
          ],
          "trigger": {
 -          "pause_status": "PAUSED",
 +          "pause_status": "UNPAUSED",
            "periodic": {
              "interval": 1,
-@@ -120,11 +113,10 @@
+@@ -127,11 +120,10 @@
          "channel": "CURRENT",
          "configuration": {
 -          "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src"
@@ -368,7 +375,7 @@ Resources:
 -        "development": true,
          "edition": "ADVANCED",
          "id": "[UUID]",
-@@ -132,10 +124,10 @@
+@@ -139,10 +131,10 @@
            {
              "notebook": {
 -              "path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/dlt_pipeline"
@@ -382,7 +389,7 @@ Resources:
 +        "schema": "project_name_[UNIQUE_NAME]_prod",
          "url": "[DATABRICKS_URL]/pipelines/[UUID]"
        }
-@@ -148,10 +140,10 @@
+@@ -155,10 +147,10 @@
    },
    "workspace": {
 -    "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts",

bundle/artifacts/build.go

@@ -14,7 +14,6 @@ import (
 	"github.com/databricks/cli/libs/exec"
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/cli/libs/patchwheel"
-	"github.com/databricks/cli/libs/python"
 	"github.com/databricks/cli/libs/utils"
 )
 
@@ -51,23 +50,6 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 				break
 			}
 
-			if a.Type == "whl" {
-				dir := a.Path
-				distPath := filepath.Join(a.Path, "dist")
-				wheels := python.FindFilesWithSuffixInPath(distPath, ".whl")
-				if len(wheels) == 0 {
-					diags = diags.Extend(diag.Errorf("cannot find built wheel in %s for package %s", dir, artifactName))
-					break
-				}
-				for _, wheel := range wheels {
-					a.Files = append(a.Files, config.ArtifactFile{
-						Source: wheel,
-					})
-				}
-			} else {
-				log.Warnf(ctx, "%s: Build succeeded", artifactName)
-			}
-
 			// We need to expand glob reference after build mutator is applied because
 			// if we do it before, any files that are generated by build command will
 			// not be included into artifact.Files and thus will not be uploaded.

bundle/artifacts/prepare.go

@@ -42,6 +42,15 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 			if artifact.BuildCommand == "" && len(artifact.Files) == 0 {
 				artifact.BuildCommand = python.GetExecutable() + " setup.py bdist_wheel"
 			}
+
+			// Wheel builds write to `./dist`. Pick up all wheel files by default if nothing is specified.
+			if len(artifact.Files) == 0 {
+				artifact.Files = []config.ArtifactFile{
+					{
+						Source: filepath.Join(artifact.Path, "dist", "*.whl"),
+					},
+				}
+			}
 		}
 
 		l := b.Config.GetLocation("artifacts." + artifactName)