Change to severless clusters

.github/workflows/cd.yml

@@ -107,6 +107,15 @@ jobs:
       DATABRICKS_AUTH_TYPE: github-oidc
       DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }}
       DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID }}
+      # staging is mode: production, so its jobs run_as the service principal. Feed the SP
+      # application id (same client id used to authenticate) to the bundle variable.
+      # Casing is deliberate: the Databricks CLI reads a bundle variable from an env var named
+      # `BUNDLE_VAR_<name>` where `<name>` must match the variable EXACTLY. Our variable is the
+      # lowercase `cicd_service_principal` (in databricks.yml), so the env var is the fixed
+      # uppercase prefix `BUNDLE_VAR_` + the lowercase name. Upper-casing the tail would make the
+      # CLI look for a variable `CICD_SERVICE_PRINCIPAL`, which does not exist, and the value
+      # would be ignored (staging's run_as would then fail to resolve).
+      BUNDLE_VAR_cicd_service_principal: ${{ secrets.DATABRICKS_CLIENT_ID }}
     steps:
       - uses: actions/checkout@v4
       - uses: databricks/setup-cli@v0.9.0

databricks.yml

@@ -7,10 +7,6 @@ bundle:
   engine: direct
 
 variables:
-  shared_cluster_id:
-    description: Existing shared cluster used by workshop jobs.
-    lookup:
-      cluster: "Shared Compute - ML Workshop"
   experiment_name:
     description: MLflow experiment for model training runs (per-user path).
     default: /Users/${workspace.current_user.userName}/${bundle.target}-mlops-workshop-fraud
@@ -21,7 +17,7 @@ variables:
     default: fraud_detection
   catalog_name:
     description: Unity Catalog catalog for the environment (dev/staging/prod).
-    default: adoption_workshop
+    default: dev
   ml_schema:
     description: >-
       ML outputs schema (feature tables, model, predictions, monitoring). Resolves to the
@@ -54,6 +50,12 @@ variables:
       notebook_path is `../${var.code_root}/...`, so the two trees must mirror each other
       (the generator guarantees this).
     default: src
+  cicd_service_principal:
+    description: >-
+      Application (client) ID of the CI/CD service principal that deploys and runs the clean
+      shared targets. CI supplies the real value (via --var or BUNDLE_VAR_cicd_service_principal)
+      when deploying staging/prod. Empty by default so the personal target is unaffected.
+    default: ""
 
 include:
   # All DAB resources (jobs, models, experiments, monitors) live in ./resources.
@@ -102,8 +104,7 @@ targets:
     mode: development
     default: true
     variables:
-      # Temporary override to adoption_workshop for the workshop; revert to dev later.
-      catalog_name: adoption_workshop
+      catalog_name: dev
     # workspace:
     #   host: https://adb-xxxxxxxxxxxxxxxx.x.azuredatabricks.net  # add your workspace URL if deploying from a local CLI
 
@@ -140,13 +141,20 @@ targets:
     #   host: https://adb-xxxxxxxxxxxxxxxx.x.azuredatabricks.net  # add your workspace URL if deploying from a local CLI
 
   staging:
+    mode: production
+    run_as:
+      service_principal_name: ${var.cicd_service_principal}
     variables:
       catalog_name: staging
       auto_approve: "false"
       code_root: solution
     resources: *shared_infra
-    # workspace:
-    #   host: https://adb-xxxxxxxxxxxxxxxx.x.azuredatabricks.net  # add your workspace URL if deploying from a local CLI
+    workspace:
+      # host: https://adb-xxxxxxxxxxxxxxxx.x.azuredatabricks.net  # add your workspace URL if deploying from a local CLI
+      # Production mode needs a fixed, non-user-scoped root_path so exactly one copy is
+      # deployed and it is not tied to the deploying user. CI deploys and runs this target as
+      # the cicd_service_principal.
+      root_path: /Workspace/mlops-workshop/.bundle/${bundle.name}/${bundle.target}
 
   prod:
     mode: production

resources/batch-inference-workflow.yml

@@ -7,7 +7,6 @@ resources:
       name: ${bundle.target}-mlops-workshop-batch-inference
       tasks:
         - task_key: batch_inference
-          existing_cluster_id: ${var.shared_cluster_id}
           notebook_task:
             notebook_path: ../${var.code_root}/deployment/batch_inference/notebooks/batch_inference.py
             base_parameters:

resources/deployment-job-workflow.yml

@@ -29,7 +29,6 @@ resources:
           default: ""
       tasks:
         - task_key: Evaluation
-          existing_cluster_id: ${var.shared_cluster_id}
           max_retries: 0
           notebook_task:
             notebook_path: ../${var.code_root}/deployment/model_deployment/notebooks/evaluation.py
@@ -43,7 +42,6 @@ resources:
           depends_on:
             - task_key: Evaluation
           max_retries: 0
-          existing_cluster_id: ${var.shared_cluster_id}
           notebook_task:
             notebook_path: ../${var.code_root}/deployment/model_deployment/notebooks/approval.py
             base_parameters:
@@ -56,7 +54,6 @@ resources:
           depends_on:
             - task_key: Approval_Check
           max_retries: 0
-          existing_cluster_id: ${var.shared_cluster_id}
           notebook_task:
             notebook_path: ../${var.code_root}/deployment/model_deployment/notebooks/deployment.py
             base_parameters:

resources/feature-engineering-workflow.yml

@@ -9,7 +9,6 @@ resources:
       name: ${bundle.target}-mlops-workshop-feature-engineering
       tasks:
         - task_key: build_features
-          existing_cluster_id: ${var.shared_cluster_id}
           notebook_task:
             notebook_path: ../${var.code_root}/feature_engineering/notebooks/build_features.py
             base_parameters:

resources/model-training-workflow.yml

@@ -8,7 +8,6 @@ resources:
       name: ${bundle.target}-mlops-workshop-training
       tasks:
         - task_key: train
-          existing_cluster_id: ${var.shared_cluster_id}
           notebook_task:
             notebook_path: ../${var.code_root}/training/notebooks/training.py
             base_parameters:

resources/monitoring-resource.yml

@@ -64,7 +64,6 @@ resources:
       name: ${bundle.target}-mlops-workshop-retraining
       tasks:
         - task_key: monitored_metric_violation_check
-          existing_cluster_id: ${var.shared_cluster_id}
           notebook_task:
             notebook_path: ../${var.code_root}/monitoring/notebooks/metric_violation_check.py
             base_parameters:

resources/schemas-resource.yml

@@ -1,7 +1,7 @@
-# Per-user ML workspace schema. Declared with a BASE name (`fraud`); in development
-# mode the bundle prefixes it to `dev_<user>_fraud`, giving each user their own schema
+# Per-user ML workspace schema. Declared with a BASE name (`fraud_ml`); in development
+# mode the bundle prefixes it to `dev_<user>_fraud_ml`, giving each user their own schema
 # for the ML artifacts they produce (feature tables, model, predictions, monitoring).
-# In production mode (no prefix) it resolves to the shared `fraud`. Reads always come
+# In production mode (no prefix) it resolves to the shared `fraud_ml`. Reads always come
 # from the shared medallion schemas (fraud_bronze / fraud_silver / fraud_gold). Jobs
 # reference ${resources.schemas.ml_workspace.name} so notebooks write to exactly this
 # schema. The layer ("gold") is intentionally NOT in the name: this schema holds ML
@@ -11,5 +11,5 @@ resources:
   schemas:
     ml_workspace:
       catalog_name: ${var.catalog_name}
-      name: fraud
+      name: fraud_ml
       comment: Per-user ML outputs (feature tables, model, predictions, monitoring). Shared in prod.

resources/serving-log-processing-workflow.yml

@@ -12,7 +12,6 @@ resources:
       name: ${bundle.target}-mlops-workshop-serving-log-processing
       tasks:
         - task_key: process_serving_logs
-          existing_cluster_id: ${var.shared_cluster_id}
           notebook_task:
             notebook_path: ../${var.code_root}/monitoring/notebooks/process_serving_logs.py
             base_parameters:

solution/data_ingestion/notebooks/load_bronze_to_silver.py

@@ -29,7 +29,7 @@
 
 # COMMAND ----------
 
-dbutils.widgets.text("catalog", "adoption_workshop")
+dbutils.widgets.text("catalog", "dev")
 dbutils.widgets.text("bronze_schema", "fraud_bronze")
 dbutils.widgets.text("silver_schema", "fraud_silver")
 dbutils.widgets.text("gold_schema", "fraud_gold")

solution/data_ingestion/notebooks/load_landing_to_bronze.py

@@ -28,7 +28,7 @@
 
 # COMMAND ----------
 
-dbutils.widgets.text("catalog", "adoption_workshop")
+dbutils.widgets.text("catalog", "dev")
 dbutils.widgets.text("bronze_schema", "fraud_bronze")
 dbutils.widgets.text("landing_catalog", "dev")
 dbutils.widgets.text("landing_schema", "fraud_landing")

solution/deployment/batch_inference/notebooks/batch_inference.py

@@ -23,7 +23,23 @@
 
 # COMMAND ----------
 
-dbutils.widgets.text("catalog_name", "adoption_workshop")
+# MAGIC %md
+# MAGIC ## Setup
+# MAGIC Install the Feature Engineering client so `score_batch` can resolve the feature
+# MAGIC lookups the model was trained with (no train/serve skew).
+
+# COMMAND ----------
+
+# MAGIC %pip install -q "mlflow>=3.0" --upgrade
+# MAGIC %pip install -q databricks-feature-engineering
+
+# COMMAND ----------
+
+dbutils.library.restartPython()
+
+# COMMAND ----------
+
+dbutils.widgets.text("catalog_name", "dev")
 dbutils.widgets.text("gold_schema", "fraud_gold")
 dbutils.widgets.text("ml_schema", "")
 dbutils.widgets.text("model_name", "fraud_detection")
@@ -40,7 +56,7 @@
 if not ml_schema:
     _user = spark.range(1).select(F.current_user()).first()[0]
     _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0])
-    ml_schema = f"dev_{_short}_fraud"
+    ml_schema = f"dev_{_short}_fraud_ml"
 
 model_alias = dbutils.widgets.get("model_alias")
 model_name = dbutils.widgets.get("model_name")

solution/deployment/model_deployment/notebooks/approval.py

@@ -20,12 +20,20 @@
 
 # COMMAND ----------
 
+# MAGIC %pip install -q "mlflow>=3.0" --upgrade
+
+# COMMAND ----------
+
+dbutils.library.restartPython()
+
+# COMMAND ----------
+
 # The deployment job injects the FULL three-level model name (catalog.schema.model) into
 # `model_name`, plus the `model_version` it is gating. Both are empty when this notebook is
 # run interactively, so the fallback below rebuilds them.
 dbutils.widgets.text("model_name", "")
 dbutils.widgets.text("model_version", "")
-dbutils.widgets.text("catalog_name", "adoption_workshop")
+dbutils.widgets.text("catalog_name", "dev")
 dbutils.widgets.text("ml_schema", "")
 dbutils.widgets.text("approval_tag_name", "")
 dbutils.widgets.text("auto_approve", "false")
@@ -54,7 +62,7 @@
 if not ml_schema:
     _user = spark.range(1).select(F.current_user()).first()[0]
     _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0])
-    ml_schema = f"dev_{_short}_fraud"
+    ml_schema = f"dev_{_short}_fraud_ml"
 if not uc_model_name:
     uc_model_name = f"{catalog_name}.{ml_schema}.fraud_detection"
 if not model_version:

solution/deployment/model_deployment/notebooks/deployment.py

@@ -18,12 +18,22 @@
 
 # COMMAND ----------
 
+# MAGIC %pip install -q "mlflow>=3.0" --upgrade
+# MAGIC %pip install -q --upgrade databricks-sdk
+# MAGIC %pip install -q databricks-feature-engineering
+
+# COMMAND ----------
+
+dbutils.library.restartPython()
+
+# COMMAND ----------
+
 # The deployment job injects the FULL three-level model name (catalog.schema.model) into
 # `model_name`, plus the `model_version` that triggered it. Both are empty when this notebook
 # is run interactively, so the fallback below rebuilds them.
 dbutils.widgets.text("model_name", "")
 dbutils.widgets.text("model_version", "")
-dbutils.widgets.text("catalog_name", "adoption_workshop")
+dbutils.widgets.text("catalog_name", "dev")
 dbutils.widgets.text("ml_schema", "")
 
 uc_model_name = dbutils.widgets.get("model_name").strip()
@@ -47,7 +57,7 @@
 if not ml_schema:
     _user = spark.range(1).select(F.current_user()).first()[0]
     _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0])
-    ml_schema = f"dev_{_short}_fraud"
+    ml_schema = f"dev_{_short}_fraud_ml"
 if not uc_model_name:
     uc_model_name = f"{catalog_name}.{ml_schema}.fraud_detection"
 if not model_version:

solution/deployment/model_deployment/notebooks/evaluation.py

@@ -21,12 +21,21 @@
 
 # COMMAND ----------
 
+# MAGIC %pip install -q "mlflow>=3.0" --upgrade
+# MAGIC %pip install -q databricks-feature-engineering
+
+# COMMAND ----------
+
+dbutils.library.restartPython()
+
+# COMMAND ----------
+
 # The deployment job injects the FULL three-level model name (catalog.schema.model) into
 # `model_name`, plus the `model_version` that triggered it. Both are empty when this notebook
 # is run interactively, so the fallback below rebuilds them.
 dbutils.widgets.text("model_name", "")
 dbutils.widgets.text("model_version", "")
-dbutils.widgets.text("catalog_name", "adoption_workshop")
+dbutils.widgets.text("catalog_name", "dev")
 dbutils.widgets.text("ml_schema", "")
 dbutils.widgets.text("metric", "roc_auc")
 dbutils.widgets.text("baseline", "0.65")
@@ -54,7 +63,7 @@
 if not ml_schema:
     _user = spark.range(1).select(F.current_user()).first()[0]
     _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0])
-    ml_schema = f"dev_{_short}_fraud"
+    ml_schema = f"dev_{_short}_fraud_ml"
 if not uc_model_name:
     uc_model_name = f"{catalog_name}.{ml_schema}.fraud_detection"
 if not model_version:
@@ -76,12 +85,22 @@
 
 # COMMAND ----------
 
+import os
+
 import mlflow
 from databricks.feature_engineering import FeatureEngineeringClient
 from mlflow.tracking import MlflowClient
 from pyspark.sql import functions as F
 from sklearn.metrics import roc_auc_score
 
+# Serverless workaround: fe.score_batch runs the model via mlflow.pyfunc.spark_udf, which on
+# serverless ships the model through the DBConnect addArtifact path and then version-checks the
+# UDF sandbox with Version(runtime_version). The serverless sandbox reports a non-PEP440 tag
+# (e.g. '18.x-photon-scala2'), so that parse throws InvalidVersion. This mlflow flag skips the
+# addArtifact path and has each executor pull the model straight from the artifact store,
+# avoiding the broken check. Remove once the serverless sandbox reports a parseable version.
+os.environ["_MLFLOW_SPARK_UDF_SERVERLESS_SKIP_DBCONNECT_ARTIFACT"] = "true"
+
 mlflow.set_registry_uri("databricks-uc")
 client = MlflowClient()
 fe = FeatureEngineeringClient()

solution/deployment/model_deployment/notebooks/query_endpoint.py

@@ -19,7 +19,7 @@
 
 # COMMAND ----------
 
-dbutils.widgets.text("catalog_name", "adoption_workshop")
+dbutils.widgets.text("catalog_name", "dev")
 dbutils.widgets.text("gold_schema", "fraud_gold")
 dbutils.widgets.text("ml_schema", "")
 dbutils.widgets.text("num_requests", "50")
@@ -35,7 +35,7 @@
 if not ml_schema:
     _user = spark.range(1).select(F.current_user()).first()[0]
     _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0])
-    ml_schema = f"dev_{_short}_fraud"
+    ml_schema = f"dev_{_short}_fraud_ml"
 
 # One endpoint per user, named after the personal schema (matches the deployment job).
 endpoint_name = f"{ml_schema}_fraud"

solution/exploration/notebooks/eda.py

@@ -15,7 +15,7 @@
 
 # COMMAND ----------
 
-dbutils.widgets.text("catalog_name", "adoption_workshop")
+dbutils.widgets.text("catalog_name", "dev")
 dbutils.widgets.text("gold_schema", "fraud_gold")
 
 catalog_name = dbutils.widgets.get("catalog_name")

solution/feature_engineering/notebooks/build_features.py

@@ -24,7 +24,22 @@
 
 # COMMAND ----------
 
-dbutils.widgets.text("catalog_name", "adoption_workshop")
+# MAGIC %md
+# MAGIC ## Setup
+# MAGIC Install the Feature Engineering client. On serverless this package is not
+# MAGIC pre-installed, so we add it and restart Python. (No-op on ML Runtime clusters.)
+
+# COMMAND ----------
+
+# MAGIC %pip install -q databricks-feature-engineering
+
+# COMMAND ----------
+
+dbutils.library.restartPython()
+
+# COMMAND ----------
+
+dbutils.widgets.text("catalog_name", "dev")
 dbutils.widgets.text("gold_schema", "fraud_gold")
 dbutils.widgets.text("ml_schema", "")
 
@@ -39,7 +54,7 @@
 if not ml_schema:
     _user = spark.range(1).select(F.current_user()).first()[0]
     _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0])
-    ml_schema = f"dev_{_short}_fraud"
+    ml_schema = f"dev_{_short}_fraud_ml"
 
 # Read from the shared gold source; write to the personal schema.
 source_table = f"{catalog_name}.{gold_schema}.transactions_enriched"