From 5548737f87e8e8fcbf52c44c211db932ba3ef297 Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Wed, 15 Jul 2026 20:09:14 +0200 Subject: [PATCH 1/4] refactor(bundle): align workshop naming with repo and clarify TODOs Feedback from the workshop run: - Rename the bundle naming token from mlops-workshop to databricks-mlops-workshop so job names, the workspace root_path and the MLflow experiment all match the repo. Also fix the training notebook's interactive experiment fallback, which pointed at a different path than the bundle-created experiment. - Rename the fraud_deployment_job bundle key to deployment_job (the fraud_ prefix did not match its resulting name). - Rename the metric_violation_check notebook and its task/widget/task-value keys to model_drift_check / is_model_drift_violated / model_drift_threshold. - Wrap generated TODO gaps in a banner with surrounding blank lines so they stand out from provided code. --- README.md | 2 +- databricks.yml | 6 ++--- resources/batch-inference-workflow.yml | 2 +- resources/data-ingestion-workflow.yml | 2 +- resources/deployment-job-workflow.yml | 4 ++-- resources/feature-engineering-workflow.yml | 2 +- resources/model-training-workflow.yml | 4 ++-- resources/monitoring-resource.yml | 16 ++++++------- resources/serving-log-processing-workflow.yml | 2 +- scripts/generate_src.py | 23 +++++++++++++++++- solution/README.md | 2 +- solution/monitoring/README.md | 2 +- .../notebooks/feature_drift_check.py | 2 +- ...iolation_check.py => model_drift_check.py} | 24 +++++++++---------- solution/monitoring/notebooks/monitoring.py | 2 +- solution/training/notebooks/training.py | 2 +- src/README.md | 2 +- .../notebooks/batch_inference.py | 4 ++++ .../notebooks/build_features.py | 4 ++++ src/monitoring/README.md | 2 +- .../notebooks/feature_drift_check.py | 2 +- ...iolation_check.py => model_drift_check.py} | 24 +++++++++---------- src/monitoring/notebooks/monitoring.py | 6 ++++- .../notebooks/process_serving_logs.py | 2 ++ src/training/notebooks/training.py | 7 +++++- 25 files changed, 95 insertions(+), 55 deletions(-) rename solution/monitoring/notebooks/{metric_violation_check.py => model_drift_check.py} (85%) rename src/monitoring/notebooks/{metric_violation_check.py => model_drift_check.py} (85%) diff --git a/README.md b/README.md index c5b5fe2..501105c 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ Accuracy is misleading on imbalanced data, so the tracked metrics are: | Serving | Deploy champion via the deployment job; test the endpoint with `query_endpoint` | | Monitoring | Batch-score to an inference table; watch drift + precision/recall; trigger retrain | -> **The Serving step is slow.** The deployment job (`fraud_deployment_job`) provisions a +> **The Serving step is slow.** The deployment job (`deployment_job`) provisions a > **Lakebase online feature store**, publishes the `card_features`/`client_features` tables to > it, and builds a first-time serving endpoint, which typically takes **15-25 minutes** > end-to-end. It runs automatically when a new model version is registered; allow it to diff --git a/databricks.yml b/databricks.yml index 7d76584..441caf0 100644 --- a/databricks.yml +++ b/databricks.yml @@ -3,13 +3,13 @@ # Docs: Declarative Automation Bundles https://learn.microsoft.com/azure/databricks/dev-tools/bundles/ # Settings & targets https://learn.microsoft.com/azure/databricks/dev-tools/bundles/settings bundle: - name: mlops_workshop + name: databricks_mlops_workshop engine: direct variables: experiment_name: description: MLflow experiment for model training runs (per-user path). - default: /Users/${workspace.current_user.userName}/${bundle.target}-mlops-workshop-fraud + default: /Users/${workspace.current_user.userName}/${bundle.target}-databricks-mlops-workshop model_name: description: >- Bare (unqualified) registered model name. Notebooks prepend catalog_name + ml_schema @@ -169,4 +169,4 @@ targets: # deployed. Use a dedicated (non-/Shared) folder: /Shared is writable by ALL workspace # users, so anyone could tamper with prod code. In a real prod you would deploy as a # service principal and restrict write ACLs on this folder to that identity. - root_path: /Workspace/mlops-workshop/.bundle/${bundle.name}/${bundle.target} + root_path: /Workspace/databricks-mlops-workshop/.bundle/${bundle.name}/${bundle.target} diff --git a/resources/batch-inference-workflow.yml b/resources/batch-inference-workflow.yml index b0d226f..2a70b12 100644 --- a/resources/batch-inference-workflow.yml +++ b/resources/batch-inference-workflow.yml @@ -4,7 +4,7 @@ resources: jobs: batch_inference_job: - name: ${bundle.target}-mlops-workshop-batch-inference + name: ${bundle.target}-databricks-mlops-workshop-batch-inference tasks: - task_key: batch_inference notebook_task: diff --git a/resources/data-ingestion-workflow.yml b/resources/data-ingestion-workflow.yml index 78244ff..a0baede 100644 --- a/resources/data-ingestion-workflow.yml +++ b/resources/data-ingestion-workflow.yml @@ -6,7 +6,7 @@ resources: jobs: data_ingestion_job: - name: ${bundle.target}-mlops-workshop-data-ingestion + name: ${bundle.target}-databricks-mlops-workshop-data-ingestion tasks: - task_key: landing_to_bronze notebook_task: diff --git a/resources/deployment-job-workflow.yml b/resources/deployment-job-workflow.yml index fa6d745..4f87298 100644 --- a/resources/deployment-job-workflow.yml +++ b/resources/deployment-job-workflow.yml @@ -17,8 +17,8 @@ # Docs: MLflow 3 deployment jobs https://learn.microsoft.com/azure/databricks/mlflow/deployment-job resources: jobs: - fraud_deployment_job: - name: ${bundle.target}-mlops-workshop-deployment-job + deployment_job: + name: ${bundle.target}-databricks-mlops-workshop-deployment-job max_concurrent_runs: 1 queue: enabled: true diff --git a/resources/feature-engineering-workflow.yml b/resources/feature-engineering-workflow.yml index b34abad..872919d 100644 --- a/resources/feature-engineering-workflow.yml +++ b/resources/feature-engineering-workflow.yml @@ -6,7 +6,7 @@ resources: jobs: feature_engineering_job: - name: ${bundle.target}-mlops-workshop-feature-engineering + name: ${bundle.target}-databricks-mlops-workshop-feature-engineering tasks: - task_key: build_features notebook_task: diff --git a/resources/model-training-workflow.yml b/resources/model-training-workflow.yml index e5a3425..1c041c6 100644 --- a/resources/model-training-workflow.yml +++ b/resources/model-training-workflow.yml @@ -5,7 +5,7 @@ resources: jobs: model_training_job: - name: ${bundle.target}-mlops-workshop-training + name: ${bundle.target}-databricks-mlops-workshop-training tasks: - task_key: train notebook_task: @@ -24,4 +24,4 @@ resources: # Connect this model to its deployment job so registering a new version # auto-triggers the governed evaluate -> approve -> deploy pipeline. The # validation + serving that used to be separate tasks now live in that job. - deployment_job_id: ${resources.jobs.fraud_deployment_job.id} + deployment_job_id: ${resources.jobs.deployment_job.id} diff --git a/resources/monitoring-resource.yml b/resources/monitoring-resource.yml index afd5852..1b21acc 100644 --- a/resources/monitoring-resource.yml +++ b/resources/monitoring-resource.yml @@ -61,24 +61,24 @@ resources: # across recent windows, run the training job (which retrains on the latest labelled # data -> the actual remedy for drift once fresh labels have landed). retraining_job: - name: ${bundle.target}-mlops-workshop-retraining + name: ${bundle.target}-databricks-mlops-workshop-retraining tasks: - - task_key: monitored_metric_violation_check + - task_key: model_drift_check notebook_task: - notebook_path: ../${var.code_root}/monitoring/notebooks/metric_violation_check.py + notebook_path: ../${var.code_root}/monitoring/notebooks/model_drift_check.py base_parameters: table_name_under_monitor: ${var.catalog_name}.${resources.schemas.ml_workspace.name}.fraud_predictions metric_to_monitor: f1_score.macro - metric_violation_threshold: "0.20" + model_drift_threshold: "0.20" num_evaluation_windows: "5" num_violation_windows: "2" - - task_key: is_metric_violated + - task_key: is_model_drift_violated depends_on: - - task_key: monitored_metric_violation_check + - task_key: model_drift_check condition_task: op: EQUAL_TO - left: "{{tasks.monitored_metric_violation_check.values.is_metric_violated}}" + left: "{{tasks.model_drift_check.values.is_model_drift_violated}}" right: "true" # Feature (data) drift check: sustained input drift on any watched feature is a @@ -107,7 +107,7 @@ resources: # (it runs as long as at least one true-branch was taken, the other is just skipped). - task_key: trigger_retraining depends_on: - - task_key: is_metric_violated + - task_key: is_model_drift_violated outcome: "true" - task_key: is_feature_drift_violated outcome: "true" diff --git a/resources/serving-log-processing-workflow.yml b/resources/serving-log-processing-workflow.yml index e6dd85e..6851c00 100644 --- a/resources/serving-log-processing-workflow.yml +++ b/resources/serving-log-processing-workflow.yml @@ -9,7 +9,7 @@ resources: jobs: serving_log_processing_job: - name: ${bundle.target}-mlops-workshop-serving-log-processing + name: ${bundle.target}-databricks-mlops-workshop-serving-log-processing tasks: - task_key: process_serving_logs notebook_task: diff --git a/scripts/generate_src.py b/scripts/generate_src.py index 08322f2..6507fdf 100644 --- a/scripts/generate_src.py +++ b/scripts/generate_src.py @@ -40,6 +40,12 @@ HINT = "# HINT" PLACEHOLDER = "# <-- Your code here" +# Width of the banner rules that wrap each generated TODO block so the gaps stand out +# clearly from the surrounding provided code. +_BANNER = "-" * 20 +TODO_TOP = f"# {_BANNER} TODO {_BANNER}" +TODO_BOTTOM = "# " + "-" * (len(TODO_TOP) - 2) + def strip_solution(text: str) -> str: """Replace every TODO-BEGIN..TODO-END block with the hint, any retained HINT @@ -49,22 +55,37 @@ def strip_solution(text: str) -> str: * the text after ``TODO-BEGIN:`` becomes the ``# TODO:`` summary line, * any ``# HINT:`` comment lines are kept verbatim (guidance for participants), * all other (solution) lines are removed and replaced by a single placeholder. - Files with no markers pass through unchanged.""" + Each generated block is wrapped in a banner and blank lines so the gap stands out + from the provided code around it. Files with no markers pass through unchanged.""" lines = text.splitlines() out: list[str] = [] in_block = False indent = "" + swallow_blank = False for line in lines: stripped = line.lstrip() + # Drop a single blank line that immediately follows a block: we add our own + # trailing blank, so this avoids a double gap before the next line. + if swallow_blank: + swallow_blank = False + if stripped == "": + continue if stripped.startswith(BEGIN): in_block = True indent = line[: len(line) - len(stripped)] hint = stripped[len(BEGIN) :].lstrip(": ").rstrip() + # Blank line before the banner for a clear separation from preceding code. + if out and out[-1].strip() != "": + out.append("") + out.append(f"{indent}{TODO_TOP}") out.append(f"{indent}# TODO: {hint}" if hint else f"{indent}# TODO") continue if stripped.startswith(END): in_block = False out.append(f"{indent}{PLACEHOLDER}") + out.append(f"{indent}{TODO_BOTTOM}") + out.append("") + swallow_blank = True continue if in_block: # Retain author-provided hints; drop everything else (the solution). diff --git a/solution/README.md b/solution/README.md index bbe4491..e5553b8 100644 --- a/solution/README.md +++ b/solution/README.md @@ -9,4 +9,4 @@ Each stage has its notebook entrypoints in `/notebooks/`. | `feature_engineering/` | Build the card/client entity feature tables and on-demand feature functions. | | `training/` | Train, track and register the fraud detection model. | | `deployment/` | Promote and serve the champion model (deployment job + endpoint) and run batch inference. | -| `monitoring/` | Refresh the Lakehouse Monitors, process serving logs, and check for metric violations that trigger retraining. | +| `monitoring/` | Refresh the Lakehouse Monitors, process serving logs, and check for model drift that triggers retraining. | diff --git a/solution/monitoring/README.md b/solution/monitoring/README.md index 2cab4b7..faaf9f6 100644 --- a/solution/monitoring/README.md +++ b/solution/monitoring/README.md @@ -4,6 +4,6 @@ |---|---| | `notebooks/monitoring.py` | Refresh the batch and online Lakehouse Monitors and inspect the profile/drift metric tables they produce. | | `notebooks/process_serving_logs.py` | Unpack the serving endpoint's raw inference-table payloads into typed rows so the online monitor can profile them. | -| `notebooks/metric_violation_check.py` | Retraining-trigger task: run the violation SQL over the monitor's profile metrics to decide whether model quality has degraded across recent windows. | +| `notebooks/model_drift_check.py` | Retraining-trigger task: run the drift SQL over the monitor's profile metrics to decide whether model quality has degraded across recent windows. | | `notebooks/feature_drift_check.py` | Retraining-trigger task: threshold the monitor's `_drift_metrics` to decide whether sustained feature (data) drift on any watched column should trigger retraining. | | `notebooks/seed_drift.py` | Demo helper (attendee-runnable): sample already-scored predictions, inflate the amount-driven features, append them to your own `fraud_predictions` with today's timestamp, and refresh your batch monitor so feature drift is visible during the session. | diff --git a/solution/monitoring/notebooks/feature_drift_check.py b/solution/monitoring/notebooks/feature_drift_check.py index 2d8bf00..1567a7a 100644 --- a/solution/monitoring/notebooks/feature_drift_check.py +++ b/solution/monitoring/notebooks/feature_drift_check.py @@ -11,7 +11,7 @@ # MAGIC # MAGIC Related notebooks: # MAGIC -# MAGIC - `metric_violation_check.py`: the same pattern for model quality (`..._profile_metrics`). +# MAGIC - `model_drift_check.py`: the same pattern for model quality (`..._profile_metrics`). # MAGIC - `monitoring.py`: the interactive dashboard that displays these metrics. # MAGIC # MAGIC Quality and drift live in separate tables: `..._profile_metrics` holds quality metrics diff --git a/solution/monitoring/notebooks/metric_violation_check.py b/solution/monitoring/notebooks/model_drift_check.py similarity index 85% rename from solution/monitoring/notebooks/metric_violation_check.py rename to solution/monitoring/notebooks/model_drift_check.py index 05b6ee9..8cad524 100644 --- a/solution/monitoring/notebooks/metric_violation_check.py +++ b/solution/monitoring/notebooks/model_drift_check.py @@ -1,12 +1,12 @@ # Databricks notebook source # MAGIC %md -# MAGIC # Monitored-metric violation check (instructor-provided) +# MAGIC # Model drift check (instructor-provided) # MAGIC # MAGIC **Session:** Monitoring & Retraining # MAGIC # MAGIC Automated task in the scheduled `retraining_job`. It reads the monitor's # MAGIC `..._profile_metrics` table, decides whether model quality has degraded across recent -# MAGIC windows, and publishes the boolean `is_metric_violated` as a task value. A downstream +# MAGIC windows, and publishes the boolean `is_model_drift_violated` as a task value. A downstream # MAGIC `condition_task` reads that value and triggers `model_training_job` only when quality # MAGIC has degraded. # MAGIC @@ -19,7 +19,7 @@ # MAGIC # MAGIC - `table_name_under_monitor`: the inference table the monitor profiles. # MAGIC - `metric_to_monitor`: quality metric column in `..._profile_metrics` (e.g. `f1_score`). -# MAGIC - `metric_violation_threshold`: retrain if the metric drops below this. +# MAGIC - `model_drift_threshold`: retrain if the metric drops below this. # MAGIC - `num_evaluation_windows` / `num_violation_windows`: how many recent windows to check, # MAGIC and how many must be in violation before retraining. # MAGIC @@ -32,7 +32,7 @@ ) dbutils.widgets.text("metric_to_monitor", "f1_score.macro", label="Quality metric to monitor") dbutils.widgets.text( - "metric_violation_threshold", "0.20", label="Retrain if the metric drops below this" + "model_drift_threshold", "0.20", label="Retrain if the metric drops below this" ) dbutils.widgets.text("num_evaluation_windows", "5", label="Number of recent windows to check") dbutils.widgets.text("num_violation_windows", "2", label="Windows that must violate to trigger") @@ -51,7 +51,7 @@ # MAGIC window also below it? # MAGIC # MAGIC Fraud quality (e.g. `f1_score.macro`) is "higher is better", so a violation means the -# MAGIC metric dropped **below** `metric_violation_threshold`. (For a drift or error metric +# MAGIC metric dropped **below** `model_drift_threshold`. (For a drift or error metric # MAGIC where "higher is worse", flip `<` to `>`.) Requiring the most recent window to be in # MAGIC violation avoids retraining on a dip that has already recovered. # MAGIC @@ -72,7 +72,7 @@ table_name_under_monitor = dbutils.widgets.get("table_name_under_monitor") metric_to_monitor = dbutils.widgets.get("metric_to_monitor") -metric_violation_threshold = float(dbutils.widgets.get("metric_violation_threshold")) +model_drift_threshold = float(dbutils.widgets.get("model_drift_threshold")) num_evaluation_windows = int(dbutils.widgets.get("num_evaluation_windows")) num_violation_windows = int(dbutils.widgets.get("num_violation_windows")) @@ -82,7 +82,7 @@ # nothing to evaluate, so report "not violated" and the job does not retrain. if not spark.catalog.tableExists(profile_metrics_table): print(f"{profile_metrics_table} does not exist yet; treating as not violated.") - dbutils.jobs.taskValues.set("is_metric_violated", False) + dbutils.jobs.taskValues.set("is_model_drift_violated", False) dbutils.notebook.exit("no_metrics") # COMMAND ---------- @@ -109,10 +109,10 @@ # Violation: at least num_violation_windows of those windows are below the threshold and the # most recent window is also below it (so a recovered dip does not trigger retraining). # Fraud quality is "higher is better"; for a drift/error metric flip < to >. -windows_in_violation = sum(1 for row in recent_metrics if row["metric_value"] < metric_violation_threshold) -latest_in_violation = bool(recent_metrics) and recent_metrics[0]["metric_value"] < metric_violation_threshold -is_metric_violated = windows_in_violation >= num_violation_windows and latest_in_violation -print(f"is_metric_violated = {is_metric_violated}") +windows_in_violation = sum(1 for row in recent_metrics if row["metric_value"] < model_drift_threshold) +latest_in_violation = bool(recent_metrics) and recent_metrics[0]["metric_value"] < model_drift_threshold +is_model_drift_violated = windows_in_violation >= num_violation_windows and latest_in_violation +print(f"is_model_drift_violated = {is_model_drift_violated}") # Publish the decision for the downstream condition_task in the retraining job. -dbutils.jobs.taskValues.set("is_metric_violated", is_metric_violated) +dbutils.jobs.taskValues.set("is_model_drift_violated", is_model_drift_violated) diff --git a/solution/monitoring/notebooks/monitoring.py b/solution/monitoring/notebooks/monitoring.py index 49acbca..7d6f89d 100644 --- a/solution/monitoring/notebooks/monitoring.py +++ b/solution/monitoring/notebooks/monitoring.py @@ -7,7 +7,7 @@ # MAGIC Interactive dashboard. It refreshes the two Lakehouse Monitors and displays the metric # MAGIC tables they produce; it makes no decisions and is not part of the retraining job. The # MAGIC automated decisions run headless in two job tasks that read the same tables: -# MAGIC `metric_violation_check.py` (model quality) and `feature_drift_check.py` (feature drift). +# MAGIC `model_drift_check.py` (model quality) and `feature_drift_check.py` (feature drift). # MAGIC # MAGIC The bundle creates two monitors: # MAGIC diff --git a/solution/training/notebooks/training.py b/solution/training/notebooks/training.py index a5d7469..798d242 100644 --- a/solution/training/notebooks/training.py +++ b/solution/training/notebooks/training.py @@ -68,7 +68,7 @@ # where the experiment/model widgets may be empty. current_user = spark.range(1).select(F.current_user()).first()[0] experiment_name = ( - dbutils.widgets.get("experiment_name") or f"/Users/{current_user}/mlops-workshop-fraud" + dbutils.widgets.get("experiment_name") or f"/Users/{current_user}/databricks-mlops-workshop" ) model_name = dbutils.widgets.get("model_name") # Accept either a bare name (prepend catalog + ml_schema) or an already-qualified 3-level name. diff --git a/src/README.md b/src/README.md index bbe4491..e5553b8 100644 --- a/src/README.md +++ b/src/README.md @@ -9,4 +9,4 @@ Each stage has its notebook entrypoints in `/notebooks/`. | `feature_engineering/` | Build the card/client entity feature tables and on-demand feature functions. | | `training/` | Train, track and register the fraud detection model. | | `deployment/` | Promote and serve the champion model (deployment job + endpoint) and run batch inference. | -| `monitoring/` | Refresh the Lakehouse Monitors, process serving logs, and check for metric violations that trigger retraining. | +| `monitoring/` | Refresh the Lakehouse Monitors, process serving logs, and check for model drift that triggers retraining. | diff --git a/src/deployment/batch_inference/notebooks/batch_inference.py b/src/deployment/batch_inference/notebooks/batch_inference.py index 4f2847e..6e6e6fe 100644 --- a/src/deployment/batch_inference/notebooks/batch_inference.py +++ b/src/deployment/batch_inference/notebooks/batch_inference.py @@ -139,10 +139,12 @@ # COMMAND ---------- +# -------------------- TODO -------------------- # TODO: score the batch with the champion model # HINT: fe.score_batch(model_uri=f"models:/{uc_model_name}@{model_alias}", df=to_score). # HINT: the result has the lookup key plus a "prediction" column (the monitor's prediction_col). # <-- Your code here +# ---------------------------------------------- # COMMAND ---------- @@ -207,10 +209,12 @@ ) ) +# -------------------- TODO -------------------- # TODO: append the predictions to the monitored inference table # HINT: predictions.write.mode("append").option("mergeSchema", "true").saveAsTable(predictions_table) # HINT: append (not overwrite) so the table grows each run and the monitor sees history. # <-- Your code here +# ---------------------------------------------- # COMMAND ---------- diff --git a/src/feature_engineering/notebooks/build_features.py b/src/feature_engineering/notebooks/build_features.py index 158044e..cc729cd 100644 --- a/src/feature_engineering/notebooks/build_features.py +++ b/src/feature_engineering/notebooks/build_features.py @@ -117,12 +117,14 @@ # COMMAND ---------- +# -------------------- TODO -------------------- # TODO: create the feature tables (first run) or merge into them (re-runs) # HINT: use the FeatureEngineeringClient `fe`, primary_keys=["card_id"] / ["client_id"]. # HINT: fe.create_table(name=..., primary_keys=..., df=..., description=...) creates it. # HINT: if it already exists, fe.write_table(name=..., df=..., mode="merge"). # HINT: enable Change Data Feed (delta.enableChangeDataFeed=true) so it can publish online. # <-- Your code here +# ---------------------------------------------- # COMMAND ---------- @@ -142,11 +144,13 @@ # the exercise. schema_fqn = f"{catalog_name}.{ml_schema}" +# -------------------- TODO -------------------- # TODO: register the ff_is_night on-demand feature function as a UC Python UDF # HINT: CREATE OR REPLACE FUNCTION {schema_fqn}.ff_is_night(transaction_hour INT) RETURNS BOOLEAN # HINT: LANGUAGE PYTHON AS $$ $$ (use spark.sql(f"""...""")). # HINT: is_night(hour) -> hour is not None and (hour < 6 or hour >= 22). # <-- Your code here +# ---------------------------------------------- # The remaining on-demand functions (provided). spark.sql(f""" diff --git a/src/monitoring/README.md b/src/monitoring/README.md index 2cab4b7..faaf9f6 100644 --- a/src/monitoring/README.md +++ b/src/monitoring/README.md @@ -4,6 +4,6 @@ |---|---| | `notebooks/monitoring.py` | Refresh the batch and online Lakehouse Monitors and inspect the profile/drift metric tables they produce. | | `notebooks/process_serving_logs.py` | Unpack the serving endpoint's raw inference-table payloads into typed rows so the online monitor can profile them. | -| `notebooks/metric_violation_check.py` | Retraining-trigger task: run the violation SQL over the monitor's profile metrics to decide whether model quality has degraded across recent windows. | +| `notebooks/model_drift_check.py` | Retraining-trigger task: run the drift SQL over the monitor's profile metrics to decide whether model quality has degraded across recent windows. | | `notebooks/feature_drift_check.py` | Retraining-trigger task: threshold the monitor's `_drift_metrics` to decide whether sustained feature (data) drift on any watched column should trigger retraining. | | `notebooks/seed_drift.py` | Demo helper (attendee-runnable): sample already-scored predictions, inflate the amount-driven features, append them to your own `fraud_predictions` with today's timestamp, and refresh your batch monitor so feature drift is visible during the session. | diff --git a/src/monitoring/notebooks/feature_drift_check.py b/src/monitoring/notebooks/feature_drift_check.py index 2d8bf00..1567a7a 100644 --- a/src/monitoring/notebooks/feature_drift_check.py +++ b/src/monitoring/notebooks/feature_drift_check.py @@ -11,7 +11,7 @@ # MAGIC # MAGIC Related notebooks: # MAGIC -# MAGIC - `metric_violation_check.py`: the same pattern for model quality (`..._profile_metrics`). +# MAGIC - `model_drift_check.py`: the same pattern for model quality (`..._profile_metrics`). # MAGIC - `monitoring.py`: the interactive dashboard that displays these metrics. # MAGIC # MAGIC Quality and drift live in separate tables: `..._profile_metrics` holds quality metrics diff --git a/src/monitoring/notebooks/metric_violation_check.py b/src/monitoring/notebooks/model_drift_check.py similarity index 85% rename from src/monitoring/notebooks/metric_violation_check.py rename to src/monitoring/notebooks/model_drift_check.py index 05b6ee9..8cad524 100644 --- a/src/monitoring/notebooks/metric_violation_check.py +++ b/src/monitoring/notebooks/model_drift_check.py @@ -1,12 +1,12 @@ # Databricks notebook source # MAGIC %md -# MAGIC # Monitored-metric violation check (instructor-provided) +# MAGIC # Model drift check (instructor-provided) # MAGIC # MAGIC **Session:** Monitoring & Retraining # MAGIC # MAGIC Automated task in the scheduled `retraining_job`. It reads the monitor's # MAGIC `..._profile_metrics` table, decides whether model quality has degraded across recent -# MAGIC windows, and publishes the boolean `is_metric_violated` as a task value. A downstream +# MAGIC windows, and publishes the boolean `is_model_drift_violated` as a task value. A downstream # MAGIC `condition_task` reads that value and triggers `model_training_job` only when quality # MAGIC has degraded. # MAGIC @@ -19,7 +19,7 @@ # MAGIC # MAGIC - `table_name_under_monitor`: the inference table the monitor profiles. # MAGIC - `metric_to_monitor`: quality metric column in `..._profile_metrics` (e.g. `f1_score`). -# MAGIC - `metric_violation_threshold`: retrain if the metric drops below this. +# MAGIC - `model_drift_threshold`: retrain if the metric drops below this. # MAGIC - `num_evaluation_windows` / `num_violation_windows`: how many recent windows to check, # MAGIC and how many must be in violation before retraining. # MAGIC @@ -32,7 +32,7 @@ ) dbutils.widgets.text("metric_to_monitor", "f1_score.macro", label="Quality metric to monitor") dbutils.widgets.text( - "metric_violation_threshold", "0.20", label="Retrain if the metric drops below this" + "model_drift_threshold", "0.20", label="Retrain if the metric drops below this" ) dbutils.widgets.text("num_evaluation_windows", "5", label="Number of recent windows to check") dbutils.widgets.text("num_violation_windows", "2", label="Windows that must violate to trigger") @@ -51,7 +51,7 @@ # MAGIC window also below it? # MAGIC # MAGIC Fraud quality (e.g. `f1_score.macro`) is "higher is better", so a violation means the -# MAGIC metric dropped **below** `metric_violation_threshold`. (For a drift or error metric +# MAGIC metric dropped **below** `model_drift_threshold`. (For a drift or error metric # MAGIC where "higher is worse", flip `<` to `>`.) Requiring the most recent window to be in # MAGIC violation avoids retraining on a dip that has already recovered. # MAGIC @@ -72,7 +72,7 @@ table_name_under_monitor = dbutils.widgets.get("table_name_under_monitor") metric_to_monitor = dbutils.widgets.get("metric_to_monitor") -metric_violation_threshold = float(dbutils.widgets.get("metric_violation_threshold")) +model_drift_threshold = float(dbutils.widgets.get("model_drift_threshold")) num_evaluation_windows = int(dbutils.widgets.get("num_evaluation_windows")) num_violation_windows = int(dbutils.widgets.get("num_violation_windows")) @@ -82,7 +82,7 @@ # nothing to evaluate, so report "not violated" and the job does not retrain. if not spark.catalog.tableExists(profile_metrics_table): print(f"{profile_metrics_table} does not exist yet; treating as not violated.") - dbutils.jobs.taskValues.set("is_metric_violated", False) + dbutils.jobs.taskValues.set("is_model_drift_violated", False) dbutils.notebook.exit("no_metrics") # COMMAND ---------- @@ -109,10 +109,10 @@ # Violation: at least num_violation_windows of those windows are below the threshold and the # most recent window is also below it (so a recovered dip does not trigger retraining). # Fraud quality is "higher is better"; for a drift/error metric flip < to >. -windows_in_violation = sum(1 for row in recent_metrics if row["metric_value"] < metric_violation_threshold) -latest_in_violation = bool(recent_metrics) and recent_metrics[0]["metric_value"] < metric_violation_threshold -is_metric_violated = windows_in_violation >= num_violation_windows and latest_in_violation -print(f"is_metric_violated = {is_metric_violated}") +windows_in_violation = sum(1 for row in recent_metrics if row["metric_value"] < model_drift_threshold) +latest_in_violation = bool(recent_metrics) and recent_metrics[0]["metric_value"] < model_drift_threshold +is_model_drift_violated = windows_in_violation >= num_violation_windows and latest_in_violation +print(f"is_model_drift_violated = {is_model_drift_violated}") # Publish the decision for the downstream condition_task in the retraining job. -dbutils.jobs.taskValues.set("is_metric_violated", is_metric_violated) +dbutils.jobs.taskValues.set("is_model_drift_violated", is_model_drift_violated) diff --git a/src/monitoring/notebooks/monitoring.py b/src/monitoring/notebooks/monitoring.py index 31f9315..bdcc622 100644 --- a/src/monitoring/notebooks/monitoring.py +++ b/src/monitoring/notebooks/monitoring.py @@ -7,7 +7,7 @@ # MAGIC Interactive dashboard. It refreshes the two Lakehouse Monitors and displays the metric # MAGIC tables they produce; it makes no decisions and is not part of the retraining job. The # MAGIC automated decisions run headless in two job tasks that read the same tables: -# MAGIC `metric_violation_check.py` (model quality) and `feature_drift_check.py` (feature drift). +# MAGIC `model_drift_check.py` (model quality) and `feature_drift_check.py` (feature drift). # MAGIC # MAGIC The bundle creates two monitors: # MAGIC @@ -74,10 +74,14 @@ for table in [batch_table, online_table]: if spark.catalog.tableExists(table) and spark.read.table(table).limit(1).count() > 0: try: + + # -------------------- TODO -------------------- # TODO: refresh the monitor for this table # HINT: w.quality_monitors.run_refresh(table_name=table) starts a refresh and # HINT: returns a run with a run.refresh_id to print. # <-- Your code here + # ---------------------------------------------- + print(f"Refresh started for {table}: refresh_id={run.refresh_id}") except Exception as exc: print(f"(could not refresh {table}: {exc})") diff --git a/src/monitoring/notebooks/process_serving_logs.py b/src/monitoring/notebooks/process_serving_logs.py index deedb3a..490fe3f 100644 --- a/src/monitoring/notebooks/process_serving_logs.py +++ b/src/monitoring/notebooks/process_serving_logs.py @@ -227,10 +227,12 @@ ) ) +# -------------------- TODO -------------------- # TODO: write the online inference table so the online monitor can read it # HINT: online.write.mode("overwrite").option("overwriteSchema", "true").saveAsTable(online_table) # HINT: overwrite (not append): this notebook rebuilds the table from the payloads each run. # <-- Your code here +# ---------------------------------------------- # COMMAND ---------- diff --git a/src/training/notebooks/training.py b/src/training/notebooks/training.py index 84ec3cc..13d6dfa 100644 --- a/src/training/notebooks/training.py +++ b/src/training/notebooks/training.py @@ -68,7 +68,7 @@ # where the experiment/model widgets may be empty. current_user = spark.range(1).select(F.current_user()).first()[0] experiment_name = ( - dbutils.widgets.get("experiment_name") or f"/Users/{current_user}/mlops-workshop-fraud" + dbutils.widgets.get("experiment_name") or f"/Users/{current_user}/databricks-mlops-workshop" ) model_name = dbutils.widgets.get("model_name") # Accept either a bare name (prepend catalog + ml_schema) or an already-qualified 3-level name. @@ -208,6 +208,7 @@ ), ] +# -------------------- TODO -------------------- # TODO: look up the entity features and assemble the training set # HINT: FeatureLookup(table_name=card_feature_table, lookup_key="card_id", # HINT: feature_names=["credit_limit", "num_cards_issued"]); same for the client table @@ -217,6 +218,7 @@ # HINT: exclude the keys and raw function inputs so the model only sees features: # HINT: transaction_id, card_id, client_id, mcc, use_chip. # <-- Your code here +# ---------------------------------------------- # Load into pandas and do a standard stratified train/test split (provided: this is ordinary # scikit-learn, not the MLOps concept this lab is about). @@ -316,6 +318,8 @@ def predict(self, context, model_input): # Package with feature metadata + an inferred signature/input example (required for # safe serving) and register to Unity Catalog in one call. The classifier is wrapped so # the served model returns a fraud probability (score), not a hard class. + + # -------------------- TODO -------------------- # TODO: log and register the model to Unity Catalog with its feature metadata # HINT: fe.log_model packages the model with the training_set's feature lookups (so serving # HINT: reproduces the same features) and registers it to UC in a single call. @@ -323,6 +327,7 @@ def predict(self, context, model_input): # HINT: flavor=mlflow.pyfunc, training_set=training_set, # HINT: registered_model_name=uc_model_name, infer_input_example=True) # <-- Your code here + # ---------------------------------------------- # MLflow 3: surface the eval metrics on the LoggedModel (and thus the UC model-version # page across workspaces), not only on the run. From 0a786525f73b9c0370d6287358f19a898c3bc74d Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Wed, 15 Jul 2026 20:18:19 +0200 Subject: [PATCH 2/4] refactor(bundle): rename ml_workspace key to fraud_ml and auto-derive experiment - The ML schema resource key was ml_workspace while its name is fraud_ml; rename the key to fraud_ml so it matches, and update every \ reference plus the docs/comments in the sections touched. - Stop hard-coding the workshop name in the training notebook's interactive experiment fallback. Derive it from the notebook's own repo folder so it tracks the repo automatically; the DAB job still passes experiment_name explicitly. --- README.md | 14 +++++++------- databricks.yml | 12 ++++++------ resources/monitoring-resource.yml | 12 ++++++------ resources/schemas-resource.yml | 4 ++-- solution/training/notebooks/training.py | 17 ++++++++++++++--- src/training/notebooks/training.py | 17 ++++++++++++++--- 6 files changed, 49 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 501105c..2af605a 100644 --- a/README.md +++ b/README.md @@ -436,14 +436,14 @@ The bundle declares the schema with a **base name** in [resources/schemas-resour ```yaml resources: schemas: - ml_workspace: + fraud_ml: catalog_name: ${var.catalog_name} - name: fraud + name: fraud_ml ``` In **development mode** the bundle automatically prepends `[dev ]` to every -resource name (sanitised for schemas), so `fraud` becomes **`dev__fraud`**, a private -schema per user. In **production mode** there is no prefix, so it stays the shared `fraud`. +resource name (sanitised for schemas), so `fraud_ml` becomes **`dev__fraud_ml`**, a private +schema per user. In **production mode** there is no prefix, so it stays the shared `fraud_ml`. This prefixing is a built-in feature of development mode. Jobs then pass the schema's **resolved** name to the notebooks so they always write to exactly @@ -452,18 +452,18 @@ what the bundle created: ```yaml base_parameters: gold_schema: ${var.gold_schema} # shared read -> fraud_gold - ml_schema: ${resources.schemas.ml_workspace.name} # personal write -> dev__fraud + ml_schema: ${resources.schemas.fraud_ml.name} # personal write -> dev__fraud_ml ``` and the model name follows the same schema: -`${var.catalog_name}.${resources.schemas.ml_workspace.name}.${var.model_name}` +`${var.catalog_name}.${resources.schemas.fraud_ml.name}.${var.model_name}` (where `model_name` is the bare name, default `fraud_detection`). The notebooks prepend the catalog and `ml_schema` to it to build the three-level `uc_model_name`. > **Why reference the resource, not a plain variable?** Development mode prefixes the schema > *resource* but not a hand-written variable, so a variable like `${short_name}_fraud` would > not match the schema the bundle actually creates. Referencing -> `${resources.schemas.ml_workspace.name}` guarantees notebooks and the provisioned schema +> `${resources.schemas.fraud_ml.name}` guarantees notebooks and the provisioned schema > use the identical name. ### Attendee flow diff --git a/databricks.yml b/databricks.yml index 441caf0..84f6cac 100644 --- a/databricks.yml +++ b/databricks.yml @@ -21,9 +21,9 @@ variables: ml_schema: description: >- ML outputs schema (feature tables, model, predictions, monitoring). Resolves to the - ml_workspace schema resource: development mode prefixes the base `fraud` to - `dev__fraud` (per-user); the clean dev/staging/prod targets use the shared `fraud`. - default: ${resources.schemas.ml_workspace.name} + fraud_ml schema resource: development mode prefixes the base `fraud_ml` to + `dev__fraud_ml` (per-user); the clean dev/staging/prod targets use the shared `fraud_ml`. + default: ${resources.schemas.fraud_ml.name} gold_schema: description: Shared, read-only gold schema holding the source consumption tables. default: fraud_gold @@ -81,12 +81,12 @@ include: # landing volume). Declared ONLY on the clean (non-development) targets so that: # * a `bundle deploy -t personal` (development mode) NEVER creates them. Attendees read # the shared, instructor-seeded schemas and write only to their own -# `dev__fraud` (ml_workspace). No empty per-user copies. +# `dev__fraud_ml` (fraud_ml). No empty per-user copies. # * a clean deploy (`-t dev` / `-t staging` / `-t prod`, no name-prefixing) provisions # them literally as the shared `fraud_landing` / `fraud_bronze` / `fraud_silver` / # `fraud_gold` + volume in that target's catalog. -# `fraud_gold` is the shared source that everyone reads; the per-user `ml_workspace` -# schema is separately named `fraud` (ML outputs), so there is no collision on the clean +# `fraud_gold` is the shared source that everyone reads; the per-user `fraud_ml` +# schema is separately named `fraud_ml` (ML outputs), so there is no collision on the clean # targets. # These resources only create the empty schemas/volume; the actual data load # (upload + landing->bronze->silver) still runs via `data_ingestion_job`. diff --git a/resources/monitoring-resource.yml b/resources/monitoring-resource.yml index 1b21acc..30a0028 100644 --- a/resources/monitoring-resource.yml +++ b/resources/monitoring-resource.yml @@ -25,8 +25,8 @@ resources: # Batch path: the weekly fraud report. Labels are joined, so this monitor tracks # model quality (accuracy/precision/recall/f1) as well as drift. batch_monitor: - table_name: ${var.catalog_name}.${resources.schemas.ml_workspace.name}.fraud_predictions - output_schema_name: ${var.catalog_name}.${resources.schemas.ml_workspace.name} + table_name: ${var.catalog_name}.${resources.schemas.fraud_ml.name}.fraud_predictions + output_schema_name: ${var.catalog_name}.${resources.schemas.fraud_ml.name} assets_dir: /Workspace/Users/${workspace.current_user.userName}/lakehouse_monitoring inference_log: granularities: [1 day] @@ -42,8 +42,8 @@ resources: # Online path: ad-hoc checks captured by the serving endpoint's AI Gateway inference # table, unpacked into the same shape. Labels lag, so this leads with drift. online_monitor: - table_name: ${var.catalog_name}.${resources.schemas.ml_workspace.name}.fraud_serving_inference - output_schema_name: ${var.catalog_name}.${resources.schemas.ml_workspace.name} + table_name: ${var.catalog_name}.${resources.schemas.fraud_ml.name}.fraud_serving_inference + output_schema_name: ${var.catalog_name}.${resources.schemas.fraud_ml.name} assets_dir: /Workspace/Users/${workspace.current_user.userName}/lakehouse_monitoring inference_log: granularities: [1 day] @@ -67,7 +67,7 @@ resources: notebook_task: notebook_path: ../${var.code_root}/monitoring/notebooks/model_drift_check.py base_parameters: - table_name_under_monitor: ${var.catalog_name}.${resources.schemas.ml_workspace.name}.fraud_predictions + table_name_under_monitor: ${var.catalog_name}.${resources.schemas.fraud_ml.name}.fraud_predictions metric_to_monitor: f1_score.macro model_drift_threshold: "0.20" num_evaluation_windows: "5" @@ -87,7 +87,7 @@ resources: notebook_task: notebook_path: ../${var.code_root}/monitoring/notebooks/feature_drift_check.py base_parameters: - table_name_under_monitor: ${var.catalog_name}.${resources.schemas.ml_workspace.name}.fraud_predictions + table_name_under_monitor: ${var.catalog_name}.${resources.schemas.fraud_ml.name}.fraud_predictions features_to_monitor: amount,credit_score,credit_limit,yearly_income,current_age,num_cards_issued,is_night,is_online,is_high_risk_mcc,amount_to_income_ratio,amount_to_credit_limit_ratio drift_metric: population_stability_index drift_violation_threshold: "0.20" diff --git a/resources/schemas-resource.yml b/resources/schemas-resource.yml index 7d207c2..0bb96a0 100644 --- a/resources/schemas-resource.yml +++ b/resources/schemas-resource.yml @@ -3,13 +3,13 @@ # for the ML artifacts they produce (feature tables, model, predictions, monitoring). # In production mode (no prefix) it resolves to the shared `fraud_ml`. Reads always come # from the shared medallion schemas (fraud_bronze / fraud_silver / fraud_gold). Jobs -# reference ${resources.schemas.ml_workspace.name} so notebooks write to exactly this +# reference ${resources.schemas.fraud_ml.name} so notebooks write to exactly this # schema. The layer ("gold") is intentionally NOT in the name: this schema holds ML # outputs (model, predictions, monitoring), not a mirror of the shared gold layer. # Docs: Bundle deployment modes (dev prefixing) https://learn.microsoft.com/azure/databricks/dev-tools/bundles/deployment-modes resources: schemas: - ml_workspace: + fraud_ml: catalog_name: ${var.catalog_name} name: fraud_ml comment: Per-user ML outputs (feature tables, model, predictions, monitoring). Shared in prod. diff --git a/solution/training/notebooks/training.py b/solution/training/notebooks/training.py index 798d242..e342961 100644 --- a/solution/training/notebooks/training.py +++ b/solution/training/notebooks/training.py @@ -67,9 +67,20 @@ # Sensible defaults so the notebook is runnable interactively (outside the DAB job), # where the experiment/model widgets may be empty. current_user = spark.range(1).select(F.current_user()).first()[0] -experiment_name = ( - dbutils.widgets.get("experiment_name") or f"/Users/{current_user}/databricks-mlops-workshop" -) +# Interactive fallback for the experiment: derive it from this notebook's repo folder so the +# name tracks the repo automatically instead of hard-coding it. In the DAB job the bundle +# always passes experiment_name (${resources.experiments.experiment.name}). +experiment_name = dbutils.widgets.get("experiment_name") +if not experiment_name: + from pathlib import PurePosixPath + + _notebook_path = ( + dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get() + ) + # //training/notebooks/training.py -> the folder name + _parents = PurePosixPath(_notebook_path).parents + _project = _parents[3].name if len(_parents) > 3 else "mlops-workshop" + experiment_name = f"/Users/{current_user}/{_project}" model_name = dbutils.widgets.get("model_name") # Accept either a bare name (prepend catalog + ml_schema) or an already-qualified 3-level name. uc_model_name = ( diff --git a/src/training/notebooks/training.py b/src/training/notebooks/training.py index 13d6dfa..82f8722 100644 --- a/src/training/notebooks/training.py +++ b/src/training/notebooks/training.py @@ -67,9 +67,20 @@ # Sensible defaults so the notebook is runnable interactively (outside the DAB job), # where the experiment/model widgets may be empty. current_user = spark.range(1).select(F.current_user()).first()[0] -experiment_name = ( - dbutils.widgets.get("experiment_name") or f"/Users/{current_user}/databricks-mlops-workshop" -) +# Interactive fallback for the experiment: derive it from this notebook's repo folder so the +# name tracks the repo automatically instead of hard-coding it. In the DAB job the bundle +# always passes experiment_name (${resources.experiments.experiment.name}). +experiment_name = dbutils.widgets.get("experiment_name") +if not experiment_name: + from pathlib import PurePosixPath + + _notebook_path = ( + dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get() + ) + # //training/notebooks/training.py -> the folder name + _parents = PurePosixPath(_notebook_path).parents + _project = _parents[3].name if len(_parents) > 3 else "mlops-workshop" + experiment_name = f"/Users/{current_user}/{_project}" model_name = dbutils.widgets.get("model_name") # Accept either a bare name (prepend catalog + ml_schema) or an already-qualified 3-level name. uc_model_name = ( From 7018a9a9ab62cece0de141294d244413da88305c Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Wed, 15 Jul 2026 20:00:01 +0100 Subject: [PATCH 3/4] fix(bundle): make the personal ML schema a general dev__fraud sandbox Shared/clean targets keep fraud_ml (alongside fraud_landing/bronze/silver/gold). The personal target overrides just the schema name to fraud, so development-mode prefixing gives each attendee a dev__fraud sandbox (not ML-specific). Drops the _ml suffix from every notebook's interactive per-user fallback and updates the docs. --- README.md | 14 +++++++----- databricks.yml | 22 +++++++++++++------ resources/README.md | 2 +- resources/schemas-resource.yml | 17 +++++++------- .../notebooks/batch_inference.py | 2 +- .../model_deployment/notebooks/approval.py | 2 +- .../model_deployment/notebooks/deployment.py | 2 +- .../model_deployment/notebooks/evaluation.py | 2 +- .../notebooks/query_endpoint.py | 2 +- .../notebooks/build_features.py | 2 +- solution/monitoring/notebooks/monitoring.py | 2 +- .../notebooks/process_serving_logs.py | 2 +- solution/monitoring/notebooks/seed_drift.py | 2 +- solution/training/notebooks/training.py | 2 +- .../notebooks/batch_inference.py | 2 +- .../model_deployment/notebooks/approval.py | 2 +- .../model_deployment/notebooks/deployment.py | 2 +- .../model_deployment/notebooks/evaluation.py | 2 +- .../notebooks/query_endpoint.py | 2 +- .../notebooks/build_features.py | 2 +- src/monitoring/notebooks/monitoring.py | 2 +- .../notebooks/process_serving_logs.py | 2 +- src/monitoring/notebooks/seed_drift.py | 2 +- src/training/notebooks/training.py | 2 +- 24 files changed, 52 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 2af605a..e3de23a 100644 --- a/README.md +++ b/README.md @@ -438,13 +438,15 @@ resources: schemas: fraud_ml: catalog_name: ${var.catalog_name} - name: fraud_ml + name: fraud_ml # personal target overrides this to `fraud` ``` -In **development mode** the bundle automatically prepends `[dev ]` to every -resource name (sanitised for schemas), so `fraud_ml` becomes **`dev__fraud_ml`**, a private -schema per user. In **production mode** there is no prefix, so it stays the shared `fraud_ml`. -This prefixing is a built-in feature of development mode. +On the clean **dev/staging/prod** targets the schema is the shared **`fraud_ml`**, sitting +alongside `fraud_landing` / `fraud_bronze` / `fraud_silver` / `fraud_gold`. The **personal** +target overrides the schema name to `fraud`, and development mode prepends +`[dev ]` (sanitised for schemas), so each attendee gets a private +**`dev__fraud`** sandbox they can use for anything. This prefixing is a built-in feature +of development mode. Jobs then pass the schema's **resolved** name to the notebooks so they always write to exactly what the bundle created: @@ -452,7 +454,7 @@ what the bundle created: ```yaml base_parameters: gold_schema: ${var.gold_schema} # shared read -> fraud_gold - ml_schema: ${resources.schemas.fraud_ml.name} # personal write -> dev__fraud_ml + ml_schema: ${resources.schemas.fraud_ml.name} # personal write -> dev__fraud ``` and the model name follows the same schema: diff --git a/databricks.yml b/databricks.yml index 84f6cac..2c932ba 100644 --- a/databricks.yml +++ b/databricks.yml @@ -20,9 +20,9 @@ variables: default: dev ml_schema: description: >- - ML outputs schema (feature tables, model, predictions, monitoring). Resolves to the - fraud_ml schema resource: development mode prefixes the base `fraud_ml` to - `dev__fraud_ml` (per-user); the clean dev/staging/prod targets use the shared `fraud_ml`. + Resolved name of the ML schema passed to notebooks. The clean dev/staging/prod targets + use the shared `fraud_ml`; the personal target overrides the schema name to `fraud`, so + development mode gives each attendee a `dev__fraud` sandbox. default: ${resources.schemas.fraud_ml.name} gold_schema: description: Shared, read-only gold schema holding the source consumption tables. @@ -81,13 +81,14 @@ include: # landing volume). Declared ONLY on the clean (non-development) targets so that: # * a `bundle deploy -t personal` (development mode) NEVER creates them. Attendees read # the shared, instructor-seeded schemas and write only to their own -# `dev__fraud_ml` (fraud_ml). No empty per-user copies. +# `dev__fraud` sandbox. No empty per-user copies. # * a clean deploy (`-t dev` / `-t staging` / `-t prod`, no name-prefixing) provisions # them literally as the shared `fraud_landing` / `fraud_bronze` / `fraud_silver` / # `fraud_gold` + volume in that target's catalog. -# `fraud_gold` is the shared source that everyone reads; the per-user `fraud_ml` -# schema is separately named `fraud_ml` (ML outputs), so there is no collision on the clean -# targets. +# `fraud_gold` is the shared source that everyone reads; the shared ML schema is separately +# named `fraud_ml` (ML outputs). On the personal target each attendee instead gets a general +# `dev__fraud` sandbox (base name overridden to `fraud`), so there is no collision on +# the clean targets. # These resources only create the empty schemas/volume; the actual data load # (upload + landing->bronze->silver) still runs via `data_ingestion_job`. @@ -105,6 +106,13 @@ targets: default: true variables: catalog_name: dev + # The shared ML schema is `fraud_ml`; on personal each attendee gets a general + # `dev__fraud` sandbox instead, so override just the schema name to `fraud` + # (development mode then prefixes it to dev__fraud). + resources: + schemas: + fraud_ml: + name: fraud # workspace: # host: https://adb-xxxxxxxxxxxxxxxx.x.azuredatabricks.net # add your workspace URL if deploying from a local CLI diff --git a/resources/README.md b/resources/README.md index 61a1ca2..ffdd9d0 100644 --- a/resources/README.md +++ b/resources/README.md @@ -6,7 +6,7 @@ reconciles them into the workspace. | File | Description | |---|---| -| `schemas-resource.yml` | Per-user ML workspace schema (base name `fraud`; dev mode → `dev__fraud`). | +| `schemas-resource.yml` | The ML schema: shared `fraud_ml` on clean targets; a per-attendee `dev__fraud` sandbox on personal. | | `ml-artifacts-resource.yml` | The MLflow experiment for the fraud project. | | `data-ingestion-workflow.yml` | Instructor-only job: land raw files into Bronze, then build Silver. | | `feature-engineering-workflow.yml` | Build the card/client entity feature tables and on-demand feature functions. | diff --git a/resources/schemas-resource.yml b/resources/schemas-resource.yml index 0bb96a0..72e6ef3 100644 --- a/resources/schemas-resource.yml +++ b/resources/schemas-resource.yml @@ -1,15 +1,14 @@ -# Per-user ML workspace schema. Declared with a BASE name (`fraud_ml`); in development -# mode the bundle prefixes it to `dev__fraud_ml`, giving each user their own schema -# for the ML artifacts they produce (feature tables, model, predictions, monitoring). -# In production mode (no prefix) it resolves to the shared `fraud_ml`. Reads always come -# from the shared medallion schemas (fraud_bronze / fraud_silver / fraud_gold). Jobs -# reference ${resources.schemas.fraud_ml.name} so notebooks write to exactly this -# schema. The layer ("gold") is intentionally NOT in the name: this schema holds ML -# outputs (model, predictions, monitoring), not a mirror of the shared gold layer. +# The ML schema. On the clean dev/staging/prod targets it is the shared `fraud_ml`, sitting +# alongside the medallion schemas (fraud_landing / fraud_bronze / fraud_silver / fraud_gold). +# The personal target overrides the name to `fraud` (see databricks.yml), and development mode +# prefixes it, so each attendee gets a private `dev__fraud` sandbox they can use for +# anything, not just ML outputs. Reads always come from the shared medallion schemas; jobs +# pass ${resources.schemas.fraud_ml.name} so notebooks write to exactly the schema the bundle +# created for that target. # Docs: Bundle deployment modes (dev prefixing) https://learn.microsoft.com/azure/databricks/dev-tools/bundles/deployment-modes resources: schemas: fraud_ml: catalog_name: ${var.catalog_name} name: fraud_ml - comment: Per-user ML outputs (feature tables, model, predictions, monitoring). Shared in prod. + comment: Shared ML outputs (fraud_ml) on clean targets; a general per-attendee sandbox (dev__fraud) on personal. diff --git a/solution/deployment/batch_inference/notebooks/batch_inference.py b/solution/deployment/batch_inference/notebooks/batch_inference.py index 2dd3b27..d76ae01 100644 --- a/solution/deployment/batch_inference/notebooks/batch_inference.py +++ b/solution/deployment/batch_inference/notebooks/batch_inference.py @@ -56,7 +56,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" model_alias = dbutils.widgets.get("model_alias") model_name = dbutils.widgets.get("model_name") diff --git a/solution/deployment/model_deployment/notebooks/approval.py b/solution/deployment/model_deployment/notebooks/approval.py index 09114b0..7ec247b 100644 --- a/solution/deployment/model_deployment/notebooks/approval.py +++ b/solution/deployment/model_deployment/notebooks/approval.py @@ -62,7 +62,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" if not uc_model_name: uc_model_name = f"{catalog_name}.{ml_schema}.fraud_detection" if not model_version: diff --git a/solution/deployment/model_deployment/notebooks/deployment.py b/solution/deployment/model_deployment/notebooks/deployment.py index cf35597..6f599c2 100644 --- a/solution/deployment/model_deployment/notebooks/deployment.py +++ b/solution/deployment/model_deployment/notebooks/deployment.py @@ -57,7 +57,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" if not uc_model_name: uc_model_name = f"{catalog_name}.{ml_schema}.fraud_detection" if not model_version: diff --git a/solution/deployment/model_deployment/notebooks/evaluation.py b/solution/deployment/model_deployment/notebooks/evaluation.py index d07f40a..db20f06 100644 --- a/solution/deployment/model_deployment/notebooks/evaluation.py +++ b/solution/deployment/model_deployment/notebooks/evaluation.py @@ -63,7 +63,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" if not uc_model_name: uc_model_name = f"{catalog_name}.{ml_schema}.fraud_detection" if not model_version: diff --git a/solution/deployment/model_deployment/notebooks/query_endpoint.py b/solution/deployment/model_deployment/notebooks/query_endpoint.py index a2b55fd..9fc54e6 100644 --- a/solution/deployment/model_deployment/notebooks/query_endpoint.py +++ b/solution/deployment/model_deployment/notebooks/query_endpoint.py @@ -35,7 +35,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" # One endpoint per user, named after the personal schema (matches the deployment job). endpoint_name = f"{ml_schema}_fraud" diff --git a/solution/feature_engineering/notebooks/build_features.py b/solution/feature_engineering/notebooks/build_features.py index 22577d8..4f97545 100644 --- a/solution/feature_engineering/notebooks/build_features.py +++ b/solution/feature_engineering/notebooks/build_features.py @@ -54,7 +54,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" # Read from the shared gold source; write to the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" diff --git a/solution/monitoring/notebooks/monitoring.py b/solution/monitoring/notebooks/monitoring.py index 7d6f89d..af329d1 100644 --- a/solution/monitoring/notebooks/monitoring.py +++ b/solution/monitoring/notebooks/monitoring.py @@ -45,7 +45,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" batch_table = f"{catalog_name}.{ml_schema}.fraud_predictions" online_table = f"{catalog_name}.{ml_schema}.fraud_serving_inference" diff --git a/solution/monitoring/notebooks/process_serving_logs.py b/solution/monitoring/notebooks/process_serving_logs.py index 6a74f08..4391f0f 100644 --- a/solution/monitoring/notebooks/process_serving_logs.py +++ b/solution/monitoring/notebooks/process_serving_logs.py @@ -46,7 +46,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" payload_table = f"{catalog_name}.{ml_schema}.fraud_serving_payload" online_table = f"{catalog_name}.{ml_schema}.fraud_serving_inference" diff --git a/solution/monitoring/notebooks/seed_drift.py b/solution/monitoring/notebooks/seed_drift.py index 142c2f7..a7c62fe 100644 --- a/solution/monitoring/notebooks/seed_drift.py +++ b/solution/monitoring/notebooks/seed_drift.py @@ -47,7 +47,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" amount_multiplier = float(dbutils.widgets.get("amount_multiplier")) num_rows = int(dbutils.widgets.get("num_rows")) diff --git a/solution/training/notebooks/training.py b/solution/training/notebooks/training.py index e342961..ebd0967 100644 --- a/solution/training/notebooks/training.py +++ b/solution/training/notebooks/training.py @@ -57,7 +57,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" # Labels come from the shared gold table; features come from the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" diff --git a/src/deployment/batch_inference/notebooks/batch_inference.py b/src/deployment/batch_inference/notebooks/batch_inference.py index 6e6e6fe..036acf8 100644 --- a/src/deployment/batch_inference/notebooks/batch_inference.py +++ b/src/deployment/batch_inference/notebooks/batch_inference.py @@ -56,7 +56,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" model_alias = dbutils.widgets.get("model_alias") model_name = dbutils.widgets.get("model_name") diff --git a/src/deployment/model_deployment/notebooks/approval.py b/src/deployment/model_deployment/notebooks/approval.py index 09114b0..7ec247b 100644 --- a/src/deployment/model_deployment/notebooks/approval.py +++ b/src/deployment/model_deployment/notebooks/approval.py @@ -62,7 +62,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" if not uc_model_name: uc_model_name = f"{catalog_name}.{ml_schema}.fraud_detection" if not model_version: diff --git a/src/deployment/model_deployment/notebooks/deployment.py b/src/deployment/model_deployment/notebooks/deployment.py index cf35597..6f599c2 100644 --- a/src/deployment/model_deployment/notebooks/deployment.py +++ b/src/deployment/model_deployment/notebooks/deployment.py @@ -57,7 +57,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" if not uc_model_name: uc_model_name = f"{catalog_name}.{ml_schema}.fraud_detection" if not model_version: diff --git a/src/deployment/model_deployment/notebooks/evaluation.py b/src/deployment/model_deployment/notebooks/evaluation.py index d07f40a..db20f06 100644 --- a/src/deployment/model_deployment/notebooks/evaluation.py +++ b/src/deployment/model_deployment/notebooks/evaluation.py @@ -63,7 +63,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" if not uc_model_name: uc_model_name = f"{catalog_name}.{ml_schema}.fraud_detection" if not model_version: diff --git a/src/deployment/model_deployment/notebooks/query_endpoint.py b/src/deployment/model_deployment/notebooks/query_endpoint.py index a2b55fd..9fc54e6 100644 --- a/src/deployment/model_deployment/notebooks/query_endpoint.py +++ b/src/deployment/model_deployment/notebooks/query_endpoint.py @@ -35,7 +35,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" # One endpoint per user, named after the personal schema (matches the deployment job). endpoint_name = f"{ml_schema}_fraud" diff --git a/src/feature_engineering/notebooks/build_features.py b/src/feature_engineering/notebooks/build_features.py index cc729cd..4d05ede 100644 --- a/src/feature_engineering/notebooks/build_features.py +++ b/src/feature_engineering/notebooks/build_features.py @@ -54,7 +54,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" # Read from the shared gold source; write to the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" diff --git a/src/monitoring/notebooks/monitoring.py b/src/monitoring/notebooks/monitoring.py index bdcc622..57df3c9 100644 --- a/src/monitoring/notebooks/monitoring.py +++ b/src/monitoring/notebooks/monitoring.py @@ -45,7 +45,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" batch_table = f"{catalog_name}.{ml_schema}.fraud_predictions" online_table = f"{catalog_name}.{ml_schema}.fraud_serving_inference" diff --git a/src/monitoring/notebooks/process_serving_logs.py b/src/monitoring/notebooks/process_serving_logs.py index 490fe3f..36a3092 100644 --- a/src/monitoring/notebooks/process_serving_logs.py +++ b/src/monitoring/notebooks/process_serving_logs.py @@ -46,7 +46,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" payload_table = f"{catalog_name}.{ml_schema}.fraud_serving_payload" online_table = f"{catalog_name}.{ml_schema}.fraud_serving_inference" diff --git a/src/monitoring/notebooks/seed_drift.py b/src/monitoring/notebooks/seed_drift.py index 142c2f7..a7c62fe 100644 --- a/src/monitoring/notebooks/seed_drift.py +++ b/src/monitoring/notebooks/seed_drift.py @@ -47,7 +47,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" amount_multiplier = float(dbutils.widgets.get("amount_multiplier")) num_rows = int(dbutils.widgets.get("num_rows")) diff --git a/src/training/notebooks/training.py b/src/training/notebooks/training.py index 82f8722..1345204 100644 --- a/src/training/notebooks/training.py +++ b/src/training/notebooks/training.py @@ -57,7 +57,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud_ml" + ml_schema = f"dev_{_short}_fraud" # Labels come from the shared gold table; features come from the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" From 75c1fa1391f8f7070fd4fc03898a0d1e0bbc4981 Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Thu, 16 Jul 2026 09:53:19 +0100 Subject: [PATCH 4/4] fix(bundle): align staging root_path with repo and simplify experiment fallback Address PR review: staging.workspace.root_path still used /Workspace/mlops-workshop while prod used /Workspace/databricks-mlops-workshop; point both at databricks-mlops-workshop. Replace the fragile parents[3] notebook-path parsing (which broke on bundle deployment paths and defaulted to the old mlops-workshop name) with a simple experiment fallback under the user's home. --- databricks.yml | 2 +- solution/training/notebooks/training.py | 21 +++++++-------------- src/training/notebooks/training.py | 21 +++++++-------------- 3 files changed, 15 insertions(+), 29 deletions(-) diff --git a/databricks.yml b/databricks.yml index 2c932ba..c6964a4 100644 --- a/databricks.yml +++ b/databricks.yml @@ -162,7 +162,7 @@ targets: # Production mode needs a fixed, non-user-scoped root_path so exactly one copy is # deployed and it is not tied to the deploying user. CI deploys and runs this target as # the cicd_service_principal. - root_path: /Workspace/mlops-workshop/.bundle/${bundle.name}/${bundle.target} + root_path: /Workspace/databricks-mlops-workshop/.bundle/${bundle.name}/${bundle.target} prod: mode: production diff --git a/solution/training/notebooks/training.py b/solution/training/notebooks/training.py index ebd0967..9ada6f3 100644 --- a/solution/training/notebooks/training.py +++ b/solution/training/notebooks/training.py @@ -67,20 +67,13 @@ # Sensible defaults so the notebook is runnable interactively (outside the DAB job), # where the experiment/model widgets may be empty. current_user = spark.range(1).select(F.current_user()).first()[0] -# Interactive fallback for the experiment: derive it from this notebook's repo folder so the -# name tracks the repo automatically instead of hard-coding it. In the DAB job the bundle -# always passes experiment_name (${resources.experiments.experiment.name}). -experiment_name = dbutils.widgets.get("experiment_name") -if not experiment_name: - from pathlib import PurePosixPath - - _notebook_path = ( - dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get() - ) - # //training/notebooks/training.py -> the folder name - _parents = PurePosixPath(_notebook_path).parents - _project = _parents[3].name if len(_parents) > 3 else "mlops-workshop" - experiment_name = f"/Users/{current_user}/{_project}" +# Interactive fallback: outside the DAB job the experiment widget is empty, so log under the +# user's home in an experiment named after the repo. In the job the bundle always passes +# experiment_name (${resources.experiments.experiment.name}), so this only applies to ad-hoc runs. +experiment_name = ( + dbutils.widgets.get("experiment_name") + or f"/Users/{current_user}/databricks-mlops-workshop" +) model_name = dbutils.widgets.get("model_name") # Accept either a bare name (prepend catalog + ml_schema) or an already-qualified 3-level name. uc_model_name = ( diff --git a/src/training/notebooks/training.py b/src/training/notebooks/training.py index 1345204..81cf075 100644 --- a/src/training/notebooks/training.py +++ b/src/training/notebooks/training.py @@ -67,20 +67,13 @@ # Sensible defaults so the notebook is runnable interactively (outside the DAB job), # where the experiment/model widgets may be empty. current_user = spark.range(1).select(F.current_user()).first()[0] -# Interactive fallback for the experiment: derive it from this notebook's repo folder so the -# name tracks the repo automatically instead of hard-coding it. In the DAB job the bundle -# always passes experiment_name (${resources.experiments.experiment.name}). -experiment_name = dbutils.widgets.get("experiment_name") -if not experiment_name: - from pathlib import PurePosixPath - - _notebook_path = ( - dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get() - ) - # //training/notebooks/training.py -> the folder name - _parents = PurePosixPath(_notebook_path).parents - _project = _parents[3].name if len(_parents) > 3 else "mlops-workshop" - experiment_name = f"/Users/{current_user}/{_project}" +# Interactive fallback: outside the DAB job the experiment widget is empty, so log under the +# user's home in an experiment named after the repo. In the job the bundle always passes +# experiment_name (${resources.experiments.experiment.name}), so this only applies to ad-hoc runs. +experiment_name = ( + dbutils.widgets.get("experiment_name") + or f"/Users/{current_user}/databricks-mlops-workshop" +) model_name = dbutils.widgets.get("model_name") # Accept either a bare name (prepend catalog + ml_schema) or an already-qualified 3-level name. uc_model_name = (