From 824ad77a8ec8c21d15397fb3885cb7de1a09a968 Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Mon, 13 Jul 2026 10:53:29 +0100 Subject: [PATCH 1/7] docs(notebooks): professionalise markdown, short and sharp with bullets Rewrite the notebook markdown across data ingestion, exploration, feature engineering, training, deployment, and monitoring into a neutral professional voice. Remove conversational headings and framing (for example 'the imbalance trap', 'but look at precision', 'try it', 'nobody watching', 'for a person to read'), tighten prose, and use bullet points for enumerable content. Code cells are unchanged; src regenerated. --- .../notebooks/load_bronze_to_silver.py | 18 ++-- .../notebooks/load_landing_to_bronze.py | 19 ++-- .../notebooks/batch_inference.py | 18 ++-- .../model_deployment/notebooks/approval.py | 2 +- .../model_deployment/notebooks/evaluation.py | 21 ++-- .../notebooks/query_endpoint.py | 20 ++-- solution/exploration/notebooks/eda.py | 14 +-- .../notebooks/build_features.py | 31 ++++-- .../notebooks/feature_drift_check.py | 32 +++--- .../notebooks/metric_violation_check.py | 24 ++--- solution/monitoring/notebooks/monitoring.py | 76 +++++++------ .../notebooks/process_serving_logs.py | 25 ++--- solution/monitoring/notebooks/seed_drift.py | 24 ++--- solution/training/notebooks/training.py | 102 ++++++++++-------- .../notebooks/load_bronze_to_silver.py | 18 ++-- .../notebooks/load_landing_to_bronze.py | 19 ++-- .../notebooks/batch_inference.py | 18 ++-- .../model_deployment/notebooks/approval.py | 2 +- .../model_deployment/notebooks/evaluation.py | 21 ++-- .../notebooks/query_endpoint.py | 20 ++-- src/exploration/notebooks/eda.py | 14 +-- .../notebooks/build_features.py | 31 ++++-- .../notebooks/feature_drift_check.py | 32 +++--- .../notebooks/metric_violation_check.py | 24 ++--- src/monitoring/notebooks/monitoring.py | 76 +++++++------ .../notebooks/process_serving_logs.py | 25 ++--- src/monitoring/notebooks/seed_drift.py | 24 ++--- src/training/notebooks/training.py | 102 ++++++++++-------- 28 files changed, 468 insertions(+), 384 deletions(-) diff --git a/solution/data_ingestion/notebooks/load_bronze_to_silver.py b/solution/data_ingestion/notebooks/load_bronze_to_silver.py index 6b667cb..78f0bdd 100644 --- a/solution/data_ingestion/notebooks/load_bronze_to_silver.py +++ b/solution/data_ingestion/notebooks/load_bronze_to_silver.py @@ -1,10 +1,15 @@ # Databricks notebook source # MAGIC %md # MAGIC # Bronze -> Silver -# MAGIC Clean, type, dedupe and join the Bronze tables into conformed Silver tables, -# MAGIC including the enriched transaction table used for feature engineering. # MAGIC -# MAGIC **Setup notebook** run by the instructor before the workshop. Not a lab exercise. +# MAGIC Instructor setup notebook, run before the workshop. Not a lab exercise. +# MAGIC +# MAGIC Turns the Bronze tables into conformed Silver tables (and the enriched Gold +# MAGIC transaction table that feature engineering reads) by: +# MAGIC +# MAGIC - cleaning and casting columns to their real types; +# MAGIC - deduplicating to one row per entity; +# MAGIC - joining the sources into a single labelled transaction table. # MAGIC # MAGIC | Layer | Location | # MAGIC |---|---| @@ -59,9 +64,10 @@ # MAGIC %md # MAGIC ## Target schemas -# MAGIC The shared `fraud_silver` / `fraud_gold` schemas are declared as bundle resources -# MAGIC (the `shared_infra` block in `databricks.yml`) and created by -# MAGIC `databricks bundle deploy -t dev` before this job runs. +# MAGIC +# MAGIC - The `fraud_silver` / `fraud_gold` schemas are declared as bundle resources (the +# MAGIC `shared_infra` block in `databricks.yml`). +# MAGIC - `databricks bundle deploy -t dev` creates them before this job runs. # COMMAND ---------- diff --git a/solution/data_ingestion/notebooks/load_landing_to_bronze.py b/solution/data_ingestion/notebooks/load_landing_to_bronze.py index 5ea278a..8e4adf2 100644 --- a/solution/data_ingestion/notebooks/load_landing_to_bronze.py +++ b/solution/data_ingestion/notebooks/load_landing_to_bronze.py @@ -1,10 +1,14 @@ # Databricks notebook source # MAGIC %md # MAGIC # Landing -> Bronze -# MAGIC Ingest the raw fraud source files from the landing volume into Bronze Delta -# MAGIC tables, applying explicit schemas and audit columns. Minimal transformation. # MAGIC -# MAGIC **Setup notebook** run by the instructor before the workshop. Not a lab exercise. +# MAGIC Instructor setup notebook, run before the workshop. Not a lab exercise. +# MAGIC +# MAGIC Reads the raw fraud source files from the landing volume into Bronze Delta tables: +# MAGIC +# MAGIC - explicit schemas, no inference; +# MAGIC - audit columns (ingest timestamp, ingest date, source file); +# MAGIC - minimal transformation, so the shape stays close to the source. # MAGIC # MAGIC | Layer | Location | # MAGIC |---|---| @@ -59,10 +63,11 @@ # MAGIC %md # MAGIC ## Target schemas and volume -# MAGIC The shared `fraud_landing` / `fraud_bronze` schemas and the `raw_data` volume are -# MAGIC declared as bundle resources (the `shared_infra` block in `databricks.yml`) and -# MAGIC created by `databricks bundle deploy -t dev` before this job runs. The raw files are -# MAGIC uploaded into the volume after the deploy, and this notebook just reads them. +# MAGIC +# MAGIC - The `fraud_landing` / `fraud_bronze` schemas and the `raw_data` volume are declared as +# MAGIC bundle resources (the `shared_infra` block in `databricks.yml`). +# MAGIC - `databricks bundle deploy -t dev` creates them before this job runs. +# MAGIC - The raw files are uploaded into the volume after the deploy; this notebook only reads them. # COMMAND ---------- diff --git a/solution/deployment/batch_inference/notebooks/batch_inference.py b/solution/deployment/batch_inference/notebooks/batch_inference.py index d214d31..1f57798 100644 --- a/solution/deployment/batch_inference/notebooks/batch_inference.py +++ b/solution/deployment/batch_inference/notebooks/batch_inference.py @@ -1,21 +1,23 @@ # Databricks notebook source # MAGIC %md -# MAGIC # Batch inference: the weekly fraud report +# MAGIC # Batch inference # MAGIC # MAGIC **Session:** Model Serving & Consumption / Monitoring & Retraining # MAGIC # MAGIC Score a batch of transactions with the governed `@champion` model and append the -# MAGIC results to an inference table. This is the batch prediction path (a scheduled "weekly -# MAGIC report"), and the table it writes is what the batch monitor profiles in the -# MAGIC Monitoring session. +# MAGIC results to an inference table. This is the scheduled batch prediction path, and the +# MAGIC table it writes is what the batch monitor profiles in the Monitoring session. # MAGIC # MAGIC - **Model (personal):** `dev._fraud.fraud_detection@champion` # MAGIC - **Inference table (personal):** `dev._fraud.fraud_predictions` # MAGIC -# MAGIC To be usable by Lakehouse Monitoring's InferenceLog analysis, every row needs four -# MAGIC things beyond the features: a prediction, the model version that produced it, a -# MAGIC timestamp, and (when it arrives) the ground-truth label. We write all four so the -# MAGIC monitor can track both drift and quality over time. +# MAGIC Lakehouse Monitoring's InferenceLog analysis needs four columns beyond the features on +# MAGIC every row, all written here so the monitor can track drift and quality: +# MAGIC +# MAGIC - the prediction; +# MAGIC - the model version that produced it; +# MAGIC - a scoring timestamp; +# MAGIC - the ground-truth label, when available. # MAGIC # MAGIC Docs: [Batch inference with Feature Engineering](https://learn.microsoft.com/azure/databricks/machine-learning/feature-store/score-batch) diff --git a/solution/deployment/model_deployment/notebooks/approval.py b/solution/deployment/model_deployment/notebooks/approval.py index 6e32002..c99dc17 100644 --- a/solution/deployment/model_deployment/notebooks/approval.py +++ b/solution/deployment/model_deployment/notebooks/approval.py @@ -9,7 +9,7 @@ # MAGIC as the signal (the mechanism MLflow 3 deployment jobs use): # MAGIC # MAGIC - **dev** (`auto_approve=true`): the task sets the approval tag itself and passes, so -# MAGIC the whole loop runs unattended in class. +# MAGIC the whole loop runs unattended. # MAGIC - **staging / prod** (`auto_approve=false`): the task fails until a human reviews the # MAGIC evaluation metrics on the model-version page and clicks **Approve** (which sets the # MAGIC tag `=Approved` and repairs the run). The task then passes and the run diff --git a/solution/deployment/model_deployment/notebooks/evaluation.py b/solution/deployment/model_deployment/notebooks/evaluation.py index 79fba5a..d245921 100644 --- a/solution/deployment/model_deployment/notebooks/evaluation.py +++ b/solution/deployment/model_deployment/notebooks/evaluation.py @@ -15,7 +15,7 @@ # MAGIC 3. **Deployment**: promote the version to `@champion` and serve it. # MAGIC # MAGIC The version being evaluated is the one that triggered the job, so there is no -# MAGIC champion/challenger bookkeeping here. We evaluate this version and gate it against a +# MAGIC champion/challenger bookkeeping here. The version is evaluated and gated against a # MAGIC metric floor (and the current champion, if any). # MAGIC See [MLflow deployment jobs](https://learn.microsoft.com/azure/databricks/mlflow/deployment-job). @@ -106,9 +106,10 @@ def score_holdout(version): # MAGIC %md # MAGIC ### Confusion matrix for the approver +# MAGIC # MAGIC Threshold the candidate's fraud score at 0.5 and show the confusion matrix on the -# MAGIC holdout, logged to an MLflow run so it sits alongside the metrics an approver reviews -# MAGIC before clicking Approve. +# MAGIC holdout, logged to an MLflow run so it sits alongside the metrics the approver reviews +# MAGIC before approving. # COMMAND ---------- @@ -134,11 +135,15 @@ def score_holdout(version): # MAGIC %md # MAGIC ### ROC and precision-recall curves -# MAGIC The confusion matrix is a single operating point; these curves show the whole -# MAGIC threshold trade-off. The **ROC** curve's area is the gate metric (threshold-independent), -# MAGIC and on rare-fraud data the **precision-recall** curve (with its average precision) is the -# MAGIC more honest read of how much precision you give up as you chase recall. Both are logged -# MAGIC to the same evaluation run, alongside the confusion matrix. +# MAGIC +# MAGIC The confusion matrix is a single operating point; these curves show the full threshold +# MAGIC trade-off: +# MAGIC +# MAGIC - the ROC curve's area is the gate metric (threshold-independent); +# MAGIC - on rare-fraud data, the precision-recall curve (and its average precision) shows how +# MAGIC much precision is given up as recall increases. +# MAGIC +# MAGIC Both are logged to the same evaluation run, alongside the confusion matrix. # COMMAND ---------- diff --git a/solution/deployment/model_deployment/notebooks/query_endpoint.py b/solution/deployment/model_deployment/notebooks/query_endpoint.py index 51807a7..544fbc9 100644 --- a/solution/deployment/model_deployment/notebooks/query_endpoint.py +++ b/solution/deployment/model_deployment/notebooks/query_endpoint.py @@ -4,16 +4,18 @@ # MAGIC # MAGIC **Session:** Model Serving & Consumption # MAGIC -# MAGIC A short test harness for the endpoint the deployment job created. It replays real -# MAGIC transactions through the endpoint one at a time, the way a payment gateway would call -# MAGIC the model as each transaction happens. +# MAGIC Test harness for the endpoint the deployment job created. It sends real transactions +# MAGIC through the endpoint one at a time, as a payment gateway would call the model per +# MAGIC transaction. # MAGIC -# MAGIC Each call sets `client_request_id` = the transaction id: a correlation id the AI -# MAGIC Gateway logs in its own column, so the Monitoring session can join these predictions -# MAGIC back to confirmed-fraud labels later. The transaction id is metadata, not a model -# MAGIC input; the payload is just the raw transaction spine (`card_id`, `client_id`, -# MAGIC `amount`, `transaction_hour`, `mcc`, `use_chip`), and the endpoint looks up the entity -# MAGIC features and runs the on-demand functions per request. +# MAGIC Each call sets `client_request_id` to the transaction id, a correlation id the AI +# MAGIC Gateway logs in its own column, so the Monitoring session can join these predictions to +# MAGIC confirmed-fraud labels later. The transaction id is metadata, not a model input; the +# MAGIC payload is the raw transaction spine (`card_id`, `client_id`, `amount`, +# MAGIC `transaction_hour`, `mcc`, `use_chip`), and the endpoint looks up the entity features +# MAGIC and runs the on-demand functions per request. +# MAGIC +# MAGIC Docs: [Query serving endpoints](https://learn.microsoft.com/azure/databricks/machine-learning/model-serving/score-model-serving-endpoints) # MAGIC # MAGIC Docs: [Query serving endpoints](https://learn.microsoft.com/azure/databricks/machine-learning/model-serving/score-model-serving-endpoints) diff --git a/solution/exploration/notebooks/eda.py b/solution/exploration/notebooks/eda.py index 378c4fc..0459ca2 100644 --- a/solution/exploration/notebooks/eda.py +++ b/solution/exploration/notebooks/eda.py @@ -2,16 +2,16 @@ # MAGIC %md # MAGIC # Exploratory data analysis: fraud transactions # MAGIC -# MAGIC A quick look at the data before the ML labs, so you understand what you are modelling. -# MAGIC We read the shared gold table `transactions_enriched` (transaction, card, user and -# MAGIC merchant-category attributes already joined) and answer four questions: +# MAGIC A short look at the data before the ML labs. It reads the shared gold table +# MAGIC `transactions_enriched` (transaction, card, user, and merchant-category attributes +# MAGIC already joined) and answers four questions: # MAGIC -# MAGIC 1. How much data is there and what does a row look like? -# MAGIC 2. How rare is fraud? (the class imbalance that drives everything later) +# MAGIC 1. How much data is there, and what does a row look like? +# MAGIC 2. How rare is fraud (the class imbalance that drives the modelling choices)? # MAGIC 3. What do the numeric features look like? -# MAGIC 4. Does fraud behave differently (by amount, time of day, category)? +# MAGIC 4. Does fraud behave differently by amount, time of day, or category? # MAGIC -# MAGIC This notebook only reads data. It writes nothing, so it is safe to run any time. +# MAGIC This notebook only reads data. It writes nothing, so it is safe to run at any time. # COMMAND ---------- diff --git a/solution/feature_engineering/notebooks/build_features.py b/solution/feature_engineering/notebooks/build_features.py index 50f82a4..38fcc6b 100644 --- a/solution/feature_engineering/notebooks/build_features.py +++ b/solution/feature_engineering/notebooks/build_features.py @@ -59,9 +59,12 @@ # MAGIC %md # MAGIC ## 1. Build the entity feature tables -# MAGIC Slow-changing card and client attributes that exist before a transaction happens and -# MAGIC can be looked up by key. One row per entity (deduplicated), keyed by `card_id` / -# MAGIC `client_id`. +# MAGIC +# MAGIC Slow-changing card and client attributes that exist before a transaction happens, so +# MAGIC they can be looked up by key at serve time: +# MAGIC +# MAGIC - `card_features`, one row per `card_id`: credit limit, cards issued. +# MAGIC - `client_features`, one row per `client_id`: credit score, income, age. # COMMAND ---------- @@ -88,10 +91,14 @@ # MAGIC %md # MAGIC ## 2. Publish the entity feature tables -# MAGIC Write them to the Feature Store as primary-key tables so they can be looked up at -# MAGIC training time and (after publishing to an online store) at serving time. Create on -# MAGIC first run, merge on re-runs. Change Data Feed is enabled so the tables can sync to an -# MAGIC online store later. +# MAGIC +# MAGIC Write them to the Feature Store as primary-key tables so they can be resolved by key: +# MAGIC +# MAGIC - at training time (offline join), and +# MAGIC - at serving time, once published to an online store. +# MAGIC +# MAGIC Create on the first run, merge on re-runs. Change Data Feed is enabled so the tables can +# MAGIC sync to an online store later. # COMMAND ---------- @@ -122,9 +129,12 @@ def publish(name, df, key, description): # MAGIC %md # MAGIC ## 3. Register the on-demand feature functions +# MAGIC # MAGIC UC feature functions (Python UDFs) that compute transaction-derived features from the -# MAGIC request payload at lookup/serving time, so they work for a transaction never seen -# MAGIC before. Training and serving call the same functions (no skew). +# MAGIC request payload itself, so: +# MAGIC +# MAGIC - they work for a transaction never seen before (nothing to look up); +# MAGIC - training and serving call the same functions, so there is no skew. # COMMAND ---------- @@ -197,8 +207,9 @@ def publish(name, df, key, description): # MAGIC %md # MAGIC ## 4. Document the feature tables (governance) +# MAGIC # MAGIC Add column comments so the tables are self-describing in Catalog Explorer. -# MAGIC (Instructor-provided, not part of the exercise.) +# MAGIC Instructor-provided, not part of the exercise. # COMMAND ---------- diff --git a/solution/monitoring/notebooks/feature_drift_check.py b/solution/monitoring/notebooks/feature_drift_check.py index 7f7cbd9..6789ea0 100644 --- a/solution/monitoring/notebooks/feature_drift_check.py +++ b/solution/monitoring/notebooks/feature_drift_check.py @@ -4,21 +4,19 @@ # MAGIC # MAGIC **Session:** Monitoring & Retraining # MAGIC -# MAGIC **What this notebook is:** an automated job task (like `metric_violation_check.py`), not -# MAGIC a lab you open. It runs headless inside the scheduled `retraining_job` and emits a -# MAGIC true/false. Contrast `monitoring.py`, which is the human-facing dashboard that just -# MAGIC *displays* the same drift numbers for a person to read. +# MAGIC Automated task in the scheduled `retraining_job`. It reads the monitor's +# MAGIC `..._drift_metrics` table, decides whether one or more features have drifted enough for +# MAGIC long enough to act on, and publishes the boolean `is_drift_violated` as a task value. A +# MAGIC `condition_task` reads that value and, with the quality check, decides whether to retrain. # MAGIC -# MAGIC A sibling of `metric_violation_check.py`, but for **feature (data) drift** instead of -# MAGIC model quality. It reads the monitor's `..._drift_metrics` table and decides whether the -# MAGIC input distribution of one or more features has shifted enough, for long enough, to act -# MAGIC on. It publishes the boolean `is_drift_violated` as a job task value; a `condition_task` -# MAGIC in the retraining job reads it and (together with the quality check) decides whether to -# MAGIC retrain. +# MAGIC Related notebooks: # MAGIC -# MAGIC Why a separate notebook from the quality check: quality metrics live in -# MAGIC `..._profile_metrics` as structs keyed by `:table`/`model_version`, while drift lives in -# MAGIC `..._drift_metrics` with one row **per feature column**. Different table, different shape. +# MAGIC - `metric_violation_check.py`: the same pattern for model quality (`..._profile_metrics`). +# MAGIC - `monitoring.py`: the interactive dashboard that displays these metrics. +# MAGIC +# MAGIC Quality and drift live in separate tables: `..._profile_metrics` holds quality metrics +# MAGIC as structs keyed by `:table` / `model_version`, while `..._drift_metrics` holds one row +# MAGIC per feature column. # MAGIC # MAGIC Parameters: # MAGIC - `table_name_under_monitor`: the inference table the monitor profiles. @@ -30,10 +28,10 @@ # MAGIC - `num_evaluation_windows` / `num_violation_windows`: how many recent windows to look at, # MAGIC and how many must be in violation before we act. # MAGIC -# MAGIC A note on acting on feature drift: retraining only helps once **fresh labels** arrive, so -# MAGIC in production feature drift usually raises an **alert** first, and the label-based quality -# MAGIC check is what actually gates retraining. Here we wire drift into the same retraining -# MAGIC trigger (OR-ed with the quality check) to show the mechanism end-to-end. +# MAGIC Note on acting on feature drift: retraining only helps once fresh labels arrive, so in +# MAGIC production feature drift typically raises an alert first, and the label-based quality +# MAGIC check gates retraining. Here drift is wired into the same trigger (OR-ed with the +# MAGIC quality check) to demonstrate the mechanism end to end. # MAGIC # MAGIC Docs: [Monitor metric tables](https://learn.microsoft.com/azure/databricks/lakehouse-monitoring/monitor-output) diff --git a/solution/monitoring/notebooks/metric_violation_check.py b/solution/monitoring/notebooks/metric_violation_check.py index 48a1111..a4cdafd 100644 --- a/solution/monitoring/notebooks/metric_violation_check.py +++ b/solution/monitoring/notebooks/metric_violation_check.py @@ -4,24 +4,24 @@ # MAGIC # MAGIC **Session:** Monitoring & Retraining # MAGIC -# MAGIC **What this notebook is:** an automated job task, not a lab you open. It runs headless -# MAGIC inside the scheduled `retraining_job` (nobody watching) and produces a machine-readable -# MAGIC true/false. Contrast `monitoring.py`, which is the human-facing dashboard that only -# MAGIC *displays* these same metrics for a person to read. +# MAGIC Automated task in the scheduled `retraining_job`. It reads the monitor's +# MAGIC `..._profile_metrics` table, decides whether model quality has degraded across recent +# MAGIC windows, and publishes the boolean `is_metric_violated` as a task value. A downstream +# MAGIC `condition_task` reads that value and triggers `model_training_job` only when quality +# MAGIC has degraded. # MAGIC -# MAGIC One task in the retraining job. It reads the monitor's -# MAGIC `..._profile_metrics` table to decide whether model quality has degraded across -# MAGIC recent windows, and publishes the boolean answer as a job task value -# MAGIC (`is_metric_violated`). The next task is a `condition_task` that reads this value and -# MAGIC only then triggers `model_training_job`, so retraining fires only when the data says -# MAGIC the model has actually gotten worse. +# MAGIC Related notebooks: +# MAGIC +# MAGIC - `feature_drift_check.py`: the same pattern for feature drift (`..._drift_metrics`). +# MAGIC - `monitoring.py`: the interactive dashboard that displays these metrics. # MAGIC # MAGIC Parameters: +# MAGIC # MAGIC - `table_name_under_monitor`: the inference table the monitor profiles. # MAGIC - `metric_to_monitor`: quality metric column in `..._profile_metrics` (e.g. `f1_score`). # MAGIC - `metric_violation_threshold`: retrain if the metric drops below this. -# MAGIC - `num_evaluation_windows` / `num_violation_windows`: how many recent windows to look -# MAGIC at, and how many must be in violation before retraining. +# MAGIC - `num_evaluation_windows` / `num_violation_windows`: how many recent windows to check, +# MAGIC and how many must be in violation before retraining. # MAGIC # MAGIC Docs: [Lakehouse Monitoring metric tables](https://learn.microsoft.com/azure/databricks/lakehouse-monitoring/monitor-output) diff --git a/solution/monitoring/notebooks/monitoring.py b/solution/monitoring/notebooks/monitoring.py index cfe07ba..da98776 100644 --- a/solution/monitoring/notebooks/monitoring.py +++ b/solution/monitoring/notebooks/monitoring.py @@ -4,25 +4,20 @@ # MAGIC # MAGIC **Session:** Monitoring & Retraining # MAGIC -# MAGIC **What this notebook is:** the human-facing dashboard. You open and run it -# MAGIC interactively to *see* the metrics; it only `display()`s tables and makes no decisions, -# MAGIC and it is NOT part of the retraining job. The automated decisions live in two sibling -# MAGIC job-task notebooks, `metric_violation_check.py` (model quality) and -# MAGIC `feature_drift_check.py` (feature drift), which read these same metric tables headless -# MAGIC and emit a true/false signal the retraining job acts on. Same tables, different reader: -# MAGIC here a person looks; there the job decides. +# MAGIC Interactive dashboard. It refreshes the two Lakehouse Monitors and displays the metric +# MAGIC tables they produce; it makes no decisions and is not part of the retraining job. The +# MAGIC automated decisions run headless in two job tasks that read the same tables: +# MAGIC `metric_violation_check.py` (model quality) and `feature_drift_check.py` (feature drift). # MAGIC -# MAGIC Two Lakehouse Monitors were created declaratively by the bundle: one over the -# MAGIC batch report table and one over the online serving table. This lab refreshes them and -# MAGIC reads the metric tables they produce, so you can see model quality and drift over time -# MAGIC and compare the two prediction distributions. +# MAGIC The bundle creates two monitors: +# MAGIC +# MAGIC - **Batch table:** `dev._fraud.fraud_predictions` (labels present, drift and quality). +# MAGIC - **Online table:** `dev._fraud.fraud_serving_inference` (labels lag, drift-led). # MAGIC # MAGIC Each monitor emits two Delta tables plus a dashboard: -# MAGIC - `..._profile_metrics`: summary stats and model-quality metrics per window / model. -# MAGIC - `..._drift_metrics`: how the data / predictions drift window-over-window. # MAGIC -# MAGIC - **Batch table:** `dev._fraud.fraud_predictions` (has labels, so drift and quality) -# MAGIC - **Online table:** `dev._fraud.fraud_serving_inference` (labels lag, drift-led) +# MAGIC - `..._profile_metrics`: summary statistics and model-quality metrics per window / model. +# MAGIC - `..._drift_metrics`: data and prediction drift, window over window. # MAGIC # MAGIC Docs: [Lakehouse Monitoring](https://learn.microsoft.com/azure/databricks/lakehouse-monitoring/) @@ -53,9 +48,10 @@ # MAGIC %md # MAGIC ## 1. Refresh the monitors -# MAGIC The bundle created the monitors on a schedule, but in a live lab we refresh on demand -# MAGIC so the metric tables reflect the rows we just scored. A refresh recomputes the -# MAGIC profile and drift metrics; it can take a few minutes. +# MAGIC +# MAGIC The monitors run on a schedule, but here they are refreshed on demand so the metric +# MAGIC tables reflect the rows just scored. A refresh recomputes the profile and drift metrics +# MAGIC and can take a few minutes. # COMMAND ---------- @@ -84,9 +80,10 @@ # MAGIC %md # MAGIC ## 2. Model quality over time (batch) +# MAGIC # MAGIC The batch table has ground-truth labels, so its `_profile_metrics` table carries -# MAGIC model-quality metrics per time window. Read the whole-table rows for the real model -# MAGIC versions and watch the quality trend; this is what the retraining trigger checks. +# MAGIC model-quality metrics per window. This reads the whole-table rows for the real model +# MAGIC versions to show the quality trend, the signal the retraining trigger checks. # COMMAND ---------- @@ -125,9 +122,10 @@ # MAGIC %md # MAGIC ## 3. Drift over time (batch + online) -# MAGIC The `_drift_metrics` table quantifies how the predictions and features shift -# MAGIC window-over-window. Because online ad-hoc traffic is skewed differently from the -# MAGIC weekly batch, comparing the two drift profiles is exactly the signal we care about. +# MAGIC +# MAGIC The `_drift_metrics` table quantifies how predictions and features shift window over +# MAGIC window. Online traffic is distributed differently from the batch, so comparing the two +# MAGIC drift profiles is a useful signal. # COMMAND ---------- @@ -160,12 +158,12 @@ # MAGIC %md # MAGIC ## 4. Feature (data) drift -# MAGIC The same `_drift_metrics` table has a row per input **feature**, so we can watch the -# MAGIC input distribution shift, not just the prediction. `population_stability_index` (PSI) -# MAGIC is populated for numeric features (Lakehouse Monitoring leaves `js_distance` null for -# MAGIC them and only fills it for categoricals). Rising drift on a key feature is an early -# MAGIC warning that the world has moved; the retraining job's feature-drift check thresholds -# MAGIC exactly these values. +# MAGIC +# MAGIC The same `_drift_metrics` table has a row per input feature, so input distribution +# MAGIC shift can be tracked, not just prediction drift. `population_stability_index` (PSI) is +# MAGIC populated for numeric features; Lakehouse Monitoring leaves `js_distance` null for +# MAGIC numeric columns and fills it only for categoricals. The retraining job's feature-drift +# MAGIC check thresholds these same values. # COMMAND ---------- @@ -207,13 +205,13 @@ # MAGIC %md # MAGIC ## Recap -# MAGIC - **Two monitors, one picture:** batch (quality + drift) and online (drift-led) cover -# MAGIC both prediction distributions. -# MAGIC - **Prediction drift vs feature drift:** section 3 watches the model's output -# MAGIC distribution; section 4 watches the input features (data drift). Both come from the -# MAGIC same `_drift_metrics` table. -# MAGIC - The auto-generated dashboard (linked from each table in Catalog Explorer) is the -# MAGIC shareable view of everything above. -# MAGIC - The retraining job acts on two signals: the batch `_profile_metrics` quality trend -# MAGIC (label-based) and sustained feature drift from `_drift_metrics`. Either one can -# MAGIC trigger training, which then picks up the latest labelled data. +# MAGIC +# MAGIC - Two monitors cover both prediction distributions: batch (quality and drift) and +# MAGIC online (drift-led). +# MAGIC - Section 3 tracks prediction drift (model output); section 4 tracks feature drift +# MAGIC (model inputs). Both come from the same `_drift_metrics` table. +# MAGIC - The auto-generated dashboard, linked from each table in Catalog Explorer, is the +# MAGIC shareable view. +# MAGIC - The retraining job acts on two signals: the batch quality trend (`_profile_metrics`, +# MAGIC label-based) and sustained feature drift (`_drift_metrics`). Either can trigger +# MAGIC training, which retrains on the latest labelled data. diff --git a/solution/monitoring/notebooks/process_serving_logs.py b/solution/monitoring/notebooks/process_serving_logs.py index 1dd8e2d..f01413e 100644 --- a/solution/monitoring/notebooks/process_serving_logs.py +++ b/solution/monitoring/notebooks/process_serving_logs.py @@ -1,22 +1,22 @@ # Databricks notebook source # MAGIC %md -# MAGIC # Process online serving logs: the ad-hoc fraud checks +# MAGIC # Process online serving logs # MAGIC # MAGIC **Session:** Monitoring & Retraining # MAGIC -# MAGIC The serving endpoint (previous session) has an AI Gateway inference table that -# MAGIC auto-captures every request/response as raw JSON. That raw payload table can't be -# MAGIC profiled directly: Lakehouse Monitoring's InferenceLog analysis needs one row per -# MAGIC prediction with typed `prediction` / `model_version` / `timestamp` / `label` columns. -# MAGIC This notebook unpacks the payload table into that shape so the online monitor can -# MAGIC profile the ad-hoc traffic, as the batch job did for the report. +# MAGIC The serving endpoint has an AI Gateway inference table that captures every +# MAGIC request/response as raw JSON. That payload table cannot be profiled directly: +# MAGIC Lakehouse Monitoring's InferenceLog analysis needs one row per prediction with typed +# MAGIC `prediction` / `model_version` / `timestamp` / `label` columns. This notebook unpacks +# MAGIC the payload table into that shape so the online monitor can profile the traffic, as the +# MAGIC batch job does for the report. # MAGIC # MAGIC - **Read (personal):** `dev._fraud.fraud_serving_payload` (raw AI Gateway logs) # MAGIC - **Write (personal):** `dev._fraud.fraud_serving_inference` (InferenceLog-shaped) # MAGIC # MAGIC > Online requests have no ground-truth label at inference time (fraud is confirmed -# MAGIC > later), so we join `is_fraud` best-effort from gold and leave it null otherwise. The -# MAGIC > online monitor watches drift immediately; quality fills in as labels land. +# MAGIC > later), so `is_fraud` is joined best-effort from gold and left null otherwise. The +# MAGIC > online monitor tracks drift immediately; quality fills in as labels land. # MAGIC # MAGIC Docs: [AI Gateway inference tables](https://learn.microsoft.com/azure/databricks/ai-gateway/) @@ -24,9 +24,10 @@ # MAGIC %md # MAGIC ## Setup -# MAGIC Pure ETL: this notebook only reshapes what the AI Gateway already captured; it does -# MAGIC not call the endpoint. (Traffic is generated by the serving lab; AI Gateway logs land -# MAGIC within ~1 hour, so in a live class the freshest calls may not appear immediately.) +# MAGIC +# MAGIC This notebook only reshapes what the AI Gateway already captured; it does not call the +# MAGIC endpoint. Traffic is generated by the serving lab, and AI Gateway logs can take up to +# MAGIC ~1 hour to land, so the most recent calls may not appear immediately. # COMMAND ---------- diff --git a/solution/monitoring/notebooks/seed_drift.py b/solution/monitoring/notebooks/seed_drift.py index 07098a7..5543954 100644 --- a/solution/monitoring/notebooks/seed_drift.py +++ b/solution/monitoring/notebooks/seed_drift.py @@ -4,22 +4,20 @@ # MAGIC # MAGIC **Session:** Monitoring & Retraining # MAGIC -# MAGIC A small, safe helper so you can *see* feature drift inside the workshop instead of -# MAGIC waiting days for it to happen naturally. The batch inference table is already fully -# MAGIC scored, so instead of running the model again this takes a sample of existing -# MAGIC predictions, inflates the amount-driven features, and stamps them with **now**, so the -# MAGIC most recent monitor window looks different from the previous one. +# MAGIC Demo helper to make feature drift visible during the session instead of waiting for it +# MAGIC to occur naturally. The batch inference table is already scored, so this samples +# MAGIC existing predictions, inflates the amount-driven features, and stamps them with the +# MAGIC current time, so the most recent monitor window differs from the previous one. # MAGIC -# MAGIC It writes to **your own** tables (the per-user `dev__fraud` schema), exactly like -# MAGIC batch inference, so every attendee can run it independently: -# MAGIC - appends the drifted batch to `dev._fraud.fraud_predictions` -# MAGIC - refreshes your `batch_monitor`, so `_drift_metrics` picks up the spike +# MAGIC It writes to the per-user `dev__fraud` schema, so each attendee runs it +# MAGIC independently: # MAGIC -# MAGIC After it finishes, re-run `monitoring.py` section 4 to see the `amount` drift, and run -# MAGIC `feature_drift_check.py` to watch it return `is_drift_violated = True`. +# MAGIC - appends the drifted batch to `dev._fraud.fraud_predictions`; +# MAGIC - refreshes `batch_monitor` so `_drift_metrics` reflects the change. # MAGIC -# MAGIC Run **after** you have run batch inference at least once (so there is a normal baseline -# MAGIC in the earlier windows to drift away from) and the monitor exists. +# MAGIC Prerequisites: run batch inference at least once (to establish a baseline) and ensure +# MAGIC the monitor exists. Afterwards, re-run `monitoring.py` section 4 to view the `amount` +# MAGIC drift and `feature_drift_check.py` to confirm `is_drift_violated = True`. # MAGIC # MAGIC Docs: [Batch inference with Feature Engineering](https://learn.microsoft.com/azure/databricks/machine-learning/feature-store/score-batch) diff --git a/solution/training/notebooks/training.py b/solution/training/notebooks/training.py index 2d3c0fd..b90a245 100644 --- a/solution/training/notebooks/training.py +++ b/solution/training/notebooks/training.py @@ -90,11 +90,16 @@ # MAGIC %md # MAGIC ## Connect the model to its deployment job +# MAGIC # MAGIC If the training job passed a `deployment_job_id`, connect it to the registered model -# MAGIC **before** registering a version, so registering the new version auto-triggers the -# MAGIC governed evaluate/approve/deploy pipeline (even for the first version). -# MAGIC `create_registered_model` makes the empty model on first run; later runs just -# MAGIC `update_registered_model`. Skipped when run interactively without a job id. +# MAGIC **before** registering a version: +# MAGIC +# MAGIC - registering the new version then auto-triggers the governed evaluate/approve/deploy +# MAGIC pipeline, even for the first version; +# MAGIC - `create_registered_model` makes the empty model on the first run, later runs just +# MAGIC `update_registered_model`; +# MAGIC - skipped when run interactively without a job id. +# MAGIC # MAGIC See [MLflow deployment jobs](https://learn.microsoft.com/azure/databricks/mlflow/deployment-job). # COMMAND ---------- @@ -117,11 +122,12 @@ # MAGIC %md # MAGIC ## 1. Assemble the training set -# MAGIC The spine is the raw transaction: keys (`card_id`, `client_id`), the request fields -# MAGIC (`amount`, `transaction_hour`, `mcc`, `use_chip`) and the label. The Feature Store -# MAGIC looks up the card/client entity features and computes the on-demand features with the -# MAGIC UC functions. `create_training_set` records this so serving reproduces the same -# MAGIC feature pipeline. +# MAGIC +# MAGIC - The spine is the raw transaction: keys (`card_id`, `client_id`), the request fields +# MAGIC (`amount`, `transaction_hour`, `mcc`, `use_chip`), and the label. +# MAGIC - The Feature Store looks up the card/client entity features and computes the on-demand +# MAGIC features with the UC functions. +# MAGIC - `create_training_set` records this spec, so serving reproduces the same feature pipeline. # COMMAND ---------- @@ -223,15 +229,20 @@ # COMMAND ---------- # MAGIC %md -# MAGIC ## 2. Train, expose the imbalance trap, then fix it -# MAGIC Fraud is very rare (well under 1% of rows), so a model can score ~99.8% accuracy by -# MAGIC always predicting "not fraud" and catch zero fraud. We first train a naive random -# MAGIC forest to see that trap, then fix it by balancing the training data (keep every fraud -# MAGIC row, down-sample non-fraud to match) so the model learns what fraud looks like. We -# MAGIC still evaluate on the untouched, imbalanced test set so metrics reflect reality. -# MAGIC `fe.log_model` packages the fixed model with its feature metadata, infers a signature -# MAGIC and input example (so it can be served safely), and registers it to Unity Catalog in -# MAGIC one step (no separate `register_model` call). +# MAGIC ## 2. Train the model and address class imbalance +# MAGIC +# MAGIC Fraud accounts for well under 1% of rows, so a model can reach roughly 99.8% accuracy +# MAGIC by always predicting "not fraud" while detecting none. The workflow: +# MAGIC +# MAGIC - Train a baseline random forest on the raw, imbalanced data to establish the problem. +# MAGIC - Retrain on balanced data (retain every fraud row, down-sample non-fraud to match) so +# MAGIC the model learns the minority class. +# MAGIC - Evaluate on the original, imbalanced test set so the reported metrics reflect +# MAGIC production conditions. +# MAGIC +# MAGIC `fe.log_model` packages the model with its feature metadata, infers a signature and +# MAGIC input example, and registers it to Unity Catalog in a single step (no separate +# MAGIC `register_model` call). # COMMAND ---------- @@ -340,12 +351,16 @@ def predict(self, context, model_input): # COMMAND ---------- # MAGIC %md -# MAGIC ### Confusion matrices: the imbalance trap, side by side -# MAGIC Read the **fraud row** (bottom): the naive model dumps almost everything into -# MAGIC "predicted legit" (false negatives) and catches ~no fraud, while the balanced model -# MAGIC finally fills the **true-positive** cell, at the cost of more false positives -# MAGIC (top-right). Same untouched, imbalanced test set. The figure is also logged to the -# MAGIC `rf_balanced` MLflow run so it sits with the model version. +# MAGIC ### Confusion matrix comparison +# MAGIC +# MAGIC Compare the two models on the fraud row (bottom): +# MAGIC +# MAGIC - The baseline model assigns almost all fraud to "predicted legit" (false negatives) +# MAGIC and detects little fraud. +# MAGIC - The balanced model recovers true positives at the cost of more false positives. +# MAGIC +# MAGIC Both use the same original, imbalanced test set. The figure is logged to the +# MAGIC `rf_balanced` MLflow run so it stays with the model version. # COMMAND ---------- @@ -371,29 +386,30 @@ def predict(self, context, model_input): # COMMAND ---------- # MAGIC %md -# MAGIC ### But look at precision: the recall fix isn't free -# MAGIC The balanced model now catches most fraud (high recall), but its precision is very -# MAGIC low: most transactions it flags are actually legitimate. That's the precision/recall -# MAGIC trade-off. Balancing the data pushed the model to flag "fraud" far more often, -# MAGIC catching real fraud and drowning in false alarms. +# MAGIC ### Precision-recall trade-off +# MAGIC +# MAGIC The balanced model achieves high recall but low precision: most flagged transactions +# MAGIC are legitimate. Balancing the data makes the model predict "fraud" far more often, +# MAGIC which recovers real fraud but raises the false-positive rate. # MAGIC -# MAGIC In production you'd tune the decision threshold (score > 0.8 instead of 0.5) to trade -# MAGIC recall back for precision, and pick the operating point from the business cost of a -# MAGIC missed fraud vs. a false alarm, often measured as precision@k or PR-AUC rather than a -# MAGIC single accuracy/recall number. That's a business decision, not a modelling one, which -# MAGIC is why the promotion gate (next section) should encode the metric the business cares -# MAGIC about. +# MAGIC In production the decision threshold is tuned (for example, score > 0.8 rather than +# MAGIC 0.5) to trade recall for precision. The operating point is chosen from the business +# MAGIC cost of a missed fraud versus a false alarm, and is typically assessed with precision@k +# MAGIC or PR-AUC rather than a single accuracy or recall figure. Because this is a business +# MAGIC decision, the promotion gate in the next section encodes the metric the business +# MAGIC requires. # COMMAND ---------- # MAGIC %md # MAGIC ## 3. Tag the new version as `challenger` (instructor-provided) -# MAGIC The training run registered a fresh model version. Point the `challenger` alias at it -# MAGIC for a clear before/after picture. Registering the version also auto-triggers the -# MAGIC deployment job, whose Evaluation task re-scores it on a fresh holdout and compares it -# MAGIC against the current `@champion` before it can be promoted. Downstream jobs always load -# MAGIC `@champion`, so promotion is just an alias reassignment: no code change and instant -# MAGIC rollback. +# MAGIC +# MAGIC The training run registered a new model version. Point the `challenger` alias at it to +# MAGIC give a clear before/after comparison. Registering the version also triggers the +# MAGIC deployment job, whose Evaluation task re-scores the version on a fresh holdout and +# MAGIC compares it against the current `@champion` before promotion. Downstream jobs always +# MAGIC load `@champion`, so promotion is an alias reassignment: no code change, and rollback +# MAGIC is immediate. # COMMAND ---------- @@ -409,12 +425,12 @@ def predict(self, context, model_input): # COMMAND ---------- # MAGIC %md -# MAGIC ## Train a challenger (try it) +# MAGIC ## Train a challenger # MAGIC The first model becomes `@champion` once the deployment job promotes it. To produce a # MAGIC challenger to compete against it, change the model and re-run this whole notebook. # MAGIC Each run registers a new version and moves `@challenger` to it: # MAGIC -# MAGIC - In **section 2**, tweak the hyper-parameters in `params`, for example: +# MAGIC - In **section 2**, adjust the hyper-parameters in `params`, for example: # MAGIC - bump `n_estimators` to `200`, or # MAGIC - add `max_depth=8` or `min_samples_leaf=5` to regularise, or # MAGIC - swap in a different estimator (e.g. `GradientBoostingClassifier`). diff --git a/src/data_ingestion/notebooks/load_bronze_to_silver.py b/src/data_ingestion/notebooks/load_bronze_to_silver.py index 6b667cb..78f0bdd 100644 --- a/src/data_ingestion/notebooks/load_bronze_to_silver.py +++ b/src/data_ingestion/notebooks/load_bronze_to_silver.py @@ -1,10 +1,15 @@ # Databricks notebook source # MAGIC %md # MAGIC # Bronze -> Silver -# MAGIC Clean, type, dedupe and join the Bronze tables into conformed Silver tables, -# MAGIC including the enriched transaction table used for feature engineering. # MAGIC -# MAGIC **Setup notebook** run by the instructor before the workshop. Not a lab exercise. +# MAGIC Instructor setup notebook, run before the workshop. Not a lab exercise. +# MAGIC +# MAGIC Turns the Bronze tables into conformed Silver tables (and the enriched Gold +# MAGIC transaction table that feature engineering reads) by: +# MAGIC +# MAGIC - cleaning and casting columns to their real types; +# MAGIC - deduplicating to one row per entity; +# MAGIC - joining the sources into a single labelled transaction table. # MAGIC # MAGIC | Layer | Location | # MAGIC |---|---| @@ -59,9 +64,10 @@ # MAGIC %md # MAGIC ## Target schemas -# MAGIC The shared `fraud_silver` / `fraud_gold` schemas are declared as bundle resources -# MAGIC (the `shared_infra` block in `databricks.yml`) and created by -# MAGIC `databricks bundle deploy -t dev` before this job runs. +# MAGIC +# MAGIC - The `fraud_silver` / `fraud_gold` schemas are declared as bundle resources (the +# MAGIC `shared_infra` block in `databricks.yml`). +# MAGIC - `databricks bundle deploy -t dev` creates them before this job runs. # COMMAND ---------- diff --git a/src/data_ingestion/notebooks/load_landing_to_bronze.py b/src/data_ingestion/notebooks/load_landing_to_bronze.py index 5ea278a..8e4adf2 100644 --- a/src/data_ingestion/notebooks/load_landing_to_bronze.py +++ b/src/data_ingestion/notebooks/load_landing_to_bronze.py @@ -1,10 +1,14 @@ # Databricks notebook source # MAGIC %md # MAGIC # Landing -> Bronze -# MAGIC Ingest the raw fraud source files from the landing volume into Bronze Delta -# MAGIC tables, applying explicit schemas and audit columns. Minimal transformation. # MAGIC -# MAGIC **Setup notebook** run by the instructor before the workshop. Not a lab exercise. +# MAGIC Instructor setup notebook, run before the workshop. Not a lab exercise. +# MAGIC +# MAGIC Reads the raw fraud source files from the landing volume into Bronze Delta tables: +# MAGIC +# MAGIC - explicit schemas, no inference; +# MAGIC - audit columns (ingest timestamp, ingest date, source file); +# MAGIC - minimal transformation, so the shape stays close to the source. # MAGIC # MAGIC | Layer | Location | # MAGIC |---|---| @@ -59,10 +63,11 @@ # MAGIC %md # MAGIC ## Target schemas and volume -# MAGIC The shared `fraud_landing` / `fraud_bronze` schemas and the `raw_data` volume are -# MAGIC declared as bundle resources (the `shared_infra` block in `databricks.yml`) and -# MAGIC created by `databricks bundle deploy -t dev` before this job runs. The raw files are -# MAGIC uploaded into the volume after the deploy, and this notebook just reads them. +# MAGIC +# MAGIC - The `fraud_landing` / `fraud_bronze` schemas and the `raw_data` volume are declared as +# MAGIC bundle resources (the `shared_infra` block in `databricks.yml`). +# MAGIC - `databricks bundle deploy -t dev` creates them before this job runs. +# MAGIC - The raw files are uploaded into the volume after the deploy; this notebook only reads them. # COMMAND ---------- diff --git a/src/deployment/batch_inference/notebooks/batch_inference.py b/src/deployment/batch_inference/notebooks/batch_inference.py index f6238f0..dc66a94 100644 --- a/src/deployment/batch_inference/notebooks/batch_inference.py +++ b/src/deployment/batch_inference/notebooks/batch_inference.py @@ -1,21 +1,23 @@ # Databricks notebook source # MAGIC %md -# MAGIC # Batch inference: the weekly fraud report +# MAGIC # Batch inference # MAGIC # MAGIC **Session:** Model Serving & Consumption / Monitoring & Retraining # MAGIC # MAGIC Score a batch of transactions with the governed `@champion` model and append the -# MAGIC results to an inference table. This is the batch prediction path (a scheduled "weekly -# MAGIC report"), and the table it writes is what the batch monitor profiles in the -# MAGIC Monitoring session. +# MAGIC results to an inference table. This is the scheduled batch prediction path, and the +# MAGIC table it writes is what the batch monitor profiles in the Monitoring session. # MAGIC # MAGIC - **Model (personal):** `dev._fraud.fraud_detection@champion` # MAGIC - **Inference table (personal):** `dev._fraud.fraud_predictions` # MAGIC -# MAGIC To be usable by Lakehouse Monitoring's InferenceLog analysis, every row needs four -# MAGIC things beyond the features: a prediction, the model version that produced it, a -# MAGIC timestamp, and (when it arrives) the ground-truth label. We write all four so the -# MAGIC monitor can track both drift and quality over time. +# MAGIC Lakehouse Monitoring's InferenceLog analysis needs four columns beyond the features on +# MAGIC every row, all written here so the monitor can track drift and quality: +# MAGIC +# MAGIC - the prediction; +# MAGIC - the model version that produced it; +# MAGIC - a scoring timestamp; +# MAGIC - the ground-truth label, when available. # MAGIC # MAGIC Docs: [Batch inference with Feature Engineering](https://learn.microsoft.com/azure/databricks/machine-learning/feature-store/score-batch) diff --git a/src/deployment/model_deployment/notebooks/approval.py b/src/deployment/model_deployment/notebooks/approval.py index 6e32002..c99dc17 100644 --- a/src/deployment/model_deployment/notebooks/approval.py +++ b/src/deployment/model_deployment/notebooks/approval.py @@ -9,7 +9,7 @@ # MAGIC as the signal (the mechanism MLflow 3 deployment jobs use): # MAGIC # MAGIC - **dev** (`auto_approve=true`): the task sets the approval tag itself and passes, so -# MAGIC the whole loop runs unattended in class. +# MAGIC the whole loop runs unattended. # MAGIC - **staging / prod** (`auto_approve=false`): the task fails until a human reviews the # MAGIC evaluation metrics on the model-version page and clicks **Approve** (which sets the # MAGIC tag `=Approved` and repairs the run). The task then passes and the run diff --git a/src/deployment/model_deployment/notebooks/evaluation.py b/src/deployment/model_deployment/notebooks/evaluation.py index 79fba5a..d245921 100644 --- a/src/deployment/model_deployment/notebooks/evaluation.py +++ b/src/deployment/model_deployment/notebooks/evaluation.py @@ -15,7 +15,7 @@ # MAGIC 3. **Deployment**: promote the version to `@champion` and serve it. # MAGIC # MAGIC The version being evaluated is the one that triggered the job, so there is no -# MAGIC champion/challenger bookkeeping here. We evaluate this version and gate it against a +# MAGIC champion/challenger bookkeeping here. The version is evaluated and gated against a # MAGIC metric floor (and the current champion, if any). # MAGIC See [MLflow deployment jobs](https://learn.microsoft.com/azure/databricks/mlflow/deployment-job). @@ -106,9 +106,10 @@ def score_holdout(version): # MAGIC %md # MAGIC ### Confusion matrix for the approver +# MAGIC # MAGIC Threshold the candidate's fraud score at 0.5 and show the confusion matrix on the -# MAGIC holdout, logged to an MLflow run so it sits alongside the metrics an approver reviews -# MAGIC before clicking Approve. +# MAGIC holdout, logged to an MLflow run so it sits alongside the metrics the approver reviews +# MAGIC before approving. # COMMAND ---------- @@ -134,11 +135,15 @@ def score_holdout(version): # MAGIC %md # MAGIC ### ROC and precision-recall curves -# MAGIC The confusion matrix is a single operating point; these curves show the whole -# MAGIC threshold trade-off. The **ROC** curve's area is the gate metric (threshold-independent), -# MAGIC and on rare-fraud data the **precision-recall** curve (with its average precision) is the -# MAGIC more honest read of how much precision you give up as you chase recall. Both are logged -# MAGIC to the same evaluation run, alongside the confusion matrix. +# MAGIC +# MAGIC The confusion matrix is a single operating point; these curves show the full threshold +# MAGIC trade-off: +# MAGIC +# MAGIC - the ROC curve's area is the gate metric (threshold-independent); +# MAGIC - on rare-fraud data, the precision-recall curve (and its average precision) shows how +# MAGIC much precision is given up as recall increases. +# MAGIC +# MAGIC Both are logged to the same evaluation run, alongside the confusion matrix. # COMMAND ---------- diff --git a/src/deployment/model_deployment/notebooks/query_endpoint.py b/src/deployment/model_deployment/notebooks/query_endpoint.py index 51807a7..544fbc9 100644 --- a/src/deployment/model_deployment/notebooks/query_endpoint.py +++ b/src/deployment/model_deployment/notebooks/query_endpoint.py @@ -4,16 +4,18 @@ # MAGIC # MAGIC **Session:** Model Serving & Consumption # MAGIC -# MAGIC A short test harness for the endpoint the deployment job created. It replays real -# MAGIC transactions through the endpoint one at a time, the way a payment gateway would call -# MAGIC the model as each transaction happens. +# MAGIC Test harness for the endpoint the deployment job created. It sends real transactions +# MAGIC through the endpoint one at a time, as a payment gateway would call the model per +# MAGIC transaction. # MAGIC -# MAGIC Each call sets `client_request_id` = the transaction id: a correlation id the AI -# MAGIC Gateway logs in its own column, so the Monitoring session can join these predictions -# MAGIC back to confirmed-fraud labels later. The transaction id is metadata, not a model -# MAGIC input; the payload is just the raw transaction spine (`card_id`, `client_id`, -# MAGIC `amount`, `transaction_hour`, `mcc`, `use_chip`), and the endpoint looks up the entity -# MAGIC features and runs the on-demand functions per request. +# MAGIC Each call sets `client_request_id` to the transaction id, a correlation id the AI +# MAGIC Gateway logs in its own column, so the Monitoring session can join these predictions to +# MAGIC confirmed-fraud labels later. The transaction id is metadata, not a model input; the +# MAGIC payload is the raw transaction spine (`card_id`, `client_id`, `amount`, +# MAGIC `transaction_hour`, `mcc`, `use_chip`), and the endpoint looks up the entity features +# MAGIC and runs the on-demand functions per request. +# MAGIC +# MAGIC Docs: [Query serving endpoints](https://learn.microsoft.com/azure/databricks/machine-learning/model-serving/score-model-serving-endpoints) # MAGIC # MAGIC Docs: [Query serving endpoints](https://learn.microsoft.com/azure/databricks/machine-learning/model-serving/score-model-serving-endpoints) diff --git a/src/exploration/notebooks/eda.py b/src/exploration/notebooks/eda.py index 378c4fc..0459ca2 100644 --- a/src/exploration/notebooks/eda.py +++ b/src/exploration/notebooks/eda.py @@ -2,16 +2,16 @@ # MAGIC %md # MAGIC # Exploratory data analysis: fraud transactions # MAGIC -# MAGIC A quick look at the data before the ML labs, so you understand what you are modelling. -# MAGIC We read the shared gold table `transactions_enriched` (transaction, card, user and -# MAGIC merchant-category attributes already joined) and answer four questions: +# MAGIC A short look at the data before the ML labs. It reads the shared gold table +# MAGIC `transactions_enriched` (transaction, card, user, and merchant-category attributes +# MAGIC already joined) and answers four questions: # MAGIC -# MAGIC 1. How much data is there and what does a row look like? -# MAGIC 2. How rare is fraud? (the class imbalance that drives everything later) +# MAGIC 1. How much data is there, and what does a row look like? +# MAGIC 2. How rare is fraud (the class imbalance that drives the modelling choices)? # MAGIC 3. What do the numeric features look like? -# MAGIC 4. Does fraud behave differently (by amount, time of day, category)? +# MAGIC 4. Does fraud behave differently by amount, time of day, or category? # MAGIC -# MAGIC This notebook only reads data. It writes nothing, so it is safe to run any time. +# MAGIC This notebook only reads data. It writes nothing, so it is safe to run at any time. # COMMAND ---------- diff --git a/src/feature_engineering/notebooks/build_features.py b/src/feature_engineering/notebooks/build_features.py index a08ff7a..c2537b8 100644 --- a/src/feature_engineering/notebooks/build_features.py +++ b/src/feature_engineering/notebooks/build_features.py @@ -59,9 +59,12 @@ # MAGIC %md # MAGIC ## 1. Build the entity feature tables -# MAGIC Slow-changing card and client attributes that exist before a transaction happens and -# MAGIC can be looked up by key. One row per entity (deduplicated), keyed by `card_id` / -# MAGIC `client_id`. +# MAGIC +# MAGIC Slow-changing card and client attributes that exist before a transaction happens, so +# MAGIC they can be looked up by key at serve time: +# MAGIC +# MAGIC - `card_features`, one row per `card_id`: credit limit, cards issued. +# MAGIC - `client_features`, one row per `client_id`: credit score, income, age. # COMMAND ---------- @@ -88,10 +91,14 @@ # MAGIC %md # MAGIC ## 2. Publish the entity feature tables -# MAGIC Write them to the Feature Store as primary-key tables so they can be looked up at -# MAGIC training time and (after publishing to an online store) at serving time. Create on -# MAGIC first run, merge on re-runs. Change Data Feed is enabled so the tables can sync to an -# MAGIC online store later. +# MAGIC +# MAGIC Write them to the Feature Store as primary-key tables so they can be resolved by key: +# MAGIC +# MAGIC - at training time (offline join), and +# MAGIC - at serving time, once published to an online store. +# MAGIC +# MAGIC Create on the first run, merge on re-runs. Change Data Feed is enabled so the tables can +# MAGIC sync to an online store later. # COMMAND ---------- @@ -107,9 +114,12 @@ # MAGIC %md # MAGIC ## 3. Register the on-demand feature functions +# MAGIC # MAGIC UC feature functions (Python UDFs) that compute transaction-derived features from the -# MAGIC request payload at lookup/serving time, so they work for a transaction never seen -# MAGIC before. Training and serving call the same functions (no skew). +# MAGIC request payload itself, so: +# MAGIC +# MAGIC - they work for a transaction never seen before (nothing to look up); +# MAGIC - training and serving call the same functions, so there is no skew. # COMMAND ---------- @@ -173,8 +183,9 @@ # MAGIC %md # MAGIC ## 4. Document the feature tables (governance) +# MAGIC # MAGIC Add column comments so the tables are self-describing in Catalog Explorer. -# MAGIC (Instructor-provided, not part of the exercise.) +# MAGIC Instructor-provided, not part of the exercise. # COMMAND ---------- diff --git a/src/monitoring/notebooks/feature_drift_check.py b/src/monitoring/notebooks/feature_drift_check.py index 7f7cbd9..6789ea0 100644 --- a/src/monitoring/notebooks/feature_drift_check.py +++ b/src/monitoring/notebooks/feature_drift_check.py @@ -4,21 +4,19 @@ # MAGIC # MAGIC **Session:** Monitoring & Retraining # MAGIC -# MAGIC **What this notebook is:** an automated job task (like `metric_violation_check.py`), not -# MAGIC a lab you open. It runs headless inside the scheduled `retraining_job` and emits a -# MAGIC true/false. Contrast `monitoring.py`, which is the human-facing dashboard that just -# MAGIC *displays* the same drift numbers for a person to read. +# MAGIC Automated task in the scheduled `retraining_job`. It reads the monitor's +# MAGIC `..._drift_metrics` table, decides whether one or more features have drifted enough for +# MAGIC long enough to act on, and publishes the boolean `is_drift_violated` as a task value. A +# MAGIC `condition_task` reads that value and, with the quality check, decides whether to retrain. # MAGIC -# MAGIC A sibling of `metric_violation_check.py`, but for **feature (data) drift** instead of -# MAGIC model quality. It reads the monitor's `..._drift_metrics` table and decides whether the -# MAGIC input distribution of one or more features has shifted enough, for long enough, to act -# MAGIC on. It publishes the boolean `is_drift_violated` as a job task value; a `condition_task` -# MAGIC in the retraining job reads it and (together with the quality check) decides whether to -# MAGIC retrain. +# MAGIC Related notebooks: # MAGIC -# MAGIC Why a separate notebook from the quality check: quality metrics live in -# MAGIC `..._profile_metrics` as structs keyed by `:table`/`model_version`, while drift lives in -# MAGIC `..._drift_metrics` with one row **per feature column**. Different table, different shape. +# MAGIC - `metric_violation_check.py`: the same pattern for model quality (`..._profile_metrics`). +# MAGIC - `monitoring.py`: the interactive dashboard that displays these metrics. +# MAGIC +# MAGIC Quality and drift live in separate tables: `..._profile_metrics` holds quality metrics +# MAGIC as structs keyed by `:table` / `model_version`, while `..._drift_metrics` holds one row +# MAGIC per feature column. # MAGIC # MAGIC Parameters: # MAGIC - `table_name_under_monitor`: the inference table the monitor profiles. @@ -30,10 +28,10 @@ # MAGIC - `num_evaluation_windows` / `num_violation_windows`: how many recent windows to look at, # MAGIC and how many must be in violation before we act. # MAGIC -# MAGIC A note on acting on feature drift: retraining only helps once **fresh labels** arrive, so -# MAGIC in production feature drift usually raises an **alert** first, and the label-based quality -# MAGIC check is what actually gates retraining. Here we wire drift into the same retraining -# MAGIC trigger (OR-ed with the quality check) to show the mechanism end-to-end. +# MAGIC Note on acting on feature drift: retraining only helps once fresh labels arrive, so in +# MAGIC production feature drift typically raises an alert first, and the label-based quality +# MAGIC check gates retraining. Here drift is wired into the same trigger (OR-ed with the +# MAGIC quality check) to demonstrate the mechanism end to end. # MAGIC # MAGIC Docs: [Monitor metric tables](https://learn.microsoft.com/azure/databricks/lakehouse-monitoring/monitor-output) diff --git a/src/monitoring/notebooks/metric_violation_check.py b/src/monitoring/notebooks/metric_violation_check.py index 48a1111..a4cdafd 100644 --- a/src/monitoring/notebooks/metric_violation_check.py +++ b/src/monitoring/notebooks/metric_violation_check.py @@ -4,24 +4,24 @@ # MAGIC # MAGIC **Session:** Monitoring & Retraining # MAGIC -# MAGIC **What this notebook is:** an automated job task, not a lab you open. It runs headless -# MAGIC inside the scheduled `retraining_job` (nobody watching) and produces a machine-readable -# MAGIC true/false. Contrast `monitoring.py`, which is the human-facing dashboard that only -# MAGIC *displays* these same metrics for a person to read. +# MAGIC Automated task in the scheduled `retraining_job`. It reads the monitor's +# MAGIC `..._profile_metrics` table, decides whether model quality has degraded across recent +# MAGIC windows, and publishes the boolean `is_metric_violated` as a task value. A downstream +# MAGIC `condition_task` reads that value and triggers `model_training_job` only when quality +# MAGIC has degraded. # MAGIC -# MAGIC One task in the retraining job. It reads the monitor's -# MAGIC `..._profile_metrics` table to decide whether model quality has degraded across -# MAGIC recent windows, and publishes the boolean answer as a job task value -# MAGIC (`is_metric_violated`). The next task is a `condition_task` that reads this value and -# MAGIC only then triggers `model_training_job`, so retraining fires only when the data says -# MAGIC the model has actually gotten worse. +# MAGIC Related notebooks: +# MAGIC +# MAGIC - `feature_drift_check.py`: the same pattern for feature drift (`..._drift_metrics`). +# MAGIC - `monitoring.py`: the interactive dashboard that displays these metrics. # MAGIC # MAGIC Parameters: +# MAGIC # MAGIC - `table_name_under_monitor`: the inference table the monitor profiles. # MAGIC - `metric_to_monitor`: quality metric column in `..._profile_metrics` (e.g. `f1_score`). # MAGIC - `metric_violation_threshold`: retrain if the metric drops below this. -# MAGIC - `num_evaluation_windows` / `num_violation_windows`: how many recent windows to look -# MAGIC at, and how many must be in violation before retraining. +# MAGIC - `num_evaluation_windows` / `num_violation_windows`: how many recent windows to check, +# MAGIC and how many must be in violation before retraining. # MAGIC # MAGIC Docs: [Lakehouse Monitoring metric tables](https://learn.microsoft.com/azure/databricks/lakehouse-monitoring/monitor-output) diff --git a/src/monitoring/notebooks/monitoring.py b/src/monitoring/notebooks/monitoring.py index 885fd25..7760926 100644 --- a/src/monitoring/notebooks/monitoring.py +++ b/src/monitoring/notebooks/monitoring.py @@ -4,25 +4,20 @@ # MAGIC # MAGIC **Session:** Monitoring & Retraining # MAGIC -# MAGIC **What this notebook is:** the human-facing dashboard. You open and run it -# MAGIC interactively to *see* the metrics; it only `display()`s tables and makes no decisions, -# MAGIC and it is NOT part of the retraining job. The automated decisions live in two sibling -# MAGIC job-task notebooks, `metric_violation_check.py` (model quality) and -# MAGIC `feature_drift_check.py` (feature drift), which read these same metric tables headless -# MAGIC and emit a true/false signal the retraining job acts on. Same tables, different reader: -# MAGIC here a person looks; there the job decides. +# MAGIC Interactive dashboard. It refreshes the two Lakehouse Monitors and displays the metric +# MAGIC tables they produce; it makes no decisions and is not part of the retraining job. The +# MAGIC automated decisions run headless in two job tasks that read the same tables: +# MAGIC `metric_violation_check.py` (model quality) and `feature_drift_check.py` (feature drift). # MAGIC -# MAGIC Two Lakehouse Monitors were created declaratively by the bundle: one over the -# MAGIC batch report table and one over the online serving table. This lab refreshes them and -# MAGIC reads the metric tables they produce, so you can see model quality and drift over time -# MAGIC and compare the two prediction distributions. +# MAGIC The bundle creates two monitors: +# MAGIC +# MAGIC - **Batch table:** `dev._fraud.fraud_predictions` (labels present, drift and quality). +# MAGIC - **Online table:** `dev._fraud.fraud_serving_inference` (labels lag, drift-led). # MAGIC # MAGIC Each monitor emits two Delta tables plus a dashboard: -# MAGIC - `..._profile_metrics`: summary stats and model-quality metrics per window / model. -# MAGIC - `..._drift_metrics`: how the data / predictions drift window-over-window. # MAGIC -# MAGIC - **Batch table:** `dev._fraud.fraud_predictions` (has labels, so drift and quality) -# MAGIC - **Online table:** `dev._fraud.fraud_serving_inference` (labels lag, drift-led) +# MAGIC - `..._profile_metrics`: summary statistics and model-quality metrics per window / model. +# MAGIC - `..._drift_metrics`: data and prediction drift, window over window. # MAGIC # MAGIC Docs: [Lakehouse Monitoring](https://learn.microsoft.com/azure/databricks/lakehouse-monitoring/) @@ -53,9 +48,10 @@ # MAGIC %md # MAGIC ## 1. Refresh the monitors -# MAGIC The bundle created the monitors on a schedule, but in a live lab we refresh on demand -# MAGIC so the metric tables reflect the rows we just scored. A refresh recomputes the -# MAGIC profile and drift metrics; it can take a few minutes. +# MAGIC +# MAGIC The monitors run on a schedule, but here they are refreshed on demand so the metric +# MAGIC tables reflect the rows just scored. A refresh recomputes the profile and drift metrics +# MAGIC and can take a few minutes. # COMMAND ---------- @@ -83,9 +79,10 @@ # MAGIC %md # MAGIC ## 2. Model quality over time (batch) +# MAGIC # MAGIC The batch table has ground-truth labels, so its `_profile_metrics` table carries -# MAGIC model-quality metrics per time window. Read the whole-table rows for the real model -# MAGIC versions and watch the quality trend; this is what the retraining trigger checks. +# MAGIC model-quality metrics per window. This reads the whole-table rows for the real model +# MAGIC versions to show the quality trend, the signal the retraining trigger checks. # COMMAND ---------- @@ -124,9 +121,10 @@ # MAGIC %md # MAGIC ## 3. Drift over time (batch + online) -# MAGIC The `_drift_metrics` table quantifies how the predictions and features shift -# MAGIC window-over-window. Because online ad-hoc traffic is skewed differently from the -# MAGIC weekly batch, comparing the two drift profiles is exactly the signal we care about. +# MAGIC +# MAGIC The `_drift_metrics` table quantifies how predictions and features shift window over +# MAGIC window. Online traffic is distributed differently from the batch, so comparing the two +# MAGIC drift profiles is a useful signal. # COMMAND ---------- @@ -159,12 +157,12 @@ # MAGIC %md # MAGIC ## 4. Feature (data) drift -# MAGIC The same `_drift_metrics` table has a row per input **feature**, so we can watch the -# MAGIC input distribution shift, not just the prediction. `population_stability_index` (PSI) -# MAGIC is populated for numeric features (Lakehouse Monitoring leaves `js_distance` null for -# MAGIC them and only fills it for categoricals). Rising drift on a key feature is an early -# MAGIC warning that the world has moved; the retraining job's feature-drift check thresholds -# MAGIC exactly these values. +# MAGIC +# MAGIC The same `_drift_metrics` table has a row per input feature, so input distribution +# MAGIC shift can be tracked, not just prediction drift. `population_stability_index` (PSI) is +# MAGIC populated for numeric features; Lakehouse Monitoring leaves `js_distance` null for +# MAGIC numeric columns and fills it only for categoricals. The retraining job's feature-drift +# MAGIC check thresholds these same values. # COMMAND ---------- @@ -206,13 +204,13 @@ # MAGIC %md # MAGIC ## Recap -# MAGIC - **Two monitors, one picture:** batch (quality + drift) and online (drift-led) cover -# MAGIC both prediction distributions. -# MAGIC - **Prediction drift vs feature drift:** section 3 watches the model's output -# MAGIC distribution; section 4 watches the input features (data drift). Both come from the -# MAGIC same `_drift_metrics` table. -# MAGIC - The auto-generated dashboard (linked from each table in Catalog Explorer) is the -# MAGIC shareable view of everything above. -# MAGIC - The retraining job acts on two signals: the batch `_profile_metrics` quality trend -# MAGIC (label-based) and sustained feature drift from `_drift_metrics`. Either one can -# MAGIC trigger training, which then picks up the latest labelled data. +# MAGIC +# MAGIC - Two monitors cover both prediction distributions: batch (quality and drift) and +# MAGIC online (drift-led). +# MAGIC - Section 3 tracks prediction drift (model output); section 4 tracks feature drift +# MAGIC (model inputs). Both come from the same `_drift_metrics` table. +# MAGIC - The auto-generated dashboard, linked from each table in Catalog Explorer, is the +# MAGIC shareable view. +# MAGIC - The retraining job acts on two signals: the batch quality trend (`_profile_metrics`, +# MAGIC label-based) and sustained feature drift (`_drift_metrics`). Either can trigger +# MAGIC training, which retrains on the latest labelled data. diff --git a/src/monitoring/notebooks/process_serving_logs.py b/src/monitoring/notebooks/process_serving_logs.py index 8f04890..bbb725a 100644 --- a/src/monitoring/notebooks/process_serving_logs.py +++ b/src/monitoring/notebooks/process_serving_logs.py @@ -1,22 +1,22 @@ # Databricks notebook source # MAGIC %md -# MAGIC # Process online serving logs: the ad-hoc fraud checks +# MAGIC # Process online serving logs # MAGIC # MAGIC **Session:** Monitoring & Retraining # MAGIC -# MAGIC The serving endpoint (previous session) has an AI Gateway inference table that -# MAGIC auto-captures every request/response as raw JSON. That raw payload table can't be -# MAGIC profiled directly: Lakehouse Monitoring's InferenceLog analysis needs one row per -# MAGIC prediction with typed `prediction` / `model_version` / `timestamp` / `label` columns. -# MAGIC This notebook unpacks the payload table into that shape so the online monitor can -# MAGIC profile the ad-hoc traffic, as the batch job did for the report. +# MAGIC The serving endpoint has an AI Gateway inference table that captures every +# MAGIC request/response as raw JSON. That payload table cannot be profiled directly: +# MAGIC Lakehouse Monitoring's InferenceLog analysis needs one row per prediction with typed +# MAGIC `prediction` / `model_version` / `timestamp` / `label` columns. This notebook unpacks +# MAGIC the payload table into that shape so the online monitor can profile the traffic, as the +# MAGIC batch job does for the report. # MAGIC # MAGIC - **Read (personal):** `dev._fraud.fraud_serving_payload` (raw AI Gateway logs) # MAGIC - **Write (personal):** `dev._fraud.fraud_serving_inference` (InferenceLog-shaped) # MAGIC # MAGIC > Online requests have no ground-truth label at inference time (fraud is confirmed -# MAGIC > later), so we join `is_fraud` best-effort from gold and leave it null otherwise. The -# MAGIC > online monitor watches drift immediately; quality fills in as labels land. +# MAGIC > later), so `is_fraud` is joined best-effort from gold and left null otherwise. The +# MAGIC > online monitor tracks drift immediately; quality fills in as labels land. # MAGIC # MAGIC Docs: [AI Gateway inference tables](https://learn.microsoft.com/azure/databricks/ai-gateway/) @@ -24,9 +24,10 @@ # MAGIC %md # MAGIC ## Setup -# MAGIC Pure ETL: this notebook only reshapes what the AI Gateway already captured; it does -# MAGIC not call the endpoint. (Traffic is generated by the serving lab; AI Gateway logs land -# MAGIC within ~1 hour, so in a live class the freshest calls may not appear immediately.) +# MAGIC +# MAGIC This notebook only reshapes what the AI Gateway already captured; it does not call the +# MAGIC endpoint. Traffic is generated by the serving lab, and AI Gateway logs can take up to +# MAGIC ~1 hour to land, so the most recent calls may not appear immediately. # COMMAND ---------- diff --git a/src/monitoring/notebooks/seed_drift.py b/src/monitoring/notebooks/seed_drift.py index 07098a7..5543954 100644 --- a/src/monitoring/notebooks/seed_drift.py +++ b/src/monitoring/notebooks/seed_drift.py @@ -4,22 +4,20 @@ # MAGIC # MAGIC **Session:** Monitoring & Retraining # MAGIC -# MAGIC A small, safe helper so you can *see* feature drift inside the workshop instead of -# MAGIC waiting days for it to happen naturally. The batch inference table is already fully -# MAGIC scored, so instead of running the model again this takes a sample of existing -# MAGIC predictions, inflates the amount-driven features, and stamps them with **now**, so the -# MAGIC most recent monitor window looks different from the previous one. +# MAGIC Demo helper to make feature drift visible during the session instead of waiting for it +# MAGIC to occur naturally. The batch inference table is already scored, so this samples +# MAGIC existing predictions, inflates the amount-driven features, and stamps them with the +# MAGIC current time, so the most recent monitor window differs from the previous one. # MAGIC -# MAGIC It writes to **your own** tables (the per-user `dev__fraud` schema), exactly like -# MAGIC batch inference, so every attendee can run it independently: -# MAGIC - appends the drifted batch to `dev._fraud.fraud_predictions` -# MAGIC - refreshes your `batch_monitor`, so `_drift_metrics` picks up the spike +# MAGIC It writes to the per-user `dev__fraud` schema, so each attendee runs it +# MAGIC independently: # MAGIC -# MAGIC After it finishes, re-run `monitoring.py` section 4 to see the `amount` drift, and run -# MAGIC `feature_drift_check.py` to watch it return `is_drift_violated = True`. +# MAGIC - appends the drifted batch to `dev._fraud.fraud_predictions`; +# MAGIC - refreshes `batch_monitor` so `_drift_metrics` reflects the change. # MAGIC -# MAGIC Run **after** you have run batch inference at least once (so there is a normal baseline -# MAGIC in the earlier windows to drift away from) and the monitor exists. +# MAGIC Prerequisites: run batch inference at least once (to establish a baseline) and ensure +# MAGIC the monitor exists. Afterwards, re-run `monitoring.py` section 4 to view the `amount` +# MAGIC drift and `feature_drift_check.py` to confirm `is_drift_violated = True`. # MAGIC # MAGIC Docs: [Batch inference with Feature Engineering](https://learn.microsoft.com/azure/databricks/machine-learning/feature-store/score-batch) diff --git a/src/training/notebooks/training.py b/src/training/notebooks/training.py index ebe0c43..7c37137 100644 --- a/src/training/notebooks/training.py +++ b/src/training/notebooks/training.py @@ -90,11 +90,16 @@ # MAGIC %md # MAGIC ## Connect the model to its deployment job +# MAGIC # MAGIC If the training job passed a `deployment_job_id`, connect it to the registered model -# MAGIC **before** registering a version, so registering the new version auto-triggers the -# MAGIC governed evaluate/approve/deploy pipeline (even for the first version). -# MAGIC `create_registered_model` makes the empty model on first run; later runs just -# MAGIC `update_registered_model`. Skipped when run interactively without a job id. +# MAGIC **before** registering a version: +# MAGIC +# MAGIC - registering the new version then auto-triggers the governed evaluate/approve/deploy +# MAGIC pipeline, even for the first version; +# MAGIC - `create_registered_model` makes the empty model on the first run, later runs just +# MAGIC `update_registered_model`; +# MAGIC - skipped when run interactively without a job id. +# MAGIC # MAGIC See [MLflow deployment jobs](https://learn.microsoft.com/azure/databricks/mlflow/deployment-job). # COMMAND ---------- @@ -117,11 +122,12 @@ # MAGIC %md # MAGIC ## 1. Assemble the training set -# MAGIC The spine is the raw transaction: keys (`card_id`, `client_id`), the request fields -# MAGIC (`amount`, `transaction_hour`, `mcc`, `use_chip`) and the label. The Feature Store -# MAGIC looks up the card/client entity features and computes the on-demand features with the -# MAGIC UC functions. `create_training_set` records this so serving reproduces the same -# MAGIC feature pipeline. +# MAGIC +# MAGIC - The spine is the raw transaction: keys (`card_id`, `client_id`), the request fields +# MAGIC (`amount`, `transaction_hour`, `mcc`, `use_chip`), and the label. +# MAGIC - The Feature Store looks up the card/client entity features and computes the on-demand +# MAGIC features with the UC functions. +# MAGIC - `create_training_set` records this spec, so serving reproduces the same feature pipeline. # COMMAND ---------- @@ -204,15 +210,20 @@ # COMMAND ---------- # MAGIC %md -# MAGIC ## 2. Train, expose the imbalance trap, then fix it -# MAGIC Fraud is very rare (well under 1% of rows), so a model can score ~99.8% accuracy by -# MAGIC always predicting "not fraud" and catch zero fraud. We first train a naive random -# MAGIC forest to see that trap, then fix it by balancing the training data (keep every fraud -# MAGIC row, down-sample non-fraud to match) so the model learns what fraud looks like. We -# MAGIC still evaluate on the untouched, imbalanced test set so metrics reflect reality. -# MAGIC `fe.log_model` packages the fixed model with its feature metadata, infers a signature -# MAGIC and input example (so it can be served safely), and registers it to Unity Catalog in -# MAGIC one step (no separate `register_model` call). +# MAGIC ## 2. Train the model and address class imbalance +# MAGIC +# MAGIC Fraud accounts for well under 1% of rows, so a model can reach roughly 99.8% accuracy +# MAGIC by always predicting "not fraud" while detecting none. The workflow: +# MAGIC +# MAGIC - Train a baseline random forest on the raw, imbalanced data to establish the problem. +# MAGIC - Retrain on balanced data (retain every fraud row, down-sample non-fraud to match) so +# MAGIC the model learns the minority class. +# MAGIC - Evaluate on the original, imbalanced test set so the reported metrics reflect +# MAGIC production conditions. +# MAGIC +# MAGIC `fe.log_model` packages the model with its feature metadata, infers a signature and +# MAGIC input example, and registers it to Unity Catalog in a single step (no separate +# MAGIC `register_model` call). # COMMAND ---------- @@ -313,12 +324,16 @@ def predict(self, context, model_input): # COMMAND ---------- # MAGIC %md -# MAGIC ### Confusion matrices: the imbalance trap, side by side -# MAGIC Read the **fraud row** (bottom): the naive model dumps almost everything into -# MAGIC "predicted legit" (false negatives) and catches ~no fraud, while the balanced model -# MAGIC finally fills the **true-positive** cell, at the cost of more false positives -# MAGIC (top-right). Same untouched, imbalanced test set. The figure is also logged to the -# MAGIC `rf_balanced` MLflow run so it sits with the model version. +# MAGIC ### Confusion matrix comparison +# MAGIC +# MAGIC Compare the two models on the fraud row (bottom): +# MAGIC +# MAGIC - The baseline model assigns almost all fraud to "predicted legit" (false negatives) +# MAGIC and detects little fraud. +# MAGIC - The balanced model recovers true positives at the cost of more false positives. +# MAGIC +# MAGIC Both use the same original, imbalanced test set. The figure is logged to the +# MAGIC `rf_balanced` MLflow run so it stays with the model version. # COMMAND ---------- @@ -344,29 +359,30 @@ def predict(self, context, model_input): # COMMAND ---------- # MAGIC %md -# MAGIC ### But look at precision: the recall fix isn't free -# MAGIC The balanced model now catches most fraud (high recall), but its precision is very -# MAGIC low: most transactions it flags are actually legitimate. That's the precision/recall -# MAGIC trade-off. Balancing the data pushed the model to flag "fraud" far more often, -# MAGIC catching real fraud and drowning in false alarms. +# MAGIC ### Precision-recall trade-off +# MAGIC +# MAGIC The balanced model achieves high recall but low precision: most flagged transactions +# MAGIC are legitimate. Balancing the data makes the model predict "fraud" far more often, +# MAGIC which recovers real fraud but raises the false-positive rate. # MAGIC -# MAGIC In production you'd tune the decision threshold (score > 0.8 instead of 0.5) to trade -# MAGIC recall back for precision, and pick the operating point from the business cost of a -# MAGIC missed fraud vs. a false alarm, often measured as precision@k or PR-AUC rather than a -# MAGIC single accuracy/recall number. That's a business decision, not a modelling one, which -# MAGIC is why the promotion gate (next section) should encode the metric the business cares -# MAGIC about. +# MAGIC In production the decision threshold is tuned (for example, score > 0.8 rather than +# MAGIC 0.5) to trade recall for precision. The operating point is chosen from the business +# MAGIC cost of a missed fraud versus a false alarm, and is typically assessed with precision@k +# MAGIC or PR-AUC rather than a single accuracy or recall figure. Because this is a business +# MAGIC decision, the promotion gate in the next section encodes the metric the business +# MAGIC requires. # COMMAND ---------- # MAGIC %md # MAGIC ## 3. Tag the new version as `challenger` (instructor-provided) -# MAGIC The training run registered a fresh model version. Point the `challenger` alias at it -# MAGIC for a clear before/after picture. Registering the version also auto-triggers the -# MAGIC deployment job, whose Evaluation task re-scores it on a fresh holdout and compares it -# MAGIC against the current `@champion` before it can be promoted. Downstream jobs always load -# MAGIC `@champion`, so promotion is just an alias reassignment: no code change and instant -# MAGIC rollback. +# MAGIC +# MAGIC The training run registered a new model version. Point the `challenger` alias at it to +# MAGIC give a clear before/after comparison. Registering the version also triggers the +# MAGIC deployment job, whose Evaluation task re-scores the version on a fresh holdout and +# MAGIC compares it against the current `@champion` before promotion. Downstream jobs always +# MAGIC load `@champion`, so promotion is an alias reassignment: no code change, and rollback +# MAGIC is immediate. # COMMAND ---------- @@ -382,12 +398,12 @@ def predict(self, context, model_input): # COMMAND ---------- # MAGIC %md -# MAGIC ## Train a challenger (try it) +# MAGIC ## Train a challenger # MAGIC The first model becomes `@champion` once the deployment job promotes it. To produce a # MAGIC challenger to compete against it, change the model and re-run this whole notebook. # MAGIC Each run registers a new version and moves `@challenger` to it: # MAGIC -# MAGIC - In **section 2**, tweak the hyper-parameters in `params`, for example: +# MAGIC - In **section 2**, adjust the hyper-parameters in `params`, for example: # MAGIC - bump `n_estimators` to `200`, or # MAGIC - add `max_depth=8` or `min_samples_leaf=5` to regularise, or # MAGIC - swap in a different estimator (e.g. `GradientBoostingClassifier`). From 464bec810cd2a9daeb2e75dcfef1559e1fe97b93 Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Mon, 13 Jul 2026 10:59:43 +0100 Subject: [PATCH 2/7] docs(notebooks): finish professional sweep of comments and prose Remove remaining first-person and conversational phrasing from markdown and code comments across all notebooks (keep the dev._fraud path placeholders and the student HINT lines). Regenerate src. --- .../notebooks/load_landing_to_bronze.py | 2 +- .../notebooks/batch_inference.py | 23 +++++++++---------- .../model_deployment/notebooks/approval.py | 4 ++-- .../model_deployment/notebooks/evaluation.py | 4 ++-- .../notebooks/query_endpoint.py | 2 +- .../notebooks/build_features.py | 21 ++++++++--------- .../notebooks/feature_drift_check.py | 16 ++++++------- .../notebooks/metric_violation_check.py | 6 ++--- solution/monitoring/notebooks/monitoring.py | 13 +++++------ .../notebooks/process_serving_logs.py | 15 ++++++------ solution/monitoring/notebooks/seed_drift.py | 10 ++++---- solution/training/notebooks/training.py | 23 +++++++++---------- .../notebooks/load_landing_to_bronze.py | 2 +- .../notebooks/batch_inference.py | 23 +++++++++---------- .../model_deployment/notebooks/approval.py | 4 ++-- .../model_deployment/notebooks/evaluation.py | 4 ++-- .../notebooks/query_endpoint.py | 2 +- .../notebooks/build_features.py | 21 ++++++++--------- .../notebooks/feature_drift_check.py | 16 ++++++------- .../notebooks/metric_violation_check.py | 6 ++--- src/monitoring/notebooks/monitoring.py | 13 +++++------ .../notebooks/process_serving_logs.py | 15 ++++++------ src/monitoring/notebooks/seed_drift.py | 10 ++++---- src/training/notebooks/training.py | 23 +++++++++---------- 24 files changed, 136 insertions(+), 142 deletions(-) diff --git a/solution/data_ingestion/notebooks/load_landing_to_bronze.py b/solution/data_ingestion/notebooks/load_landing_to_bronze.py index 8e4adf2..dcfd536 100644 --- a/solution/data_ingestion/notebooks/load_landing_to_bronze.py +++ b/solution/data_ingestion/notebooks/load_landing_to_bronze.py @@ -226,7 +226,7 @@ def read_csv_with_audit(path: str, schema: StructType): # COMMAND ---------- # Governance: table-level comments so the raw layer is self-documenting. Bronze keeps -# source fidelity, so we do not comment individual columns here (that happens in silver). +# source fidelity, so individual columns are not commented here; that happens in silver. bronze_table_comments = { transactions_fqn: "Bronze: raw card transactions ingested as-is from the landing volume, plus audit columns.", cards_fqn: "Bronze: raw card metadata ingested as-is from the landing volume, plus audit columns.", diff --git a/solution/deployment/batch_inference/notebooks/batch_inference.py b/solution/deployment/batch_inference/notebooks/batch_inference.py index 1f57798..e94e608 100644 --- a/solution/deployment/batch_inference/notebooks/batch_inference.py +++ b/solution/deployment/batch_inference/notebooks/batch_inference.py @@ -46,7 +46,7 @@ # The batch-inference job passes the full three-level model name; fall back for interactive runs. model_name = dbutils.widgets.get("model_name") or f"{catalog_name}.{user_schema}.fraud_detection" -# Read the transactions to score from the shared gold source; write predictions to your schema. +# Read the transactions to score from the shared gold source; write predictions to the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" predictions_table = f"{catalog_name}.{user_schema}.fraud_predictions" @@ -74,8 +74,8 @@ mlflow.set_registry_uri("databricks-uc") fe = FeatureEngineeringClient() -# Resolve the champion alias to a concrete version so we can stamp every scored row with -# the model version that produced it (the monitor's model_id_col groups metrics by this). +# Resolve the champion alias to a concrete version, to stamp every scored row with the model +# version that produced it (the monitor's model_id_col groups metrics by this). client = MlflowClient() champion = client.get_model_version_by_alias(model_name, model_alias) model_version = str(champion.version) @@ -88,15 +88,14 @@ # MAGIC In production this would be the newly-arrived, not-yet-scored transactions. Build the # MAGIC same request spine the model was trained on: the raw transaction keys and fields # MAGIC (`card_id`, `client_id`, `amount`, `transaction_hour`, `mcc`, `use_chip`). The Feature -# MAGIC Store looks up the card/client entity features and runs the on-demand functions, so we -# MAGIC pass exactly what a live caller would send. `amount` is cast to double to match the +# MAGIC Store looks up the card/client entity features and runs the on-demand functions, so +# MAGIC the input matches what a live caller would send. `amount` is cast to double to match the # MAGIC feature tables and the on-demand function signatures (no train/serve skew). # COMMAND ---------- # The request spine mirrors the training spine (raw transaction keys and fields, no label). -# Building it is plain Spark, so it is provided; scoring with the champion is the step you -# fill in below. +# Building it is plain Spark and is provided; scoring with the champion is the exercise below. to_score = ( spark.read.table(source_table) .select( @@ -141,8 +140,8 @@ # MAGIC Then append. # MAGIC # MAGIC > The label (`is_fraud`) is joined here for the workshop so quality metrics compute -# MAGIC > immediately. In the real world fraud is confirmed later (chargebacks), so you would -# MAGIC > backfill the label into this table when it arrives; drift is watched in the meantime. +# MAGIC > immediately. In the real world fraud is confirmed later (chargebacks), so the label +# MAGIC > would be backfilled into this table when it arrives; drift is tracked in the meantime. # COMMAND ---------- @@ -155,9 +154,9 @@ ) # fe.score_batch already returns the looked-up entity features (credit_score, credit_limit, -# yearly_income, current_age) plus the input `amount`, so we log those columns straight from -# `scored`. Logging them is what makes FEATURE (data) drift monitorable downstream. (Re-joining -# them from the source would duplicate the columns and make the reference ambiguous.) +# yearly_income, current_age) plus the input `amount`, so those columns are logged straight +# from `scored`. Logging them is what makes feature (data) drift monitorable downstream. +# (Re-joining them from the source would duplicate the columns and make the reference ambiguous.) scored_at = F.to_timestamp( F.date_sub(F.current_date(), (F.col("transaction_id") % F.lit(14)).cast("int")) diff --git a/solution/deployment/model_deployment/notebooks/approval.py b/solution/deployment/model_deployment/notebooks/approval.py index c99dc17..7c518af 100644 --- a/solution/deployment/model_deployment/notebooks/approval.py +++ b/solution/deployment/model_deployment/notebooks/approval.py @@ -44,8 +44,8 @@ client = MlflowClient(registry_uri="databricks-uc") -# In dev we auto-approve: set the tag so this run (and the UI) reflect an approved state, -# then pass. In staging/prod we require a human to have set the tag to "Approved". +# In dev, auto-approve: set the tag so this run (and the UI) reflect an approved state, then +# pass. In staging/prod, a human must have set the tag to "Approved". if auto_approve: client.set_model_version_tag(model_name, model_version, tag_name, "Approved") print(f"Auto-approved (dev): set {tag_name}=Approved on v{model_version}.") diff --git a/solution/deployment/model_deployment/notebooks/evaluation.py b/solution/deployment/model_deployment/notebooks/evaluation.py index d245921..ed6f564 100644 --- a/solution/deployment/model_deployment/notebooks/evaluation.py +++ b/solution/deployment/model_deployment/notebooks/evaluation.py @@ -52,8 +52,8 @@ # Independent evaluation: score the REGISTERED artifact on a fresh, labelled holdout rather # than trusting the metrics training self-reported. fe.score_batch replays the exact feature # lookups + on-demand functions recorded at training, and the served model returns a fraud -# probability, so we can compute a real ROC AUC. (In production this holdout would be a -# curated, leakage-controlled evaluation table; here we sample the shared gold source.) +# probability, so a real ROC AUC can be computed. In production this holdout would be a +# curated, leakage-controlled evaluation table; here it is sampled from the shared gold source. catalog_name = model_name.split(".")[0] gold_table = f"{catalog_name}.fraud_gold.transactions_enriched" diff --git a/solution/deployment/model_deployment/notebooks/query_endpoint.py b/solution/deployment/model_deployment/notebooks/query_endpoint.py index 544fbc9..819450b 100644 --- a/solution/deployment/model_deployment/notebooks/query_endpoint.py +++ b/solution/deployment/model_deployment/notebooks/query_endpoint.py @@ -125,4 +125,4 @@ def score_live(row): # MAGIC - Every call carried `client_request_id` = transaction id, the correlation key the # MAGIC AI Gateway logs so `process_serving_logs` can join labels for the online monitor. # MAGIC - AI Gateway delivery lags (up to ~1 hour), so these calls appear in the inference -# MAGIC table a little after you run this. +# MAGIC table shortly after this runs. diff --git a/solution/feature_engineering/notebooks/build_features.py b/solution/feature_engineering/notebooks/build_features.py index 38fcc6b..53b5081 100644 --- a/solution/feature_engineering/notebooks/build_features.py +++ b/solution/feature_engineering/notebooks/build_features.py @@ -5,7 +5,7 @@ # MAGIC **Session:** Model Development & Experimentation # MAGIC # MAGIC Real-time scoring has to work for a brand-new transaction that isn't in any table -# MAGIC yet, so we split features the way production systems do: +# MAGIC yet, so features are split the way production systems do: # MAGIC # MAGIC 1. **Entity features:** slow-changing card and client attributes, looked up by key at # MAGIC serving time. Published to the Unity Catalog Feature Store: @@ -41,7 +41,7 @@ _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) user_schema = f"dev_{_short}_fraud" -# Read from the shared gold source; write to your personal schema. +# Read from the shared gold source; write to the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" card_feature_table = f"{catalog_name}.{user_schema}.card_features" client_feature_table = f"{catalog_name}.{user_schema}.client_features" @@ -69,13 +69,12 @@ # COMMAND ---------- # One row per entity (deduplicated), keyed by card_id / client_id. This is plain Spark -# aggregation, so it is provided; publishing these to the Feature Store below is the step -# you fill in. -# For simplicity we take the card/client attributes from the single transactions_enriched -# table, picking one value per entity with F.first. In the real world we would read them -# from the entity dimension itself (the cards / users tables), because these attribute -# values can change over time, so a per-transaction snapshot is arbitrary and the dimension -# is the source of truth. +# aggregation and is provided; publishing these to the Feature Store below is the exercise. +# For simplicity the card/client attributes come from the single transactions_enriched +# table, taking one value per entity with F.first. In production they would come from the +# entity dimension itself (the cards / users tables), because these attribute values can +# change over time, so a per-transaction snapshot is arbitrary and the dimension is the +# source of truth. card_features = enriched.groupBy("card_id").agg( F.first("credit_limit", ignorenulls=True).cast("double").alias("credit_limit"), F.first("num_cards_issued", ignorenulls=True).cast("int").alias("num_cards_issued"), @@ -139,8 +138,8 @@ def publish(name, df, key, description): # COMMAND ---------- # On-demand feature functions are UC Python UDFs. Training and serving call the same -# functions, so there is no train/serve skew. Four of them are provided; you write the first -# one (ff_is_night) to learn the pattern. +# functions, so there is no train/serve skew. Four are provided; the first (ff_is_night) is +# the exercise. schema_fqn = f"{catalog_name}.{user_schema}" # TODO-BEGIN: register the ff_is_night on-demand feature function as a UC Python UDF diff --git a/solution/monitoring/notebooks/feature_drift_check.py b/solution/monitoring/notebooks/feature_drift_check.py index 6789ea0..99a9e52 100644 --- a/solution/monitoring/notebooks/feature_drift_check.py +++ b/solution/monitoring/notebooks/feature_drift_check.py @@ -26,7 +26,7 @@ # MAGIC - `drift_violation_threshold`: drift is a "higher is worse" metric, so a value **above** # MAGIC this counts as a violation. # MAGIC - `num_evaluation_windows` / `num_violation_windows`: how many recent windows to look at, -# MAGIC and how many must be in violation before we act. +# MAGIC and how many must be in violation before action is taken. # MAGIC # MAGIC Note on acting on feature drift: retraining only helps once fresh labels arrive, so in # MAGIC production feature drift typically raises an alert first, and the label-based quality @@ -60,7 +60,7 @@ # MAGIC ## The violation query # MAGIC # MAGIC Lakehouse Monitoring writes a `_drift_metrics` Delta table with one row per -# MAGIC column, per time `window`, per drift comparison. For each monitored feature we look at +# MAGIC column, per time `window`, per drift comparison. For each monitored feature, it checks # MAGIC the whole-population rows and ask: of the last `num_evaluation_windows` windows, did at # MAGIC least `num_violation_windows` have `drift_metric` **above** `drift_violation_threshold`, # MAGIC and is the most recent window also above it? @@ -68,11 +68,11 @@ # MAGIC The row filters mean: # MAGIC - `column_name` is one of the monitored features (not `:table`), # MAGIC - `slice_key IS NULL` -> the whole population, not a single data slice, -# MAGIC - `drift_type = "CONSECUTIVE"` -> window-over-window drift (we have no baseline table; a +# MAGIC - `drift_type = "CONSECUTIVE"` -> window-over-window drift (there is no baseline table; a # MAGIC baseline-vs-training comparison would use `drift_type = "BASELINE"`). # MAGIC # MAGIC The table can carry more than one row per window (one per `model_version` plus a `*` -# MAGIC aggregate), so we take the max metric per window to get one value each. +# MAGIC aggregate), so the max metric per window is used to get one value each. # MAGIC # MAGIC `population_stability_index` (PSI) measures how far a feature's distribution has moved # MAGIC between the previous window and the current one: both windows are cut into the same @@ -80,10 +80,10 @@ # MAGIC two distributions match and grows as they pull apart, so "higher is worse". A common # MAGIC reading is >0.1 a moderate shift and >0.25 a significant shift. # MAGIC -# MAGIC We threshold PSI (not `js_distance`) because Lakehouse Monitoring leaves `js_distance` +# MAGIC PSI is thresholded (not `js_distance`) because Lakehouse Monitoring leaves `js_distance` # MAGIC null for numeric columns and only fills it for categoricals, so PSI is the drift knob # MAGIC that actually fires on numeric features like `amount`. If **any** monitored feature is -# MAGIC in violation, we flag `is_drift_violated`. +# MAGIC in violation, `is_drift_violated` is set. # COMMAND ---------- @@ -129,8 +129,8 @@ if not recent: continue # Violation: at least num_violation_windows above the threshold AND the most recent window - # is also above it (so we don't act on a spike that has already settled). Drift is "higher - # is worse", so we compare with >. + # is also above it (so a spike that has already settled is ignored). Drift is "higher is + # worse", so the comparison uses >. windows_in_violation = sum(1 for row in recent if row["metric_value"] > drift_violation_threshold) latest_in_violation = recent[0]["metric_value"] > drift_violation_threshold if windows_in_violation >= num_violation_windows and latest_in_violation: diff --git a/solution/monitoring/notebooks/metric_violation_check.py b/solution/monitoring/notebooks/metric_violation_check.py index a4cdafd..5f36d5a 100644 --- a/solution/monitoring/notebooks/metric_violation_check.py +++ b/solution/monitoring/notebooks/metric_violation_check.py @@ -45,7 +45,7 @@ # MAGIC Lakehouse Monitoring writes a `
_profile_metrics` Delta table. For an # MAGIC InferenceLog monitor with a `label_col`, that table holds model-quality metrics # MAGIC (accuracy, precision, recall, f1, ...) computed per time `window`, per -# MAGIC `model_version`, and per `slice`. We look at the whole-table rows for the real model +# MAGIC `model_version`, and per `slice`. It reads the whole-table rows for the real model # MAGIC versions and ask: of the last `num_evaluation_windows` windows, did at least # MAGIC `num_violation_windows` fall **below** the quality threshold, and is the most recent # MAGIC window also below it? @@ -107,8 +107,8 @@ ) # Violation: at least num_violation_windows of those windows are below the threshold and the -# most recent window is also below it (so we don't retrain on a dip that has already -# recovered). Fraud quality is "higher is better"; for a drift/error metric flip < to >. +# most recent window is also below it (so a recovered dip does not trigger retraining). +# Fraud quality is "higher is better"; for a drift/error metric flip < to >. windows_in_violation = sum(1 for row in recent_metrics if row["metric_value"] < metric_violation_threshold) latest_in_violation = bool(recent_metrics) and recent_metrics[0]["metric_value"] < metric_violation_threshold is_metric_violated = windows_in_violation >= num_violation_windows and latest_in_violation diff --git a/solution/monitoring/notebooks/monitoring.py b/solution/monitoring/notebooks/monitoring.py index da98776..581584f 100644 --- a/solution/monitoring/notebooks/monitoring.py +++ b/solution/monitoring/notebooks/monitoring.py @@ -55,9 +55,9 @@ # COMMAND ---------- -# A Lakehouse Monitor recomputes its profile and drift metrics on refresh. The bundle -# created the monitors on a schedule; here we refresh on demand so the metric tables reflect -# the rows we just scored. The client and the guard loop are provided. +# A Lakehouse Monitor recomputes its profile and drift metrics on refresh. The monitors run +# on a schedule; here they are refreshed on demand so the metric tables reflect the rows just +# scored. The client and the guard loop are provided. from databricks.sdk import WorkspaceClient w = WorkspaceClient() @@ -87,10 +87,9 @@ # COMMAND ---------- -# The monitor writes a _profile_metrics table with model-quality metrics per time window. -# The point is that monitoring produces a queryable metrics table you can trend and alert on. -# Classification metrics are STRUCTs, so we dereference the fields (f1_score.macro etc.) with -# F.col. +# The monitor writes a _profile_metrics table with model-quality metrics per time window, +# a queryable table for trending and alerting. Classification metrics are STRUCTs, so the +# fields (f1_score.macro etc.) are dereferenced with F.col. from pyspark.sql import functions as F profile_metrics = f"{batch_table}_profile_metrics" diff --git a/solution/monitoring/notebooks/process_serving_logs.py b/solution/monitoring/notebooks/process_serving_logs.py index f01413e..28aa3f6 100644 --- a/solution/monitoring/notebooks/process_serving_logs.py +++ b/solution/monitoring/notebooks/process_serving_logs.py @@ -69,7 +69,7 @@ # Unpack the AI Gateway payload table into one row per prediction. This is heavy Spark # plumbing (guarding the delivery lag, from_json, explode), so it is provided; the label -# join and write at the end are the step you fill in. +# join and write at the end are the exercise. from pyspark.sql import functions as F from pyspark.sql import types as T @@ -166,17 +166,18 @@ # MAGIC %md # MAGIC ## 3. Best-effort label join and persist -# MAGIC Attach the confirmed label where we already have it (null otherwise, online labels -# MAGIC lag), then overwrite the online inference table. Same schema as the batch -# MAGIC `fraud_predictions` table, so both monitors line up. +# MAGIC +# MAGIC Attach the confirmed label where available (null otherwise, online labels lag), then +# MAGIC overwrite the online inference table. Same schema as the batch `fraud_predictions` +# MAGIC table, so both monitors line up. # COMMAND ---------- # Attach the confirmed label AND the same feature columns as the batch fraud_predictions # table, so the two monitors are symmetric and track drift on every model feature. The serving -# payload only logs raw inputs, so we resolve the features the model used, by transaction_id, +# payload only logs raw inputs, so the features the model used are resolved by transaction_id # from the gold source: entity features are columns, and the on-demand features are computed -# with the SAME UC functions the model uses (no train/serve skew). The join is provided. +# with the same UC functions the model uses (no train/serve skew). The join is provided. schema_fqn = f"{catalog_name}.{user_schema}" gold_raw = spark.read.table(gold_table).select( "transaction_id", @@ -237,7 +238,7 @@ # MAGIC %md # MAGIC ## Recap -# MAGIC - The AI Gateway captured online traffic automatically; we only reshaped it. +# MAGIC - The AI Gateway captured online traffic automatically; this notebook only reshaped it. # MAGIC - `transaction_id` came from the `client_request_id` correlation id the caller set: # MAGIC the clean way to join model outputs back to confirmed-fraud labels later. # MAGIC - `fraud_serving_inference` now matches the batch `fraud_predictions` schema, so the diff --git a/solution/monitoring/notebooks/seed_drift.py b/solution/monitoring/notebooks/seed_drift.py index 5543954..40ee50c 100644 --- a/solution/monitoring/notebooks/seed_drift.py +++ b/solution/monitoring/notebooks/seed_drift.py @@ -54,16 +54,16 @@ # MAGIC %md # MAGIC ## 1. Build a drifted batch from already-scored rows -# MAGIC The batch inference table is already fully scored with every feature column, so we do -# MAGIC not re-run the model. Take a sample of existing predictions, inflate the amount-driven +# MAGIC The batch inference table is already fully scored with every feature column, so the +# MAGIC model is not re-run. Take a sample of existing predictions, inflate the amount-driven # MAGIC features (`amount` and the two ratios that scale with it), and stamp them with the -# MAGIC current time so they land in the newest monitor window. This is the "world changed" we -# MAGIC want the monitor to catch. +# MAGIC current time so they land in the newest monitor window. This simulates a change in the +# MAGIC input distribution for the monitor to catch. # COMMAND ---------- # Build the drifted sample by selecting from the same table (keeps the schema identical, no -# mergeSchema needed) and scaling the amount-driven features. We stage it to a separate table +# mergeSchema needed) and scaling the amount-driven features. Stage it to a separate table # first so the append does not read from the table it is writing to; serverless does not allow # cache/checkpoint, and a staging table is the clean way to break that dependency. stage_table = f"{catalog_name}.{user_schema}._seed_drift_stage" diff --git a/solution/training/notebooks/training.py b/solution/training/notebooks/training.py index b90a245..5dd1c97 100644 --- a/solution/training/notebooks/training.py +++ b/solution/training/notebooks/training.py @@ -41,7 +41,7 @@ _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) user_schema = f"dev_{_short}_fraud" -# Labels come from the shared gold table; features come from your personal schema. +# Labels come from the shared gold table; features come from the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" card_feature_table = f"{catalog_name}.{user_schema}.card_features" client_feature_table = f"{catalog_name}.{user_schema}.client_features" @@ -155,9 +155,9 @@ .limit(SAMPLE_ROWS) ) -# The on-demand feature functions are provided (they mirror the UC UDFs you registered in -# the feature engineering notebook). You add the two entity FeatureLookups and assemble the -# training set. +# The on-demand feature functions are provided (they mirror the UC UDFs registered in the +# feature engineering notebook). Adding the two entity FeatureLookups and assembling the +# training set is the exercise. feature_functions = [ FeatureFunction( udf_name=f"{schema_fqn}.ff_is_night", @@ -285,13 +285,13 @@ def predict(self, context, model_input): naive_metrics = evaluate(naive, X_test, y_test) mlflow.log_params({**naive_params, "training_data": "imbalanced"}) mlflow.log_metrics(naive_metrics) - print("rf_imbalanced:", naive_metrics) # high accuracy, but look at recall + print("rf_imbalanced:", naive_metrics) # high accuracy, low recall -# Fixed random forest: the model we track and register. +# Fixed random forest: the model that is tracked and registered. with mlflow.start_run(run_name="rf_balanced") as run: # Balance the training rows: keep every fraud row and randomly sample an equal number of - # non-fraud rows (provided: this is a data-science fix, not the MLOps concept). We still - # evaluate on the untouched, imbalanced X_test/y_test so metrics reflect reality. + # non-fraud rows (provided: this is a data-science fix, not the MLOps concept). Evaluation + # still uses the untouched, imbalanced X_test/y_test so metrics reflect reality. fraud_idx = y_train[y_train == 1].index legit_idx = y_train[y_train == 0].sample(n=len(fraud_idx), random_state=42).index bal_idx = fraud_idx.union(legit_idx) @@ -311,8 +311,8 @@ def predict(self, context, model_input): mlflow.log_metrics(metrics) # Package with feature metadata + an inferred signature/input example (required for - # safe serving) and register to Unity Catalog in one call. We wrap the classifier so the - # served model returns a fraud PROBABILITY (score), not a hard class. + # safe serving) and register to Unity Catalog in one call. The classifier is wrapped so + # the served model returns a fraud probability (score), not a hard class. # TODO-BEGIN: log and register the model to Unity Catalog with its feature metadata # HINT: fe.log_model packages the model with the training_set's feature lookups (so serving # HINT: reproduces the same features) and registers it to UC in a single call. @@ -415,8 +415,7 @@ def predict(self, context, model_input): client = MlflowClient() # Unity Catalog's search_model_versions only supports a `name='...'` filter (no run_id -# filtering), so fetch this model's versions and take the highest: the one we just -# registered above. +# filtering), so fetch this model's versions and take the highest: the one registered above. versions = client.search_model_versions(f"name='{model_name}'") new_version = max(int(mv.version) for mv in versions) client.set_registered_model_alias(model_name, "challenger", new_version) diff --git a/src/data_ingestion/notebooks/load_landing_to_bronze.py b/src/data_ingestion/notebooks/load_landing_to_bronze.py index 8e4adf2..dcfd536 100644 --- a/src/data_ingestion/notebooks/load_landing_to_bronze.py +++ b/src/data_ingestion/notebooks/load_landing_to_bronze.py @@ -226,7 +226,7 @@ def read_csv_with_audit(path: str, schema: StructType): # COMMAND ---------- # Governance: table-level comments so the raw layer is self-documenting. Bronze keeps -# source fidelity, so we do not comment individual columns here (that happens in silver). +# source fidelity, so individual columns are not commented here; that happens in silver. bronze_table_comments = { transactions_fqn: "Bronze: raw card transactions ingested as-is from the landing volume, plus audit columns.", cards_fqn: "Bronze: raw card metadata ingested as-is from the landing volume, plus audit columns.", diff --git a/src/deployment/batch_inference/notebooks/batch_inference.py b/src/deployment/batch_inference/notebooks/batch_inference.py index dc66a94..70efb23 100644 --- a/src/deployment/batch_inference/notebooks/batch_inference.py +++ b/src/deployment/batch_inference/notebooks/batch_inference.py @@ -46,7 +46,7 @@ # The batch-inference job passes the full three-level model name; fall back for interactive runs. model_name = dbutils.widgets.get("model_name") or f"{catalog_name}.{user_schema}.fraud_detection" -# Read the transactions to score from the shared gold source; write predictions to your schema. +# Read the transactions to score from the shared gold source; write predictions to the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" predictions_table = f"{catalog_name}.{user_schema}.fraud_predictions" @@ -74,8 +74,8 @@ mlflow.set_registry_uri("databricks-uc") fe = FeatureEngineeringClient() -# Resolve the champion alias to a concrete version so we can stamp every scored row with -# the model version that produced it (the monitor's model_id_col groups metrics by this). +# Resolve the champion alias to a concrete version, to stamp every scored row with the model +# version that produced it (the monitor's model_id_col groups metrics by this). client = MlflowClient() champion = client.get_model_version_by_alias(model_name, model_alias) model_version = str(champion.version) @@ -88,15 +88,14 @@ # MAGIC In production this would be the newly-arrived, not-yet-scored transactions. Build the # MAGIC same request spine the model was trained on: the raw transaction keys and fields # MAGIC (`card_id`, `client_id`, `amount`, `transaction_hour`, `mcc`, `use_chip`). The Feature -# MAGIC Store looks up the card/client entity features and runs the on-demand functions, so we -# MAGIC pass exactly what a live caller would send. `amount` is cast to double to match the +# MAGIC Store looks up the card/client entity features and runs the on-demand functions, so +# MAGIC the input matches what a live caller would send. `amount` is cast to double to match the # MAGIC feature tables and the on-demand function signatures (no train/serve skew). # COMMAND ---------- # The request spine mirrors the training spine (raw transaction keys and fields, no label). -# Building it is plain Spark, so it is provided; scoring with the champion is the step you -# fill in below. +# Building it is plain Spark and is provided; scoring with the champion is the exercise below. to_score = ( spark.read.table(source_table) .select( @@ -137,8 +136,8 @@ # MAGIC Then append. # MAGIC # MAGIC > The label (`is_fraud`) is joined here for the workshop so quality metrics compute -# MAGIC > immediately. In the real world fraud is confirmed later (chargebacks), so you would -# MAGIC > backfill the label into this table when it arrives; drift is watched in the meantime. +# MAGIC > immediately. In the real world fraud is confirmed later (chargebacks), so the label +# MAGIC > would be backfilled into this table when it arrives; drift is tracked in the meantime. # COMMAND ---------- @@ -151,9 +150,9 @@ ) # fe.score_batch already returns the looked-up entity features (credit_score, credit_limit, -# yearly_income, current_age) plus the input `amount`, so we log those columns straight from -# `scored`. Logging them is what makes FEATURE (data) drift monitorable downstream. (Re-joining -# them from the source would duplicate the columns and make the reference ambiguous.) +# yearly_income, current_age) plus the input `amount`, so those columns are logged straight +# from `scored`. Logging them is what makes feature (data) drift monitorable downstream. +# (Re-joining them from the source would duplicate the columns and make the reference ambiguous.) scored_at = F.to_timestamp( F.date_sub(F.current_date(), (F.col("transaction_id") % F.lit(14)).cast("int")) diff --git a/src/deployment/model_deployment/notebooks/approval.py b/src/deployment/model_deployment/notebooks/approval.py index c99dc17..7c518af 100644 --- a/src/deployment/model_deployment/notebooks/approval.py +++ b/src/deployment/model_deployment/notebooks/approval.py @@ -44,8 +44,8 @@ client = MlflowClient(registry_uri="databricks-uc") -# In dev we auto-approve: set the tag so this run (and the UI) reflect an approved state, -# then pass. In staging/prod we require a human to have set the tag to "Approved". +# In dev, auto-approve: set the tag so this run (and the UI) reflect an approved state, then +# pass. In staging/prod, a human must have set the tag to "Approved". if auto_approve: client.set_model_version_tag(model_name, model_version, tag_name, "Approved") print(f"Auto-approved (dev): set {tag_name}=Approved on v{model_version}.") diff --git a/src/deployment/model_deployment/notebooks/evaluation.py b/src/deployment/model_deployment/notebooks/evaluation.py index d245921..ed6f564 100644 --- a/src/deployment/model_deployment/notebooks/evaluation.py +++ b/src/deployment/model_deployment/notebooks/evaluation.py @@ -52,8 +52,8 @@ # Independent evaluation: score the REGISTERED artifact on a fresh, labelled holdout rather # than trusting the metrics training self-reported. fe.score_batch replays the exact feature # lookups + on-demand functions recorded at training, and the served model returns a fraud -# probability, so we can compute a real ROC AUC. (In production this holdout would be a -# curated, leakage-controlled evaluation table; here we sample the shared gold source.) +# probability, so a real ROC AUC can be computed. In production this holdout would be a +# curated, leakage-controlled evaluation table; here it is sampled from the shared gold source. catalog_name = model_name.split(".")[0] gold_table = f"{catalog_name}.fraud_gold.transactions_enriched" diff --git a/src/deployment/model_deployment/notebooks/query_endpoint.py b/src/deployment/model_deployment/notebooks/query_endpoint.py index 544fbc9..819450b 100644 --- a/src/deployment/model_deployment/notebooks/query_endpoint.py +++ b/src/deployment/model_deployment/notebooks/query_endpoint.py @@ -125,4 +125,4 @@ def score_live(row): # MAGIC - Every call carried `client_request_id` = transaction id, the correlation key the # MAGIC AI Gateway logs so `process_serving_logs` can join labels for the online monitor. # MAGIC - AI Gateway delivery lags (up to ~1 hour), so these calls appear in the inference -# MAGIC table a little after you run this. +# MAGIC table shortly after this runs. diff --git a/src/feature_engineering/notebooks/build_features.py b/src/feature_engineering/notebooks/build_features.py index c2537b8..5e0dcec 100644 --- a/src/feature_engineering/notebooks/build_features.py +++ b/src/feature_engineering/notebooks/build_features.py @@ -5,7 +5,7 @@ # MAGIC **Session:** Model Development & Experimentation # MAGIC # MAGIC Real-time scoring has to work for a brand-new transaction that isn't in any table -# MAGIC yet, so we split features the way production systems do: +# MAGIC yet, so features are split the way production systems do: # MAGIC # MAGIC 1. **Entity features:** slow-changing card and client attributes, looked up by key at # MAGIC serving time. Published to the Unity Catalog Feature Store: @@ -41,7 +41,7 @@ _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) user_schema = f"dev_{_short}_fraud" -# Read from the shared gold source; write to your personal schema. +# Read from the shared gold source; write to the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" card_feature_table = f"{catalog_name}.{user_schema}.card_features" client_feature_table = f"{catalog_name}.{user_schema}.client_features" @@ -69,13 +69,12 @@ # COMMAND ---------- # One row per entity (deduplicated), keyed by card_id / client_id. This is plain Spark -# aggregation, so it is provided; publishing these to the Feature Store below is the step -# you fill in. -# For simplicity we take the card/client attributes from the single transactions_enriched -# table, picking one value per entity with F.first. In the real world we would read them -# from the entity dimension itself (the cards / users tables), because these attribute -# values can change over time, so a per-transaction snapshot is arbitrary and the dimension -# is the source of truth. +# aggregation and is provided; publishing these to the Feature Store below is the exercise. +# For simplicity the card/client attributes come from the single transactions_enriched +# table, taking one value per entity with F.first. In production they would come from the +# entity dimension itself (the cards / users tables), because these attribute values can +# change over time, so a per-transaction snapshot is arbitrary and the dimension is the +# source of truth. card_features = enriched.groupBy("card_id").agg( F.first("credit_limit", ignorenulls=True).cast("double").alias("credit_limit"), F.first("num_cards_issued", ignorenulls=True).cast("int").alias("num_cards_issued"), @@ -124,8 +123,8 @@ # COMMAND ---------- # On-demand feature functions are UC Python UDFs. Training and serving call the same -# functions, so there is no train/serve skew. Four of them are provided; you write the first -# one (ff_is_night) to learn the pattern. +# functions, so there is no train/serve skew. Four are provided; the first (ff_is_night) is +# the exercise. schema_fqn = f"{catalog_name}.{user_schema}" # TODO: register the ff_is_night on-demand feature function as a UC Python UDF diff --git a/src/monitoring/notebooks/feature_drift_check.py b/src/monitoring/notebooks/feature_drift_check.py index 6789ea0..99a9e52 100644 --- a/src/monitoring/notebooks/feature_drift_check.py +++ b/src/monitoring/notebooks/feature_drift_check.py @@ -26,7 +26,7 @@ # MAGIC - `drift_violation_threshold`: drift is a "higher is worse" metric, so a value **above** # MAGIC this counts as a violation. # MAGIC - `num_evaluation_windows` / `num_violation_windows`: how many recent windows to look at, -# MAGIC and how many must be in violation before we act. +# MAGIC and how many must be in violation before action is taken. # MAGIC # MAGIC Note on acting on feature drift: retraining only helps once fresh labels arrive, so in # MAGIC production feature drift typically raises an alert first, and the label-based quality @@ -60,7 +60,7 @@ # MAGIC ## The violation query # MAGIC # MAGIC Lakehouse Monitoring writes a `
_drift_metrics` Delta table with one row per -# MAGIC column, per time `window`, per drift comparison. For each monitored feature we look at +# MAGIC column, per time `window`, per drift comparison. For each monitored feature, it checks # MAGIC the whole-population rows and ask: of the last `num_evaluation_windows` windows, did at # MAGIC least `num_violation_windows` have `drift_metric` **above** `drift_violation_threshold`, # MAGIC and is the most recent window also above it? @@ -68,11 +68,11 @@ # MAGIC The row filters mean: # MAGIC - `column_name` is one of the monitored features (not `:table`), # MAGIC - `slice_key IS NULL` -> the whole population, not a single data slice, -# MAGIC - `drift_type = "CONSECUTIVE"` -> window-over-window drift (we have no baseline table; a +# MAGIC - `drift_type = "CONSECUTIVE"` -> window-over-window drift (there is no baseline table; a # MAGIC baseline-vs-training comparison would use `drift_type = "BASELINE"`). # MAGIC # MAGIC The table can carry more than one row per window (one per `model_version` plus a `*` -# MAGIC aggregate), so we take the max metric per window to get one value each. +# MAGIC aggregate), so the max metric per window is used to get one value each. # MAGIC # MAGIC `population_stability_index` (PSI) measures how far a feature's distribution has moved # MAGIC between the previous window and the current one: both windows are cut into the same @@ -80,10 +80,10 @@ # MAGIC two distributions match and grows as they pull apart, so "higher is worse". A common # MAGIC reading is >0.1 a moderate shift and >0.25 a significant shift. # MAGIC -# MAGIC We threshold PSI (not `js_distance`) because Lakehouse Monitoring leaves `js_distance` +# MAGIC PSI is thresholded (not `js_distance`) because Lakehouse Monitoring leaves `js_distance` # MAGIC null for numeric columns and only fills it for categoricals, so PSI is the drift knob # MAGIC that actually fires on numeric features like `amount`. If **any** monitored feature is -# MAGIC in violation, we flag `is_drift_violated`. +# MAGIC in violation, `is_drift_violated` is set. # COMMAND ---------- @@ -129,8 +129,8 @@ if not recent: continue # Violation: at least num_violation_windows above the threshold AND the most recent window - # is also above it (so we don't act on a spike that has already settled). Drift is "higher - # is worse", so we compare with >. + # is also above it (so a spike that has already settled is ignored). Drift is "higher is + # worse", so the comparison uses >. windows_in_violation = sum(1 for row in recent if row["metric_value"] > drift_violation_threshold) latest_in_violation = recent[0]["metric_value"] > drift_violation_threshold if windows_in_violation >= num_violation_windows and latest_in_violation: diff --git a/src/monitoring/notebooks/metric_violation_check.py b/src/monitoring/notebooks/metric_violation_check.py index a4cdafd..5f36d5a 100644 --- a/src/monitoring/notebooks/metric_violation_check.py +++ b/src/monitoring/notebooks/metric_violation_check.py @@ -45,7 +45,7 @@ # MAGIC Lakehouse Monitoring writes a `
_profile_metrics` Delta table. For an # MAGIC InferenceLog monitor with a `label_col`, that table holds model-quality metrics # MAGIC (accuracy, precision, recall, f1, ...) computed per time `window`, per -# MAGIC `model_version`, and per `slice`. We look at the whole-table rows for the real model +# MAGIC `model_version`, and per `slice`. It reads the whole-table rows for the real model # MAGIC versions and ask: of the last `num_evaluation_windows` windows, did at least # MAGIC `num_violation_windows` fall **below** the quality threshold, and is the most recent # MAGIC window also below it? @@ -107,8 +107,8 @@ ) # Violation: at least num_violation_windows of those windows are below the threshold and the -# most recent window is also below it (so we don't retrain on a dip that has already -# recovered). Fraud quality is "higher is better"; for a drift/error metric flip < to >. +# most recent window is also below it (so a recovered dip does not trigger retraining). +# Fraud quality is "higher is better"; for a drift/error metric flip < to >. windows_in_violation = sum(1 for row in recent_metrics if row["metric_value"] < metric_violation_threshold) latest_in_violation = bool(recent_metrics) and recent_metrics[0]["metric_value"] < metric_violation_threshold is_metric_violated = windows_in_violation >= num_violation_windows and latest_in_violation diff --git a/src/monitoring/notebooks/monitoring.py b/src/monitoring/notebooks/monitoring.py index 7760926..15ca8ed 100644 --- a/src/monitoring/notebooks/monitoring.py +++ b/src/monitoring/notebooks/monitoring.py @@ -55,9 +55,9 @@ # COMMAND ---------- -# A Lakehouse Monitor recomputes its profile and drift metrics on refresh. The bundle -# created the monitors on a schedule; here we refresh on demand so the metric tables reflect -# the rows we just scored. The client and the guard loop are provided. +# A Lakehouse Monitor recomputes its profile and drift metrics on refresh. The monitors run +# on a schedule; here they are refreshed on demand so the metric tables reflect the rows just +# scored. The client and the guard loop are provided. from databricks.sdk import WorkspaceClient w = WorkspaceClient() @@ -86,10 +86,9 @@ # COMMAND ---------- -# The monitor writes a _profile_metrics table with model-quality metrics per time window. -# The point is that monitoring produces a queryable metrics table you can trend and alert on. -# Classification metrics are STRUCTs, so we dereference the fields (f1_score.macro etc.) with -# F.col. +# The monitor writes a _profile_metrics table with model-quality metrics per time window, +# a queryable table for trending and alerting. Classification metrics are STRUCTs, so the +# fields (f1_score.macro etc.) are dereferenced with F.col. from pyspark.sql import functions as F profile_metrics = f"{batch_table}_profile_metrics" diff --git a/src/monitoring/notebooks/process_serving_logs.py b/src/monitoring/notebooks/process_serving_logs.py index bbb725a..62dfba0 100644 --- a/src/monitoring/notebooks/process_serving_logs.py +++ b/src/monitoring/notebooks/process_serving_logs.py @@ -69,7 +69,7 @@ # Unpack the AI Gateway payload table into one row per prediction. This is heavy Spark # plumbing (guarding the delivery lag, from_json, explode), so it is provided; the label -# join and write at the end are the step you fill in. +# join and write at the end are the exercise. from pyspark.sql import functions as F from pyspark.sql import types as T @@ -166,17 +166,18 @@ # MAGIC %md # MAGIC ## 3. Best-effort label join and persist -# MAGIC Attach the confirmed label where we already have it (null otherwise, online labels -# MAGIC lag), then overwrite the online inference table. Same schema as the batch -# MAGIC `fraud_predictions` table, so both monitors line up. +# MAGIC +# MAGIC Attach the confirmed label where available (null otherwise, online labels lag), then +# MAGIC overwrite the online inference table. Same schema as the batch `fraud_predictions` +# MAGIC table, so both monitors line up. # COMMAND ---------- # Attach the confirmed label AND the same feature columns as the batch fraud_predictions # table, so the two monitors are symmetric and track drift on every model feature. The serving -# payload only logs raw inputs, so we resolve the features the model used, by transaction_id, +# payload only logs raw inputs, so the features the model used are resolved by transaction_id # from the gold source: entity features are columns, and the on-demand features are computed -# with the SAME UC functions the model uses (no train/serve skew). The join is provided. +# with the same UC functions the model uses (no train/serve skew). The join is provided. schema_fqn = f"{catalog_name}.{user_schema}" gold_raw = spark.read.table(gold_table).select( "transaction_id", @@ -235,7 +236,7 @@ # MAGIC %md # MAGIC ## Recap -# MAGIC - The AI Gateway captured online traffic automatically; we only reshaped it. +# MAGIC - The AI Gateway captured online traffic automatically; this notebook only reshaped it. # MAGIC - `transaction_id` came from the `client_request_id` correlation id the caller set: # MAGIC the clean way to join model outputs back to confirmed-fraud labels later. # MAGIC - `fraud_serving_inference` now matches the batch `fraud_predictions` schema, so the diff --git a/src/monitoring/notebooks/seed_drift.py b/src/monitoring/notebooks/seed_drift.py index 5543954..40ee50c 100644 --- a/src/monitoring/notebooks/seed_drift.py +++ b/src/monitoring/notebooks/seed_drift.py @@ -54,16 +54,16 @@ # MAGIC %md # MAGIC ## 1. Build a drifted batch from already-scored rows -# MAGIC The batch inference table is already fully scored with every feature column, so we do -# MAGIC not re-run the model. Take a sample of existing predictions, inflate the amount-driven +# MAGIC The batch inference table is already fully scored with every feature column, so the +# MAGIC model is not re-run. Take a sample of existing predictions, inflate the amount-driven # MAGIC features (`amount` and the two ratios that scale with it), and stamp them with the -# MAGIC current time so they land in the newest monitor window. This is the "world changed" we -# MAGIC want the monitor to catch. +# MAGIC current time so they land in the newest monitor window. This simulates a change in the +# MAGIC input distribution for the monitor to catch. # COMMAND ---------- # Build the drifted sample by selecting from the same table (keeps the schema identical, no -# mergeSchema needed) and scaling the amount-driven features. We stage it to a separate table +# mergeSchema needed) and scaling the amount-driven features. Stage it to a separate table # first so the append does not read from the table it is writing to; serverless does not allow # cache/checkpoint, and a staging table is the clean way to break that dependency. stage_table = f"{catalog_name}.{user_schema}._seed_drift_stage" diff --git a/src/training/notebooks/training.py b/src/training/notebooks/training.py index 7c37137..909f31e 100644 --- a/src/training/notebooks/training.py +++ b/src/training/notebooks/training.py @@ -41,7 +41,7 @@ _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) user_schema = f"dev_{_short}_fraud" -# Labels come from the shared gold table; features come from your personal schema. +# Labels come from the shared gold table; features come from the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" card_feature_table = f"{catalog_name}.{user_schema}.card_features" client_feature_table = f"{catalog_name}.{user_schema}.client_features" @@ -155,9 +155,9 @@ .limit(SAMPLE_ROWS) ) -# The on-demand feature functions are provided (they mirror the UC UDFs you registered in -# the feature engineering notebook). You add the two entity FeatureLookups and assemble the -# training set. +# The on-demand feature functions are provided (they mirror the UC UDFs registered in the +# feature engineering notebook). Adding the two entity FeatureLookups and assembling the +# training set is the exercise. feature_functions = [ FeatureFunction( udf_name=f"{schema_fqn}.ff_is_night", @@ -266,13 +266,13 @@ def predict(self, context, model_input): naive_metrics = evaluate(naive, X_test, y_test) mlflow.log_params({**naive_params, "training_data": "imbalanced"}) mlflow.log_metrics(naive_metrics) - print("rf_imbalanced:", naive_metrics) # high accuracy, but look at recall + print("rf_imbalanced:", naive_metrics) # high accuracy, low recall -# Fixed random forest: the model we track and register. +# Fixed random forest: the model that is tracked and registered. with mlflow.start_run(run_name="rf_balanced") as run: # Balance the training rows: keep every fraud row and randomly sample an equal number of - # non-fraud rows (provided: this is a data-science fix, not the MLOps concept). We still - # evaluate on the untouched, imbalanced X_test/y_test so metrics reflect reality. + # non-fraud rows (provided: this is a data-science fix, not the MLOps concept). Evaluation + # still uses the untouched, imbalanced X_test/y_test so metrics reflect reality. fraud_idx = y_train[y_train == 1].index legit_idx = y_train[y_train == 0].sample(n=len(fraud_idx), random_state=42).index bal_idx = fraud_idx.union(legit_idx) @@ -292,8 +292,8 @@ def predict(self, context, model_input): mlflow.log_metrics(metrics) # Package with feature metadata + an inferred signature/input example (required for - # safe serving) and register to Unity Catalog in one call. We wrap the classifier so the - # served model returns a fraud PROBABILITY (score), not a hard class. + # safe serving) and register to Unity Catalog in one call. The classifier is wrapped so + # the served model returns a fraud probability (score), not a hard class. # TODO: log and register the model to Unity Catalog with its feature metadata # HINT: fe.log_model packages the model with the training_set's feature lookups (so serving # HINT: reproduces the same features) and registers it to UC in a single call. @@ -388,8 +388,7 @@ def predict(self, context, model_input): client = MlflowClient() # Unity Catalog's search_model_versions only supports a `name='...'` filter (no run_id -# filtering), so fetch this model's versions and take the highest: the one we just -# registered above. +# filtering), so fetch this model's versions and take the highest: the one registered above. versions = client.search_model_versions(f"name='{model_name}'") new_version = max(int(mv.version) for mv in versions) client.set_registered_model_alias(model_name, "challenger", new_version) From 5174f867c83b6a5200a299ed12b971ab3c579161 Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Mon, 13 Jul 2026 11:02:52 +0100 Subject: [PATCH 3/7] docs(training): remove last second-person phrase in a HINT The HINT/TODO markers are otherwise imperative and technical; drop 'you can print' in the monitoring refresh hint. --- solution/monitoring/notebooks/monitoring.py | 2 +- src/monitoring/notebooks/monitoring.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/solution/monitoring/notebooks/monitoring.py b/solution/monitoring/notebooks/monitoring.py index 581584f..73bed0e 100644 --- a/solution/monitoring/notebooks/monitoring.py +++ b/solution/monitoring/notebooks/monitoring.py @@ -67,7 +67,7 @@ try: # TODO-BEGIN: refresh the monitor for this table # HINT: w.quality_monitors.run_refresh(table_name=table) starts a refresh and - # HINT: returns a run whose run.refresh_id you can print. + # HINT: returns a run with a run.refresh_id to print. run = w.quality_monitors.run_refresh(table_name=table) # TODO-END print(f"Refresh started for {table}: refresh_id={run.refresh_id}") diff --git a/src/monitoring/notebooks/monitoring.py b/src/monitoring/notebooks/monitoring.py index 15ca8ed..17d8280 100644 --- a/src/monitoring/notebooks/monitoring.py +++ b/src/monitoring/notebooks/monitoring.py @@ -67,7 +67,7 @@ try: # TODO: refresh the monitor for this table # HINT: w.quality_monitors.run_refresh(table_name=table) starts a refresh and - # HINT: returns a run whose run.refresh_id you can print. + # HINT: returns a run with a run.refresh_id to print. # <-- Your code here print(f"Refresh started for {table}: refresh_id={run.refresh_id}") except Exception as exc: From 9f07e6305bb457ad7eb843d912a706d2a6008e71 Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Mon, 13 Jul 2026 11:09:19 +0100 Subject: [PATCH 4/7] docs(deployment): fix duplicate link, workshop phrasing, heading levels Remove a duplicated Docs link in query_endpoint, reword the batch_inference label note to production framing (drop 'for the workshop' / 'in the real world'), and promote evaluation's two stray level-3 headings to level 2 for consistency. --- .../deployment/batch_inference/notebooks/batch_inference.py | 6 +++--- .../deployment/model_deployment/notebooks/evaluation.py | 4 ++-- .../deployment/model_deployment/notebooks/query_endpoint.py | 2 -- src/deployment/batch_inference/notebooks/batch_inference.py | 6 +++--- src/deployment/model_deployment/notebooks/evaluation.py | 4 ++-- src/deployment/model_deployment/notebooks/query_endpoint.py | 2 -- 6 files changed, 10 insertions(+), 14 deletions(-) diff --git a/solution/deployment/batch_inference/notebooks/batch_inference.py b/solution/deployment/batch_inference/notebooks/batch_inference.py index e94e608..063cca0 100644 --- a/solution/deployment/batch_inference/notebooks/batch_inference.py +++ b/solution/deployment/batch_inference/notebooks/batch_inference.py @@ -139,9 +139,9 @@ # MAGIC not just prediction drift (a monitor can only drift columns that are in the table). # MAGIC Then append. # MAGIC -# MAGIC > The label (`is_fraud`) is joined here for the workshop so quality metrics compute -# MAGIC > immediately. In the real world fraud is confirmed later (chargebacks), so the label -# MAGIC > would be backfilled into this table when it arrives; drift is tracked in the meantime. +# MAGIC > The label (`is_fraud`) is joined here so quality metrics compute immediately. In +# MAGIC > production, fraud is confirmed later (chargebacks), so the label would be backfilled +# MAGIC > into this table when it arrives; drift is tracked in the meantime. # COMMAND ---------- diff --git a/solution/deployment/model_deployment/notebooks/evaluation.py b/solution/deployment/model_deployment/notebooks/evaluation.py index ed6f564..010fdfe 100644 --- a/solution/deployment/model_deployment/notebooks/evaluation.py +++ b/solution/deployment/model_deployment/notebooks/evaluation.py @@ -105,7 +105,7 @@ def score_holdout(version): # COMMAND ---------- # MAGIC %md -# MAGIC ### Confusion matrix for the approver +# MAGIC ## Confusion matrix for the approver # MAGIC # MAGIC Threshold the candidate's fraud score at 0.5 and show the confusion matrix on the # MAGIC holdout, logged to an MLflow run so it sits alongside the metrics the approver reviews @@ -134,7 +134,7 @@ def score_holdout(version): # COMMAND ---------- # MAGIC %md -# MAGIC ### ROC and precision-recall curves +# MAGIC ## ROC and precision-recall curves # MAGIC # MAGIC The confusion matrix is a single operating point; these curves show the full threshold # MAGIC trade-off: diff --git a/solution/deployment/model_deployment/notebooks/query_endpoint.py b/solution/deployment/model_deployment/notebooks/query_endpoint.py index 819450b..5c14dd4 100644 --- a/solution/deployment/model_deployment/notebooks/query_endpoint.py +++ b/solution/deployment/model_deployment/notebooks/query_endpoint.py @@ -16,8 +16,6 @@ # MAGIC and runs the on-demand functions per request. # MAGIC # MAGIC Docs: [Query serving endpoints](https://learn.microsoft.com/azure/databricks/machine-learning/model-serving/score-model-serving-endpoints) -# MAGIC -# MAGIC Docs: [Query serving endpoints](https://learn.microsoft.com/azure/databricks/machine-learning/model-serving/score-model-serving-endpoints) # COMMAND ---------- diff --git a/src/deployment/batch_inference/notebooks/batch_inference.py b/src/deployment/batch_inference/notebooks/batch_inference.py index 70efb23..ecb9655 100644 --- a/src/deployment/batch_inference/notebooks/batch_inference.py +++ b/src/deployment/batch_inference/notebooks/batch_inference.py @@ -135,9 +135,9 @@ # MAGIC not just prediction drift (a monitor can only drift columns that are in the table). # MAGIC Then append. # MAGIC -# MAGIC > The label (`is_fraud`) is joined here for the workshop so quality metrics compute -# MAGIC > immediately. In the real world fraud is confirmed later (chargebacks), so the label -# MAGIC > would be backfilled into this table when it arrives; drift is tracked in the meantime. +# MAGIC > The label (`is_fraud`) is joined here so quality metrics compute immediately. In +# MAGIC > production, fraud is confirmed later (chargebacks), so the label would be backfilled +# MAGIC > into this table when it arrives; drift is tracked in the meantime. # COMMAND ---------- diff --git a/src/deployment/model_deployment/notebooks/evaluation.py b/src/deployment/model_deployment/notebooks/evaluation.py index ed6f564..010fdfe 100644 --- a/src/deployment/model_deployment/notebooks/evaluation.py +++ b/src/deployment/model_deployment/notebooks/evaluation.py @@ -105,7 +105,7 @@ def score_holdout(version): # COMMAND ---------- # MAGIC %md -# MAGIC ### Confusion matrix for the approver +# MAGIC ## Confusion matrix for the approver # MAGIC # MAGIC Threshold the candidate's fraud score at 0.5 and show the confusion matrix on the # MAGIC holdout, logged to an MLflow run so it sits alongside the metrics the approver reviews @@ -134,7 +134,7 @@ def score_holdout(version): # COMMAND ---------- # MAGIC %md -# MAGIC ### ROC and precision-recall curves +# MAGIC ## ROC and precision-recall curves # MAGIC # MAGIC The confusion matrix is a single operating point; these curves show the full threshold # MAGIC trade-off: diff --git a/src/deployment/model_deployment/notebooks/query_endpoint.py b/src/deployment/model_deployment/notebooks/query_endpoint.py index 819450b..5c14dd4 100644 --- a/src/deployment/model_deployment/notebooks/query_endpoint.py +++ b/src/deployment/model_deployment/notebooks/query_endpoint.py @@ -16,8 +16,6 @@ # MAGIC and runs the on-demand functions per request. # MAGIC # MAGIC Docs: [Query serving endpoints](https://learn.microsoft.com/azure/databricks/machine-learning/model-serving/score-model-serving-endpoints) -# MAGIC -# MAGIC Docs: [Query serving endpoints](https://learn.microsoft.com/azure/databricks/machine-learning/model-serving/score-model-serving-endpoints) # COMMAND ---------- From 1aa0949e62ce7f020659089a50ae81bad8e00c28 Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Mon, 13 Jul 2026 11:11:19 +0100 Subject: [PATCH 5/7] docs(feature-engineering): note point-in-time and dimension-source caveats Add a Production considerations section documenting the two deliberate simplifications: primary-key-only feature tables with no timestamp_lookup_key (point-in-time / leakage), and sourcing entity attributes from the flattened gold table rather than the cards/users dimension of record. --- .../notebooks/build_features.py | 20 +++++++++++++++++++ .../notebooks/build_features.py | 20 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/solution/feature_engineering/notebooks/build_features.py b/solution/feature_engineering/notebooks/build_features.py index 53b5081..9b8a043 100644 --- a/solution/feature_engineering/notebooks/build_features.py +++ b/solution/feature_engineering/notebooks/build_features.py @@ -228,5 +228,25 @@ def publish(name, df, key, description): for column, text in CLIENT_COMMENTS.items(): spark.sql(f"ALTER TABLE {client_feature_table} ALTER COLUMN {column} COMMENT '{text}'") +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Production considerations +# MAGIC +# MAGIC This notebook makes two deliberate simplifications: +# MAGIC +# MAGIC - **Point-in-time correctness.** The feature tables have a primary key only (no +# MAGIC `timestamp_keys`), and the training-set `FeatureLookup`s set no `timestamp_lookup_key`, +# MAGIC so each transaction joins the entity's current attribute values rather than the values +# MAGIC as of the transaction time. For attributes that change over time (for example a credit +# MAGIC score), this risks label leakage. In production, make the feature tables time-series +# MAGIC tables (add a feature timestamp and `timestamp_keys=[...]`), carry `transaction_ts` on +# MAGIC the training spine, and set `timestamp_lookup_key="transaction_ts"` so each row joins the +# MAGIC value effective at its own time. +# MAGIC - **Source of the entity attributes.** Here the card/client attributes are taken from the +# MAGIC flattened `transactions_enriched` table with `F.first`. In production, source them from +# MAGIC the entity dimension of record (the `cards` / `users` tables), which is authoritative, +# MAGIC carries history, and supports the point-in-time joins above. + display(spark.read.table(card_feature_table).limit(10)) display(spark.read.table(client_feature_table).limit(10)) diff --git a/src/feature_engineering/notebooks/build_features.py b/src/feature_engineering/notebooks/build_features.py index 5e0dcec..86d8cdb 100644 --- a/src/feature_engineering/notebooks/build_features.py +++ b/src/feature_engineering/notebooks/build_features.py @@ -204,5 +204,25 @@ for column, text in CLIENT_COMMENTS.items(): spark.sql(f"ALTER TABLE {client_feature_table} ALTER COLUMN {column} COMMENT '{text}'") +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Production considerations +# MAGIC +# MAGIC This notebook makes two deliberate simplifications: +# MAGIC +# MAGIC - **Point-in-time correctness.** The feature tables have a primary key only (no +# MAGIC `timestamp_keys`), and the training-set `FeatureLookup`s set no `timestamp_lookup_key`, +# MAGIC so each transaction joins the entity's current attribute values rather than the values +# MAGIC as of the transaction time. For attributes that change over time (for example a credit +# MAGIC score), this risks label leakage. In production, make the feature tables time-series +# MAGIC tables (add a feature timestamp and `timestamp_keys=[...]`), carry `transaction_ts` on +# MAGIC the training spine, and set `timestamp_lookup_key="transaction_ts"` so each row joins the +# MAGIC value effective at its own time. +# MAGIC - **Source of the entity attributes.** Here the card/client attributes are taken from the +# MAGIC flattened `transactions_enriched` table with `F.first`. In production, source them from +# MAGIC the entity dimension of record (the `cards` / `users` tables), which is authoritative, +# MAGIC carries history, and supports the point-in-time joins above. + display(spark.read.table(card_feature_table).limit(10)) display(spark.read.table(client_feature_table).limit(10)) From 4e1c4d48ac338e62089db68554a86cdc86762dfb Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Mon, 13 Jul 2026 11:16:28 +0100 Subject: [PATCH 6/7] docs(readmes): professionalise root README and align stage READMEs Remove first-person plural and chatty asides from the root README (we train, what we do, let it run while you talk, just works, our shared), rename the developer-workflow heading, and drop the weekly fraud report framing from the deployment READMEs to match the notebooks. Regenerate src. --- README.md | 32 +++++++++---------- solution/deployment/README.md | 2 +- solution/deployment/batch_inference/README.md | 2 +- src/deployment/README.md | 2 +- src/deployment/batch_inference/README.md | 2 +- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 6744e37..eb4302f 100644 --- a/README.md +++ b/README.md @@ -146,13 +146,13 @@ The same feature definitions are used at training and serving, so there is no tr ### Proposed models (progression) -We deliberately keep models simple so the focus stays on the **MLOps lifecycle**, not -hyper-tuning. We train a progression and let MLflow pick a champion: +The models are kept deliberately simple so the focus stays on the **MLOps lifecycle**, not +hyper-tuning. A short progression is trained and MLflow selects a champion: | Stage | Model | Why | |---|---|---| -| Trap | **Random Forest** (raw imbalanced data) | Shows the imbalance trap: ~99.8% accuracy but recall ≈ 0 (flags no fraud) | -| Fixed / registered | **Random Forest** (balanced training data) | Keep all fraud + down-sample non-fraud so the model actually catches fraud; evaluated on the real imbalanced test set. This is the version registered + promoted | +| Baseline | **Random Forest** (raw imbalanced data) | Establishes the imbalance problem: ~99.8% accuracy but recall ≈ 0 (flags no fraud) | +| Balanced (registered) | **Random Forest** (balanced training data) | Keeps all fraud and down-samples non-fraud so the model detects fraud; evaluated on the real imbalanced test set. This version is registered and promoted | Class imbalance is handled by **balancing the training data** (keep every fraud row, down-sample non-fraud to a 1:1 ratio), not by class weights. The model is still evaluated on @@ -161,7 +161,7 @@ trade-off this creates and why the decision threshold is a business choice. ### Evaluation metrics -Accuracy is misleading on imbalanced data, so we track: +Accuracy is misleading on imbalanced data, so the tracked metrics are: - **ROC-AUC**, the metric the promotion gate scores the model on. - **precision**, **recall**, **F1** (macro), and accuracy for context. @@ -170,7 +170,7 @@ Accuracy is misleading on imbalanced data, so we track: ### How it maps to the workshop -| Session | What we do with the model | +| Session | Model activity | |---|---| | Model Development | Build features (Feature Store), train the progression, log to MLflow | | Governance | Govern the model registered during training; aliases, versioning + lineage | @@ -181,18 +181,18 @@ Accuracy is misleading on imbalanced data, so we track: > **The Serving step is slow.** The deployment job (`fraud_deployment_job`) provisions a > **Lakebase online feature store**, publishes the `card_features`/`client_features` tables to > it, and builds a first-time serving endpoint, which typically takes **15-25 minutes** -> end-to-end. It runs automatically when a new model version is registered; let it run while -> you talk through the concepts. Once it's ready, run `deployment/model_deployment/notebooks/query_endpoint.py` +> end-to-end. It runs automatically when a new model version is registered; allow it to +> complete before running `deployment/model_deployment/notebooks/query_endpoint.py` > to score live transactions against the endpoint (this also seeds the online monitor's traffic). --- -## How you work with this repo (developer workflow) +## Developer workflow This repo is a **Declarative Automation Bundle** (formerly a Databricks Asset Bundle / DAB) with a -`databricks.yml` at the root. You -develop either **in the Databricks UI** or **in a local IDE**, commit to Git, and let -**CI/CD** promote the bundle through environments. You don't edit prod directly. +`databricks.yml` at the root. Development happens either **in the Databricks UI** or **in a +local IDE**; changes are committed to Git, and **CI/CD** promotes the bundle through +environments. Prod is never edited directly. ```mermaid flowchart LR @@ -399,7 +399,7 @@ targets. CI/CD just picks the target, and the bundle does the rest. ## Data isolation: shared reads, personal writes Attendees **read** from shared medallion data but **write** to their **own** schema, so 20 -people never overwrite each other. This is wired through the bundle so it "just works" when +people never overwrite each other. The bundle wires this up, so it applies automatically when each attendee clones the repo and deploys. > **Why a personal schema and not more medallion layers?** Medallion (bronze/silver/gold) and @@ -407,7 +407,7 @@ each attendee clones the repo and deploys. > here: > - **Medallion** organises the *shared enterprise data* by quality. It's "a recommended best > practice but not a requirement" and each layer lives in its own schema of a shared catalog -> ([medallion docs](https://docs.databricks.com/aws/en/lakehouse/medallion)). Our shared +> ([medallion docs](https://docs.databricks.com/aws/en/lakehouse/medallion)). The shared > `fraud_bronze/silver/gold` (including the gold `transactions_enriched`) is unchanged. > - **Personal schemas** isolate each *developer's own outputs* in dev. Databricks explicitly > recommends "a personal schema per user, for example `dev_${user_name}`… This prevents @@ -416,8 +416,8 @@ each attendee clones the repo and deploys. > > So each attendee still *reads* the shared medallion; their feature tables / model / predictions > are dev outputs that go in their personal `dev__fraud` schema. In **production** one -> pipeline would instead write the feature tables to the shared schema. We don't sub-layer -> the personal schema into bronze/silver/gold, since that's a shared-data concept, and doing so +> pipeline would instead write the feature tables to the shared schema. The personal schema is +> not sub-layered into bronze/silver/gold, since that is a shared-data concept, and doing so > per user would explode the schema count. **Shared, read-only** (instructor provisions once): diff --git a/solution/deployment/README.md b/solution/deployment/README.md index c85cc04..529238e 100644 --- a/solution/deployment/README.md +++ b/solution/deployment/README.md @@ -3,4 +3,4 @@ | Folder | Description | |---|---| | `model_deployment/` | The MLflow deployment job (evaluate → approve → promote to `@champion` and update the serving endpoint) plus a manual endpoint-test lab. | -| `batch_inference/` | Batch scoring with the `@champion` model (the weekly fraud report). | +| `batch_inference/` | Batch scoring with the `@champion` model (the scheduled batch prediction path). | diff --git a/solution/deployment/batch_inference/README.md b/solution/deployment/batch_inference/README.md index 834a36e..9c32d15 100644 --- a/solution/deployment/batch_inference/README.md +++ b/solution/deployment/batch_inference/README.md @@ -2,4 +2,4 @@ | File | Description | |---|---| -| `notebooks/batch_inference.py` | Score a batch of transactions with the `@champion` model and append the results to the inference table (the weekly fraud report). | +| `notebooks/batch_inference.py` | Score a batch of transactions with the `@champion` model and append the results to the inference table. | diff --git a/src/deployment/README.md b/src/deployment/README.md index c85cc04..529238e 100644 --- a/src/deployment/README.md +++ b/src/deployment/README.md @@ -3,4 +3,4 @@ | Folder | Description | |---|---| | `model_deployment/` | The MLflow deployment job (evaluate → approve → promote to `@champion` and update the serving endpoint) plus a manual endpoint-test lab. | -| `batch_inference/` | Batch scoring with the `@champion` model (the weekly fraud report). | +| `batch_inference/` | Batch scoring with the `@champion` model (the scheduled batch prediction path). | diff --git a/src/deployment/batch_inference/README.md b/src/deployment/batch_inference/README.md index 834a36e..9c32d15 100644 --- a/src/deployment/batch_inference/README.md +++ b/src/deployment/batch_inference/README.md @@ -2,4 +2,4 @@ | File | Description | |---|---| -| `notebooks/batch_inference.py` | Score a batch of transactions with the `@champion` model and append the results to the inference table (the weekly fraud report). | +| `notebooks/batch_inference.py` | Score a batch of transactions with the `@champion` model and append the results to the inference table. | From 3e77da7ed76b827e70efe4054adbbb8b6b5acff2 Mon Sep 17 00:00:00 2001 From: Ben Constable Date: Mon, 13 Jul 2026 11:19:07 +0100 Subject: [PATCH 7/7] docs(notebooks): drop filler words (simply, actually) Remove filler adverbs from the monitoring and training notebook comments and markdown per the repo style. Regenerate src. --- solution/monitoring/notebooks/feature_drift_check.py | 4 ++-- solution/monitoring/notebooks/metric_violation_check.py | 2 +- solution/monitoring/notebooks/process_serving_logs.py | 2 +- solution/training/notebooks/training.py | 2 +- src/monitoring/notebooks/feature_drift_check.py | 4 ++-- src/monitoring/notebooks/metric_violation_check.py | 2 +- src/monitoring/notebooks/process_serving_logs.py | 2 +- src/training/notebooks/training.py | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/solution/monitoring/notebooks/feature_drift_check.py b/solution/monitoring/notebooks/feature_drift_check.py index 99a9e52..2d8bf00 100644 --- a/solution/monitoring/notebooks/feature_drift_check.py +++ b/solution/monitoring/notebooks/feature_drift_check.py @@ -82,7 +82,7 @@ # MAGIC # MAGIC PSI is thresholded (not `js_distance`) because Lakehouse Monitoring leaves `js_distance` # MAGIC null for numeric columns and only fills it for categoricals, so PSI is the drift knob -# MAGIC that actually fires on numeric features like `amount`. If **any** monitored feature is +# MAGIC that fires on numeric features like `amount`. If **any** monitored feature is # MAGIC in violation, `is_drift_violated` is set. # COMMAND ---------- @@ -99,7 +99,7 @@ drift_metrics_table = f"{table_name_under_monitor}_drift_metrics" # If the monitor hasn't produced drift metrics yet (first run, or never refreshed), there is -# nothing to evaluate, so report "not violated" and the job simply doesn't retrain. +# nothing to evaluate, so report "not violated" and the job does not retrain. if not spark.catalog.tableExists(drift_metrics_table): print(f"{drift_metrics_table} does not exist yet; treating as not violated.") dbutils.jobs.taskValues.set("is_drift_violated", False) diff --git a/solution/monitoring/notebooks/metric_violation_check.py b/solution/monitoring/notebooks/metric_violation_check.py index 5f36d5a..05b6ee9 100644 --- a/solution/monitoring/notebooks/metric_violation_check.py +++ b/solution/monitoring/notebooks/metric_violation_check.py @@ -79,7 +79,7 @@ profile_metrics_table = f"{table_name_under_monitor}_profile_metrics" # If the monitor hasn't produced metrics yet (first run, or never refreshed), there is -# nothing to evaluate, so report "not violated" and the job simply doesn't retrain. +# nothing to evaluate, so report "not violated" and the job does not retrain. if not spark.catalog.tableExists(profile_metrics_table): print(f"{profile_metrics_table} does not exist yet; treating as not violated.") dbutils.jobs.taskValues.set("is_metric_violated", False) diff --git a/solution/monitoring/notebooks/process_serving_logs.py b/solution/monitoring/notebooks/process_serving_logs.py index 28aa3f6..f823279 100644 --- a/solution/monitoring/notebooks/process_serving_logs.py +++ b/solution/monitoring/notebooks/process_serving_logs.py @@ -97,7 +97,7 @@ ) # The AI Gateway inference table starts as a placeholder (only databricks_request_id) and -# only gains its real columns once a request/response has actually been logged. Treat a +# only gains its real columns once a request/response has been logged. Treat a # missing table OR a not-yet-populated schema the same way: write an empty table and exit. required_cols = {"status_code", "response", "client_request_id", "request_time", "served_entity_id"} payload_cols = ( diff --git a/solution/training/notebooks/training.py b/solution/training/notebooks/training.py index 5dd1c97..6e73119 100644 --- a/solution/training/notebooks/training.py +++ b/solution/training/notebooks/training.py @@ -276,7 +276,7 @@ def predict(self, context, model_input): # Naive random forest (instructor-provided): the imbalance trap. Trained on the raw, # highly-imbalanced data it just learns to predict the majority class, so accuracy looks -# great while recall is ~0 (it never actually flags a fraud). Logged for comparison, NOT +# great while recall is ~0 (it never flags a fraud). Logged for comparison, NOT # registered. with mlflow.start_run(run_name="rf_imbalanced"): naive_params = {"n_estimators": 100, "random_state": 42} diff --git a/src/monitoring/notebooks/feature_drift_check.py b/src/monitoring/notebooks/feature_drift_check.py index 99a9e52..2d8bf00 100644 --- a/src/monitoring/notebooks/feature_drift_check.py +++ b/src/monitoring/notebooks/feature_drift_check.py @@ -82,7 +82,7 @@ # MAGIC # MAGIC PSI is thresholded (not `js_distance`) because Lakehouse Monitoring leaves `js_distance` # MAGIC null for numeric columns and only fills it for categoricals, so PSI is the drift knob -# MAGIC that actually fires on numeric features like `amount`. If **any** monitored feature is +# MAGIC that fires on numeric features like `amount`. If **any** monitored feature is # MAGIC in violation, `is_drift_violated` is set. # COMMAND ---------- @@ -99,7 +99,7 @@ drift_metrics_table = f"{table_name_under_monitor}_drift_metrics" # If the monitor hasn't produced drift metrics yet (first run, or never refreshed), there is -# nothing to evaluate, so report "not violated" and the job simply doesn't retrain. +# nothing to evaluate, so report "not violated" and the job does not retrain. if not spark.catalog.tableExists(drift_metrics_table): print(f"{drift_metrics_table} does not exist yet; treating as not violated.") dbutils.jobs.taskValues.set("is_drift_violated", False) diff --git a/src/monitoring/notebooks/metric_violation_check.py b/src/monitoring/notebooks/metric_violation_check.py index 5f36d5a..05b6ee9 100644 --- a/src/monitoring/notebooks/metric_violation_check.py +++ b/src/monitoring/notebooks/metric_violation_check.py @@ -79,7 +79,7 @@ profile_metrics_table = f"{table_name_under_monitor}_profile_metrics" # If the monitor hasn't produced metrics yet (first run, or never refreshed), there is -# nothing to evaluate, so report "not violated" and the job simply doesn't retrain. +# nothing to evaluate, so report "not violated" and the job does not retrain. if not spark.catalog.tableExists(profile_metrics_table): print(f"{profile_metrics_table} does not exist yet; treating as not violated.") dbutils.jobs.taskValues.set("is_metric_violated", False) diff --git a/src/monitoring/notebooks/process_serving_logs.py b/src/monitoring/notebooks/process_serving_logs.py index 62dfba0..da29e6f 100644 --- a/src/monitoring/notebooks/process_serving_logs.py +++ b/src/monitoring/notebooks/process_serving_logs.py @@ -97,7 +97,7 @@ ) # The AI Gateway inference table starts as a placeholder (only databricks_request_id) and -# only gains its real columns once a request/response has actually been logged. Treat a +# only gains its real columns once a request/response has been logged. Treat a # missing table OR a not-yet-populated schema the same way: write an empty table and exit. required_cols = {"status_code", "response", "client_request_id", "request_time", "served_entity_id"} payload_cols = ( diff --git a/src/training/notebooks/training.py b/src/training/notebooks/training.py index 909f31e..8e19b38 100644 --- a/src/training/notebooks/training.py +++ b/src/training/notebooks/training.py @@ -257,7 +257,7 @@ def predict(self, context, model_input): # Naive random forest (instructor-provided): the imbalance trap. Trained on the raw, # highly-imbalanced data it just learns to predict the majority class, so accuracy looks -# great while recall is ~0 (it never actually flags a fraud). Logged for comparison, NOT +# great while recall is ~0 (it never flags a fraud). Logged for comparison, NOT # registered. with mlflow.start_run(run_name="rf_imbalanced"): naive_params = {"n_estimators": 100, "random_state": 42}