diff --git a/README.md b/README.md index eb4302f..c824edf 100644 --- a/README.md +++ b/README.md @@ -436,7 +436,7 @@ The bundle declares the schema with a **base name** in [resources/schemas-resour ```yaml resources: schemas: - user_workspace: + ml_workspace: catalog_name: ${var.catalog_name} name: fraud ``` @@ -451,17 +451,17 @@ what the bundle created: ```yaml base_parameters: - gold_schema: ${var.gold_schema} # shared read -> fraud_gold - user_schema: ${resources.schemas.user_workspace.name} # personal write -> dev__fraud + gold_schema: ${var.gold_schema} # shared read -> fraud_gold + ml_schema: ${resources.schemas.ml_workspace.name} # personal write -> dev__fraud ``` and the model name follows the same schema: -`${var.catalog_name}.${resources.schemas.user_workspace.name}.fraud_detection`. +`${var.catalog_name}.${resources.schemas.ml_workspace.name}.fraud_detection`. > **Why reference the resource, not a plain variable?** Development mode prefixes the schema > *resource* but not a hand-written variable, so a variable like `${short_name}_fraud` would > not match the schema the bundle actually creates. Referencing -> `${resources.schemas.user_workspace.name}` guarantees notebooks and the provisioned schema +> `${resources.schemas.ml_workspace.name}` guarantees notebooks and the provisioned schema > use the identical name. ### Attendee flow diff --git a/databricks.yml b/databricks.yml index 8fdb3bc..ad0dc07 100644 --- a/databricks.yml +++ b/databricks.yml @@ -16,18 +16,28 @@ variables: default: /Users/${workspace.current_user.userName}/${bundle.target}-mlops-workshop-fraud model_name: description: Unity Catalog registered model name (in the per-user workspace schema). - default: ${var.catalog_name}.${var.user_schema}.fraud_detection + default: ${var.catalog_name}.${var.ml_schema}.fraud_detection catalog_name: description: Unity Catalog catalog for the environment (dev/staging/prod). default: adoption_workshop - user_schema: + ml_schema: description: >- - Per-user ML outputs schema. Resolves to the user_workspace schema resource, so it - picks up development mode's prefix (base `fraud` -> `dev__fraud`). - default: ${resources.schemas.user_workspace.name} + ML outputs schema (feature tables, model, predictions, monitoring). Resolves to the + ml_workspace schema resource: development mode prefixes the base `fraud` to + `dev__fraud` (per-user); the clean dev/staging/prod targets use the shared `fraud`. + default: ${resources.schemas.ml_workspace.name} gold_schema: description: Shared, read-only gold schema holding the source consumption tables. default: fraud_gold + silver_schema: + description: Shared silver schema (cleaned, typed, joined). Used by the instructor ingestion job. + default: fraud_silver + bronze_schema: + description: Shared bronze schema (raw ingested tables). Used by the instructor ingestion job. + default: fraud_bronze + landing_schema: + description: Landing schema holding the raw_data volume of source files. Instructor ingestion job. + default: fraud_landing auto_approve: description: >- Whether the deployment job's Approval_Check auto-approves (dev) or requires a human @@ -67,11 +77,11 @@ include: # landing volume). Declared ONLY on the clean (non-development) targets so that: # * a `bundle deploy -t personal` (development mode) NEVER creates them. Attendees read # the shared, instructor-seeded schemas and write only to their own -# `dev__fraud` (user_workspace). No empty per-user copies. +# `dev__fraud` (ml_workspace). No empty per-user copies. # * a clean deploy (`-t dev` / `-t staging` / `-t prod`, no name-prefixing) provisions # them literally as the shared `fraud_landing` / `fraud_bronze` / `fraud_silver` / # `fraud_gold` + volume in that target's catalog. -# `fraud_gold` is the shared source that everyone reads; the per-user `user_workspace` +# `fraud_gold` is the shared source that everyone reads; the per-user `ml_workspace` # schema is separately named `fraud` (ML outputs), so there is no collision on the clean # targets. # These resources only create the empty schemas/volume; the actual data load diff --git a/resources/batch-inference-workflow.yml b/resources/batch-inference-workflow.yml index bca8831..1a8e0cc 100644 --- a/resources/batch-inference-workflow.yml +++ b/resources/batch-inference-workflow.yml @@ -14,4 +14,4 @@ resources: model_name: ${var.model_name} catalog_name: ${var.catalog_name} gold_schema: ${var.gold_schema} - user_schema: ${var.user_schema} + ml_schema: ${var.ml_schema} diff --git a/resources/data-ingestion-workflow.yml b/resources/data-ingestion-workflow.yml index 55f388b..78244ff 100644 --- a/resources/data-ingestion-workflow.yml +++ b/resources/data-ingestion-workflow.yml @@ -13,9 +13,9 @@ resources: notebook_path: ../${var.code_root}/data_ingestion/notebooks/load_landing_to_bronze.py base_parameters: catalog: ${var.catalog_name} - bronze_schema: fraud_bronze + bronze_schema: ${var.bronze_schema} landing_catalog: ${var.catalog_name} - landing_schema: fraud_landing + landing_schema: ${var.landing_schema} landing_volume: raw_data - task_key: bronze_to_silver depends_on: @@ -24,9 +24,9 @@ resources: notebook_path: ../${var.code_root}/data_ingestion/notebooks/load_bronze_to_silver.py base_parameters: catalog: ${var.catalog_name} - bronze_schema: fraud_bronze - silver_schema: fraud_silver - gold_schema: fraud_gold + bronze_schema: ${var.bronze_schema} + silver_schema: ${var.silver_schema} + gold_schema: ${var.gold_schema} # In production the medallion pipeline runs on a schedule (or a file-arrival trigger) # so bronze/silver/gold stay fresh as new transactions land. It's commented out for # the workshop, where the shared data is seeded once against the clean `dev` target diff --git a/resources/deployment-job-workflow.yml b/resources/deployment-job-workflow.yml index 9676b63..4325023 100644 --- a/resources/deployment-job-workflow.yml +++ b/resources/deployment-job-workflow.yml @@ -34,6 +34,7 @@ resources: notebook_task: notebook_path: ../${var.code_root}/deployment/model_deployment/notebooks/evaluation.py base_parameters: + catalog_name: ${var.catalog_name} metric: roc_auc baseline: "0.65" @@ -57,3 +58,6 @@ resources: existing_cluster_id: ${var.shared_cluster_id} notebook_task: notebook_path: ../${var.code_root}/deployment/model_deployment/notebooks/deployment.py + base_parameters: + catalog_name: ${var.catalog_name} + ml_schema: ${var.ml_schema} diff --git a/resources/feature-engineering-workflow.yml b/resources/feature-engineering-workflow.yml index e3a6411..ae843de 100644 --- a/resources/feature-engineering-workflow.yml +++ b/resources/feature-engineering-workflow.yml @@ -15,4 +15,4 @@ resources: base_parameters: catalog_name: ${var.catalog_name} gold_schema: ${var.gold_schema} - user_schema: ${var.user_schema} + ml_schema: ${var.ml_schema} diff --git a/resources/model-training-workflow.yml b/resources/model-training-workflow.yml index 4ebc22f..8037ef3 100644 --- a/resources/model-training-workflow.yml +++ b/resources/model-training-workflow.yml @@ -21,7 +21,7 @@ resources: model_name: ${var.model_name} catalog_name: ${var.catalog_name} gold_schema: ${var.gold_schema} - user_schema: ${var.user_schema} + ml_schema: ${var.ml_schema} # Connect this model to its deployment job so registering a new version # auto-triggers the governed evaluate -> approve -> deploy pipeline. The # validation + serving that used to be separate tasks now live in that job. diff --git a/resources/monitoring-resource.yml b/resources/monitoring-resource.yml index 495df77..6cea809 100644 --- a/resources/monitoring-resource.yml +++ b/resources/monitoring-resource.yml @@ -25,8 +25,8 @@ resources: # Batch path: the weekly fraud report. Labels are joined, so this monitor tracks # model quality (accuracy/precision/recall/f1) as well as drift. batch_monitor: - table_name: ${var.catalog_name}.${resources.schemas.user_workspace.name}.fraud_predictions - output_schema_name: ${var.catalog_name}.${resources.schemas.user_workspace.name} + table_name: ${var.catalog_name}.${resources.schemas.ml_workspace.name}.fraud_predictions + output_schema_name: ${var.catalog_name}.${resources.schemas.ml_workspace.name} assets_dir: /Workspace/Users/${workspace.current_user.userName}/lakehouse_monitoring inference_log: granularities: [1 day] @@ -42,8 +42,8 @@ resources: # Online path: ad-hoc checks captured by the serving endpoint's AI Gateway inference # table, unpacked into the same shape. Labels lag, so this leads with drift. online_monitor: - table_name: ${var.catalog_name}.${resources.schemas.user_workspace.name}.fraud_serving_inference - output_schema_name: ${var.catalog_name}.${resources.schemas.user_workspace.name} + table_name: ${var.catalog_name}.${resources.schemas.ml_workspace.name}.fraud_serving_inference + output_schema_name: ${var.catalog_name}.${resources.schemas.ml_workspace.name} assets_dir: /Workspace/Users/${workspace.current_user.userName}/lakehouse_monitoring inference_log: granularities: [1 day] @@ -68,7 +68,7 @@ resources: notebook_task: notebook_path: ../${var.code_root}/monitoring/notebooks/metric_violation_check.py base_parameters: - table_name_under_monitor: ${var.catalog_name}.${resources.schemas.user_workspace.name}.fraud_predictions + table_name_under_monitor: ${var.catalog_name}.${resources.schemas.ml_workspace.name}.fraud_predictions metric_to_monitor: f1_score.macro metric_violation_threshold: "0.20" num_evaluation_windows: "5" @@ -88,7 +88,7 @@ resources: notebook_task: notebook_path: ../${var.code_root}/monitoring/notebooks/feature_drift_check.py base_parameters: - table_name_under_monitor: ${var.catalog_name}.${resources.schemas.user_workspace.name}.fraud_predictions + table_name_under_monitor: ${var.catalog_name}.${resources.schemas.ml_workspace.name}.fraud_predictions features_to_monitor: amount,credit_score,credit_limit,yearly_income,current_age,num_cards_issued,is_night,is_online,is_high_risk_mcc,amount_to_income_ratio,amount_to_credit_limit_ratio drift_metric: population_stability_index drift_violation_threshold: "0.20" diff --git a/resources/schemas-resource.yml b/resources/schemas-resource.yml index bffd10c..4e5e452 100644 --- a/resources/schemas-resource.yml +++ b/resources/schemas-resource.yml @@ -3,13 +3,13 @@ # for the ML artifacts they produce (feature tables, model, predictions, monitoring). # In production mode (no prefix) it resolves to the shared `fraud`. Reads always come # from the shared medallion schemas (fraud_bronze / fraud_silver / fraud_gold). Jobs -# reference ${resources.schemas.user_workspace.name} so notebooks write to exactly this +# reference ${resources.schemas.ml_workspace.name} so notebooks write to exactly this # schema. The layer ("gold") is intentionally NOT in the name: this schema holds ML # outputs (model, predictions, monitoring), not a mirror of the shared gold layer. # Docs: Bundle deployment modes (dev prefixing) https://learn.microsoft.com/azure/databricks/dev-tools/bundles/deployment-modes resources: schemas: - user_workspace: + ml_workspace: catalog_name: ${var.catalog_name} name: fraud comment: Per-user ML outputs (feature tables, model, predictions, monitoring). Shared in prod. diff --git a/resources/serving-log-processing-workflow.yml b/resources/serving-log-processing-workflow.yml index a7047a2..7a0ddc5 100644 --- a/resources/serving-log-processing-workflow.yml +++ b/resources/serving-log-processing-workflow.yml @@ -18,7 +18,7 @@ resources: base_parameters: catalog_name: ${var.catalog_name} gold_schema: ${var.gold_schema} - user_schema: ${var.user_schema} + ml_schema: ${var.ml_schema} # In production you'd run this on a schedule so the online inference table stays # fresh ahead of the monitor refresh. It's commented out for the workshop so nothing # auto-runs per attendee; the instructor triggers it on demand with diff --git a/solution/deployment/batch_inference/notebooks/batch_inference.py b/solution/deployment/batch_inference/notebooks/batch_inference.py index 0078e40..37f4237 100644 --- a/solution/deployment/batch_inference/notebooks/batch_inference.py +++ b/solution/deployment/batch_inference/notebooks/batch_inference.py @@ -25,30 +25,30 @@ dbutils.widgets.text("catalog_name", "adoption_workshop") dbutils.widgets.text("gold_schema", "fraud_gold") -dbutils.widgets.text("user_schema", "") +dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("model_name", "") dbutils.widgets.text("model_alias", "champion") catalog_name = dbutils.widgets.get("catalog_name") gold_schema = dbutils.widgets.get("gold_schema") -user_schema = dbutils.widgets.get("user_schema") +ml_schema = dbutils.widgets.get("ml_schema") from pyspark.sql import functions as F # Interactive fallback: jobs pass the resolved personal schema; running standalone derives # the same per-user name the bundle uses (dev__fraud) so runs stay isolated. -if not user_schema: +if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - user_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud" model_alias = dbutils.widgets.get("model_alias") # The batch-inference job passes the full three-level model name; fall back for interactive runs. -model_name = dbutils.widgets.get("model_name") or f"{catalog_name}.{user_schema}.fraud_detection" +model_name = dbutils.widgets.get("model_name") or f"{catalog_name}.{ml_schema}.fraud_detection" # Read the transactions to score from the shared gold source; write predictions to the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" -predictions_table = f"{catalog_name}.{user_schema}.fraud_predictions" +predictions_table = f"{catalog_name}.{ml_schema}.fraud_predictions" print(f"Model: {model_name}@{model_alias}") print(f"Score from: {source_table}") diff --git a/solution/deployment/model_deployment/notebooks/deployment.py b/solution/deployment/model_deployment/notebooks/deployment.py index f0bbcc7..12c7de0 100644 --- a/solution/deployment/model_deployment/notebooks/deployment.py +++ b/solution/deployment/model_deployment/notebooks/deployment.py @@ -21,19 +21,30 @@ # The deployment job injects model_name + model_version as JOB-level parameters. dbutils.widgets.text("model_name", "") dbutils.widgets.text("model_version", "") +dbutils.widgets.text("catalog_name", "adoption_workshop") +dbutils.widgets.text("ml_schema", "") model_name = dbutils.widgets.get("model_name") model_version = dbutils.widgets.get("model_version") +catalog_name = dbutils.widgets.get("catalog_name") +ml_schema = dbutils.widgets.get("ml_schema") assert model_name and model_version, ( "model_name and model_version are injected by the deployment job." ) -# The three-level model name carries the catalog + personal schema. -catalog_name, user_schema, _ = model_name.split(".") -endpoint_name = f"{user_schema}_fraud" +from pyspark.sql import functions as F + +# Interactive fallback: jobs pass the resolved personal schema; running standalone derives +# the same per-user name the bundle uses (dev__fraud) so runs stay isolated. +if not ml_schema: + _user = spark.range(1).select(F.current_user()).first()[0] + _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) + ml_schema = f"dev_{_short}_fraud" + +endpoint_name = f"{ml_schema}_fraud" online_store_name = "fraud-workshop-online" -card_feature_table = f"{catalog_name}.{user_schema}.card_features" -client_feature_table = f"{catalog_name}.{user_schema}.client_features" +card_feature_table = f"{catalog_name}.{ml_schema}.card_features" +client_feature_table = f"{catalog_name}.{ml_schema}.client_features" print(f"Deploying {model_name} v{model_version} -> endpoint {endpoint_name}") @@ -128,7 +139,7 @@ def wait_online_store(name, minutes=20): ] inference_table = AiGatewayInferenceTableConfig( catalog_name=catalog_name, - schema_name=user_schema, + schema_name=ml_schema, table_name_prefix="fraud_serving", enabled=True, ) diff --git a/solution/deployment/model_deployment/notebooks/evaluation.py b/solution/deployment/model_deployment/notebooks/evaluation.py index 010fdfe..9d2003c 100644 --- a/solution/deployment/model_deployment/notebooks/evaluation.py +++ b/solution/deployment/model_deployment/notebooks/evaluation.py @@ -24,11 +24,13 @@ # The deployment job injects model_name + model_version as JOB-level parameters. dbutils.widgets.text("model_name", "") dbutils.widgets.text("model_version", "") +dbutils.widgets.text("catalog_name", "adoption_workshop") dbutils.widgets.text("metric", "roc_auc") dbutils.widgets.text("baseline", "0.65") model_name = dbutils.widgets.get("model_name") model_version = dbutils.widgets.get("model_version") +catalog_name = dbutils.widgets.get("catalog_name") metric = dbutils.widgets.get("metric") baseline = float(dbutils.widgets.get("baseline")) @@ -54,7 +56,6 @@ # lookups + on-demand functions recorded at training, and the served model returns a fraud # probability, so a real ROC AUC can be computed. In production this holdout would be a # curated, leakage-controlled evaluation table; here it is sampled from the shared gold source. -catalog_name = model_name.split(".")[0] gold_table = f"{catalog_name}.fraud_gold.transactions_enriched" eval_spine = ( diff --git a/solution/deployment/model_deployment/notebooks/query_endpoint.py b/solution/deployment/model_deployment/notebooks/query_endpoint.py index e6699f7..9bf1ccb 100644 --- a/solution/deployment/model_deployment/notebooks/query_endpoint.py +++ b/solution/deployment/model_deployment/notebooks/query_endpoint.py @@ -21,24 +21,24 @@ dbutils.widgets.text("catalog_name", "adoption_workshop") dbutils.widgets.text("gold_schema", "fraud_gold") -dbutils.widgets.text("user_schema", "") +dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("num_requests", "50") catalog_name = dbutils.widgets.get("catalog_name") gold_schema = dbutils.widgets.get("gold_schema") -user_schema = dbutils.widgets.get("user_schema") +ml_schema = dbutils.widgets.get("ml_schema") num_requests = int(dbutils.widgets.get("num_requests")) from pyspark.sql import functions as F # Interactive fallback: derive the same per-user name the bundle uses (dev__fraud). -if not user_schema: +if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - user_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud" # One endpoint per user, named after the personal schema (matches the deployment job). -endpoint_name = f"{user_schema}_fraud" +endpoint_name = f"{ml_schema}_fraud" source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" print(f"Endpoint: {endpoint_name}") print(f"Sample: {num_requests} rows from {source_table}") diff --git a/solution/feature_engineering/notebooks/build_features.py b/solution/feature_engineering/notebooks/build_features.py index c8c4d24..fec875f 100644 --- a/solution/feature_engineering/notebooks/build_features.py +++ b/solution/feature_engineering/notebooks/build_features.py @@ -26,25 +26,25 @@ dbutils.widgets.text("catalog_name", "adoption_workshop") dbutils.widgets.text("gold_schema", "fraud_gold") -dbutils.widgets.text("user_schema", "") +dbutils.widgets.text("ml_schema", "") catalog_name = dbutils.widgets.get("catalog_name") gold_schema = dbutils.widgets.get("gold_schema") -user_schema = dbutils.widgets.get("user_schema") +ml_schema = dbutils.widgets.get("ml_schema") from pyspark.sql import functions as F # Interactive fallback: jobs pass the resolved personal schema; running standalone derives # the same per-user name the bundle uses (dev__fraud) so runs stay isolated. -if not user_schema: +if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - user_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud" # Read from the shared gold source; write to the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" -card_feature_table = f"{catalog_name}.{user_schema}.card_features" -client_feature_table = f"{catalog_name}.{user_schema}.client_features" +card_feature_table = f"{catalog_name}.{ml_schema}.card_features" +client_feature_table = f"{catalog_name}.{ml_schema}.client_features" # COMMAND ---------- @@ -140,7 +140,7 @@ def publish(name, df, key, description): # On-demand feature functions are UC Python UDFs. Training and serving call the same # functions, so there is no train/serve skew. Four are provided; the first (ff_is_night) is # the exercise. -schema_fqn = f"{catalog_name}.{user_schema}" +schema_fqn = f"{catalog_name}.{ml_schema}" # TODO-BEGIN: register the ff_is_night on-demand feature function as a UC Python UDF # HINT: CREATE OR REPLACE FUNCTION {schema_fqn}.ff_is_night(transaction_hour INT) RETURNS BOOLEAN diff --git a/solution/monitoring/notebooks/monitoring.py b/solution/monitoring/notebooks/monitoring.py index f839291..601bddf 100644 --- a/solution/monitoring/notebooks/monitoring.py +++ b/solution/monitoring/notebooks/monitoring.py @@ -24,22 +24,22 @@ # COMMAND ---------- dbutils.widgets.text("catalog_name", "adoption_workshop") -dbutils.widgets.text("user_schema", "") +dbutils.widgets.text("ml_schema", "") catalog_name = dbutils.widgets.get("catalog_name") -user_schema = dbutils.widgets.get("user_schema") +ml_schema = dbutils.widgets.get("ml_schema") from pyspark.sql import functions as F # Interactive fallback: jobs pass the resolved personal schema; running standalone derives # the same per-user name the bundle uses (dev__fraud) so runs stay isolated. -if not user_schema: +if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - user_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud" -batch_table = f"{catalog_name}.{user_schema}.fraud_predictions" -online_table = f"{catalog_name}.{user_schema}.fraud_serving_inference" +batch_table = f"{catalog_name}.{ml_schema}.fraud_predictions" +online_table = f"{catalog_name}.{ml_schema}.fraud_serving_inference" print(f"Batch monitor table: {batch_table}") print(f"Online monitor table: {online_table}") diff --git a/solution/monitoring/notebooks/process_serving_logs.py b/solution/monitoring/notebooks/process_serving_logs.py index e73dee8..819de39 100644 --- a/solution/monitoring/notebooks/process_serving_logs.py +++ b/solution/monitoring/notebooks/process_serving_logs.py @@ -33,23 +33,23 @@ dbutils.widgets.text("catalog_name", "adoption_workshop") dbutils.widgets.text("gold_schema", "fraud_gold") -dbutils.widgets.text("user_schema", "") +dbutils.widgets.text("ml_schema", "") catalog_name = dbutils.widgets.get("catalog_name") gold_schema = dbutils.widgets.get("gold_schema") -user_schema = dbutils.widgets.get("user_schema") +ml_schema = dbutils.widgets.get("ml_schema") from pyspark.sql import functions as F # Interactive fallback: jobs pass the resolved personal schema; running standalone derives # the same per-user name the bundle uses (dev__fraud) so runs stay isolated. -if not user_schema: +if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - user_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud" -payload_table = f"{catalog_name}.{user_schema}.fraud_serving_payload" -online_table = f"{catalog_name}.{user_schema}.fraud_serving_inference" +payload_table = f"{catalog_name}.{ml_schema}.fraud_serving_payload" +online_table = f"{catalog_name}.{ml_schema}.fraud_serving_inference" gold_table = f"{catalog_name}.{gold_schema}.transactions_enriched" print(f"Payload (raw): {payload_table}") @@ -178,7 +178,7 @@ # payload only logs raw inputs, so the features the model used are resolved by transaction_id # from the gold source: entity features are columns, and the on-demand features are computed # with the same UC functions the model uses (no train/serve skew). The join is provided. -schema_fqn = f"{catalog_name}.{user_schema}" +schema_fqn = f"{catalog_name}.{ml_schema}" gold_raw = spark.read.table(gold_table).select( "transaction_id", F.col("amount").cast("double").alias("amount"), diff --git a/solution/monitoring/notebooks/seed_drift.py b/solution/monitoring/notebooks/seed_drift.py index 1015d4d..d0d6530 100644 --- a/solution/monitoring/notebooks/seed_drift.py +++ b/solution/monitoring/notebooks/seed_drift.py @@ -24,7 +24,7 @@ # COMMAND ---------- dbutils.widgets.text("catalog_name", "adoption_workshop") -dbutils.widgets.text("user_schema", "") +dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("amount_multiplier", "6.0", label="Scale amount by this to force drift") dbutils.widgets.text("num_rows", "5000", label="How many drifted rows to append") dbutils.widgets.text("refresh_monitor", "true", label="Refresh the batch monitor after writing") @@ -32,20 +32,20 @@ from pyspark.sql import functions as F catalog_name = dbutils.widgets.get("catalog_name") -user_schema = dbutils.widgets.get("user_schema") +ml_schema = dbutils.widgets.get("ml_schema") # Interactive fallback: jobs pass the resolved personal schema; running standalone derives # the same per-user name the bundle uses (dev__fraud) so runs stay isolated. -if not user_schema: +if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - user_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud" amount_multiplier = float(dbutils.widgets.get("amount_multiplier")) num_rows = int(dbutils.widgets.get("num_rows")) refresh_monitor = dbutils.widgets.get("refresh_monitor").strip().lower() == "true" -predictions_table = f"{catalog_name}.{user_schema}.fraud_predictions" +predictions_table = f"{catalog_name}.{ml_schema}.fraud_predictions" print(f"Table: {predictions_table}") print(f"Drift: amount x{amount_multiplier} on {num_rows} rows, stamped now") @@ -66,7 +66,7 @@ # mergeSchema needed) and scaling the amount-driven features. Stage it to a separate table # first so the append does not read from the table it is writing to; serverless does not allow # cache/checkpoint, and a staging table is the clean way to break that dependency. -stage_table = f"{catalog_name}.{user_schema}._seed_drift_stage" +stage_table = f"{catalog_name}.{ml_schema}._seed_drift_stage" ( spark.read.table(predictions_table) .limit(num_rows) diff --git a/solution/training/notebooks/training.py b/solution/training/notebooks/training.py index 42eed9a..1a55220 100644 --- a/solution/training/notebooks/training.py +++ b/solution/training/notebooks/training.py @@ -23,28 +23,28 @@ dbutils.widgets.text("catalog_name", "adoption_workshop") dbutils.widgets.text("gold_schema", "fraud_gold") -dbutils.widgets.text("user_schema", "") +dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("experiment_name", "") dbutils.widgets.text("model_name", "") dbutils.widgets.text("deployment_job_id", "") catalog_name = dbutils.widgets.get("catalog_name") gold_schema = dbutils.widgets.get("gold_schema") -user_schema = dbutils.widgets.get("user_schema") +ml_schema = dbutils.widgets.get("ml_schema") from pyspark.sql import functions as F # Interactive fallback: jobs pass the resolved personal schema; running standalone derives # the same per-user name the bundle uses (dev__fraud) so runs stay isolated. -if not user_schema: +if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - user_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud" # Labels come from the shared gold table; features come from the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" -card_feature_table = f"{catalog_name}.{user_schema}.card_features" -client_feature_table = f"{catalog_name}.{user_schema}.client_features" +card_feature_table = f"{catalog_name}.{ml_schema}.card_features" +client_feature_table = f"{catalog_name}.{ml_schema}.client_features" # Sensible defaults so the notebook is runnable interactively (outside the DAB job), # where the experiment/model widgets may be empty. @@ -52,7 +52,7 @@ experiment_name = ( dbutils.widgets.get("experiment_name") or f"/Users/{current_user}/mlops-workshop-fraud" ) -model_name = dbutils.widgets.get("model_name") or f"{catalog_name}.{user_schema}.fraud_detection" +model_name = dbutils.widgets.get("model_name") or f"{catalog_name}.{ml_schema}.fraud_detection" print(f"Card features: {card_feature_table}") print(f"Client features: {client_feature_table}") @@ -131,7 +131,7 @@ # COMMAND ---------- -schema_fqn = f"{catalog_name}.{user_schema}" +schema_fqn = f"{catalog_name}.{ml_schema}" # Cap the training data at a random sample so the notebook runs in a couple of minutes. # Fraud is extremely rare, so a random sample (not the first N rows) is what keeps enough diff --git a/src/deployment/batch_inference/notebooks/batch_inference.py b/src/deployment/batch_inference/notebooks/batch_inference.py index f4600f1..1dc8932 100644 --- a/src/deployment/batch_inference/notebooks/batch_inference.py +++ b/src/deployment/batch_inference/notebooks/batch_inference.py @@ -25,30 +25,30 @@ dbutils.widgets.text("catalog_name", "adoption_workshop") dbutils.widgets.text("gold_schema", "fraud_gold") -dbutils.widgets.text("user_schema", "") +dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("model_name", "") dbutils.widgets.text("model_alias", "champion") catalog_name = dbutils.widgets.get("catalog_name") gold_schema = dbutils.widgets.get("gold_schema") -user_schema = dbutils.widgets.get("user_schema") +ml_schema = dbutils.widgets.get("ml_schema") from pyspark.sql import functions as F # Interactive fallback: jobs pass the resolved personal schema; running standalone derives # the same per-user name the bundle uses (dev__fraud) so runs stay isolated. -if not user_schema: +if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - user_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud" model_alias = dbutils.widgets.get("model_alias") # The batch-inference job passes the full three-level model name; fall back for interactive runs. -model_name = dbutils.widgets.get("model_name") or f"{catalog_name}.{user_schema}.fraud_detection" +model_name = dbutils.widgets.get("model_name") or f"{catalog_name}.{ml_schema}.fraud_detection" # Read the transactions to score from the shared gold source; write predictions to the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" -predictions_table = f"{catalog_name}.{user_schema}.fraud_predictions" +predictions_table = f"{catalog_name}.{ml_schema}.fraud_predictions" print(f"Model: {model_name}@{model_alias}") print(f"Score from: {source_table}") diff --git a/src/deployment/model_deployment/notebooks/deployment.py b/src/deployment/model_deployment/notebooks/deployment.py index f0bbcc7..12c7de0 100644 --- a/src/deployment/model_deployment/notebooks/deployment.py +++ b/src/deployment/model_deployment/notebooks/deployment.py @@ -21,19 +21,30 @@ # The deployment job injects model_name + model_version as JOB-level parameters. dbutils.widgets.text("model_name", "") dbutils.widgets.text("model_version", "") +dbutils.widgets.text("catalog_name", "adoption_workshop") +dbutils.widgets.text("ml_schema", "") model_name = dbutils.widgets.get("model_name") model_version = dbutils.widgets.get("model_version") +catalog_name = dbutils.widgets.get("catalog_name") +ml_schema = dbutils.widgets.get("ml_schema") assert model_name and model_version, ( "model_name and model_version are injected by the deployment job." ) -# The three-level model name carries the catalog + personal schema. -catalog_name, user_schema, _ = model_name.split(".") -endpoint_name = f"{user_schema}_fraud" +from pyspark.sql import functions as F + +# Interactive fallback: jobs pass the resolved personal schema; running standalone derives +# the same per-user name the bundle uses (dev__fraud) so runs stay isolated. +if not ml_schema: + _user = spark.range(1).select(F.current_user()).first()[0] + _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) + ml_schema = f"dev_{_short}_fraud" + +endpoint_name = f"{ml_schema}_fraud" online_store_name = "fraud-workshop-online" -card_feature_table = f"{catalog_name}.{user_schema}.card_features" -client_feature_table = f"{catalog_name}.{user_schema}.client_features" +card_feature_table = f"{catalog_name}.{ml_schema}.card_features" +client_feature_table = f"{catalog_name}.{ml_schema}.client_features" print(f"Deploying {model_name} v{model_version} -> endpoint {endpoint_name}") @@ -128,7 +139,7 @@ def wait_online_store(name, minutes=20): ] inference_table = AiGatewayInferenceTableConfig( catalog_name=catalog_name, - schema_name=user_schema, + schema_name=ml_schema, table_name_prefix="fraud_serving", enabled=True, ) diff --git a/src/deployment/model_deployment/notebooks/evaluation.py b/src/deployment/model_deployment/notebooks/evaluation.py index 010fdfe..9d2003c 100644 --- a/src/deployment/model_deployment/notebooks/evaluation.py +++ b/src/deployment/model_deployment/notebooks/evaluation.py @@ -24,11 +24,13 @@ # The deployment job injects model_name + model_version as JOB-level parameters. dbutils.widgets.text("model_name", "") dbutils.widgets.text("model_version", "") +dbutils.widgets.text("catalog_name", "adoption_workshop") dbutils.widgets.text("metric", "roc_auc") dbutils.widgets.text("baseline", "0.65") model_name = dbutils.widgets.get("model_name") model_version = dbutils.widgets.get("model_version") +catalog_name = dbutils.widgets.get("catalog_name") metric = dbutils.widgets.get("metric") baseline = float(dbutils.widgets.get("baseline")) @@ -54,7 +56,6 @@ # lookups + on-demand functions recorded at training, and the served model returns a fraud # probability, so a real ROC AUC can be computed. In production this holdout would be a # curated, leakage-controlled evaluation table; here it is sampled from the shared gold source. -catalog_name = model_name.split(".")[0] gold_table = f"{catalog_name}.fraud_gold.transactions_enriched" eval_spine = ( diff --git a/src/deployment/model_deployment/notebooks/query_endpoint.py b/src/deployment/model_deployment/notebooks/query_endpoint.py index e6699f7..9bf1ccb 100644 --- a/src/deployment/model_deployment/notebooks/query_endpoint.py +++ b/src/deployment/model_deployment/notebooks/query_endpoint.py @@ -21,24 +21,24 @@ dbutils.widgets.text("catalog_name", "adoption_workshop") dbutils.widgets.text("gold_schema", "fraud_gold") -dbutils.widgets.text("user_schema", "") +dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("num_requests", "50") catalog_name = dbutils.widgets.get("catalog_name") gold_schema = dbutils.widgets.get("gold_schema") -user_schema = dbutils.widgets.get("user_schema") +ml_schema = dbutils.widgets.get("ml_schema") num_requests = int(dbutils.widgets.get("num_requests")) from pyspark.sql import functions as F # Interactive fallback: derive the same per-user name the bundle uses (dev__fraud). -if not user_schema: +if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - user_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud" # One endpoint per user, named after the personal schema (matches the deployment job). -endpoint_name = f"{user_schema}_fraud" +endpoint_name = f"{ml_schema}_fraud" source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" print(f"Endpoint: {endpoint_name}") print(f"Sample: {num_requests} rows from {source_table}") diff --git a/src/feature_engineering/notebooks/build_features.py b/src/feature_engineering/notebooks/build_features.py index 7ce711b..b5a39cd 100644 --- a/src/feature_engineering/notebooks/build_features.py +++ b/src/feature_engineering/notebooks/build_features.py @@ -26,25 +26,25 @@ dbutils.widgets.text("catalog_name", "adoption_workshop") dbutils.widgets.text("gold_schema", "fraud_gold") -dbutils.widgets.text("user_schema", "") +dbutils.widgets.text("ml_schema", "") catalog_name = dbutils.widgets.get("catalog_name") gold_schema = dbutils.widgets.get("gold_schema") -user_schema = dbutils.widgets.get("user_schema") +ml_schema = dbutils.widgets.get("ml_schema") from pyspark.sql import functions as F # Interactive fallback: jobs pass the resolved personal schema; running standalone derives # the same per-user name the bundle uses (dev__fraud) so runs stay isolated. -if not user_schema: +if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - user_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud" # Read from the shared gold source; write to the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" -card_feature_table = f"{catalog_name}.{user_schema}.card_features" -client_feature_table = f"{catalog_name}.{user_schema}.client_features" +card_feature_table = f"{catalog_name}.{ml_schema}.card_features" +client_feature_table = f"{catalog_name}.{ml_schema}.client_features" # COMMAND ---------- @@ -125,7 +125,7 @@ # On-demand feature functions are UC Python UDFs. Training and serving call the same # functions, so there is no train/serve skew. Four are provided; the first (ff_is_night) is # the exercise. -schema_fqn = f"{catalog_name}.{user_schema}" +schema_fqn = f"{catalog_name}.{ml_schema}" # TODO: register the ff_is_night on-demand feature function as a UC Python UDF # HINT: CREATE OR REPLACE FUNCTION {schema_fqn}.ff_is_night(transaction_hour INT) RETURNS BOOLEAN diff --git a/src/monitoring/notebooks/monitoring.py b/src/monitoring/notebooks/monitoring.py index f7e136b..6fc202d 100644 --- a/src/monitoring/notebooks/monitoring.py +++ b/src/monitoring/notebooks/monitoring.py @@ -24,22 +24,22 @@ # COMMAND ---------- dbutils.widgets.text("catalog_name", "adoption_workshop") -dbutils.widgets.text("user_schema", "") +dbutils.widgets.text("ml_schema", "") catalog_name = dbutils.widgets.get("catalog_name") -user_schema = dbutils.widgets.get("user_schema") +ml_schema = dbutils.widgets.get("ml_schema") from pyspark.sql import functions as F # Interactive fallback: jobs pass the resolved personal schema; running standalone derives # the same per-user name the bundle uses (dev__fraud) so runs stay isolated. -if not user_schema: +if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - user_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud" -batch_table = f"{catalog_name}.{user_schema}.fraud_predictions" -online_table = f"{catalog_name}.{user_schema}.fraud_serving_inference" +batch_table = f"{catalog_name}.{ml_schema}.fraud_predictions" +online_table = f"{catalog_name}.{ml_schema}.fraud_serving_inference" print(f"Batch monitor table: {batch_table}") print(f"Online monitor table: {online_table}") diff --git a/src/monitoring/notebooks/process_serving_logs.py b/src/monitoring/notebooks/process_serving_logs.py index fc045aa..00bc9b0 100644 --- a/src/monitoring/notebooks/process_serving_logs.py +++ b/src/monitoring/notebooks/process_serving_logs.py @@ -33,23 +33,23 @@ dbutils.widgets.text("catalog_name", "adoption_workshop") dbutils.widgets.text("gold_schema", "fraud_gold") -dbutils.widgets.text("user_schema", "") +dbutils.widgets.text("ml_schema", "") catalog_name = dbutils.widgets.get("catalog_name") gold_schema = dbutils.widgets.get("gold_schema") -user_schema = dbutils.widgets.get("user_schema") +ml_schema = dbutils.widgets.get("ml_schema") from pyspark.sql import functions as F # Interactive fallback: jobs pass the resolved personal schema; running standalone derives # the same per-user name the bundle uses (dev__fraud) so runs stay isolated. -if not user_schema: +if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - user_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud" -payload_table = f"{catalog_name}.{user_schema}.fraud_serving_payload" -online_table = f"{catalog_name}.{user_schema}.fraud_serving_inference" +payload_table = f"{catalog_name}.{ml_schema}.fraud_serving_payload" +online_table = f"{catalog_name}.{ml_schema}.fraud_serving_inference" gold_table = f"{catalog_name}.{gold_schema}.transactions_enriched" print(f"Payload (raw): {payload_table}") @@ -178,7 +178,7 @@ # payload only logs raw inputs, so the features the model used are resolved by transaction_id # from the gold source: entity features are columns, and the on-demand features are computed # with the same UC functions the model uses (no train/serve skew). The join is provided. -schema_fqn = f"{catalog_name}.{user_schema}" +schema_fqn = f"{catalog_name}.{ml_schema}" gold_raw = spark.read.table(gold_table).select( "transaction_id", F.col("amount").cast("double").alias("amount"), diff --git a/src/monitoring/notebooks/seed_drift.py b/src/monitoring/notebooks/seed_drift.py index 1015d4d..d0d6530 100644 --- a/src/monitoring/notebooks/seed_drift.py +++ b/src/monitoring/notebooks/seed_drift.py @@ -24,7 +24,7 @@ # COMMAND ---------- dbutils.widgets.text("catalog_name", "adoption_workshop") -dbutils.widgets.text("user_schema", "") +dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("amount_multiplier", "6.0", label="Scale amount by this to force drift") dbutils.widgets.text("num_rows", "5000", label="How many drifted rows to append") dbutils.widgets.text("refresh_monitor", "true", label="Refresh the batch monitor after writing") @@ -32,20 +32,20 @@ from pyspark.sql import functions as F catalog_name = dbutils.widgets.get("catalog_name") -user_schema = dbutils.widgets.get("user_schema") +ml_schema = dbutils.widgets.get("ml_schema") # Interactive fallback: jobs pass the resolved personal schema; running standalone derives # the same per-user name the bundle uses (dev__fraud) so runs stay isolated. -if not user_schema: +if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - user_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud" amount_multiplier = float(dbutils.widgets.get("amount_multiplier")) num_rows = int(dbutils.widgets.get("num_rows")) refresh_monitor = dbutils.widgets.get("refresh_monitor").strip().lower() == "true" -predictions_table = f"{catalog_name}.{user_schema}.fraud_predictions" +predictions_table = f"{catalog_name}.{ml_schema}.fraud_predictions" print(f"Table: {predictions_table}") print(f"Drift: amount x{amount_multiplier} on {num_rows} rows, stamped now") @@ -66,7 +66,7 @@ # mergeSchema needed) and scaling the amount-driven features. Stage it to a separate table # first so the append does not read from the table it is writing to; serverless does not allow # cache/checkpoint, and a staging table is the clean way to break that dependency. -stage_table = f"{catalog_name}.{user_schema}._seed_drift_stage" +stage_table = f"{catalog_name}.{ml_schema}._seed_drift_stage" ( spark.read.table(predictions_table) .limit(num_rows) diff --git a/src/training/notebooks/training.py b/src/training/notebooks/training.py index cdb56e2..838d773 100644 --- a/src/training/notebooks/training.py +++ b/src/training/notebooks/training.py @@ -23,28 +23,28 @@ dbutils.widgets.text("catalog_name", "adoption_workshop") dbutils.widgets.text("gold_schema", "fraud_gold") -dbutils.widgets.text("user_schema", "") +dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("experiment_name", "") dbutils.widgets.text("model_name", "") dbutils.widgets.text("deployment_job_id", "") catalog_name = dbutils.widgets.get("catalog_name") gold_schema = dbutils.widgets.get("gold_schema") -user_schema = dbutils.widgets.get("user_schema") +ml_schema = dbutils.widgets.get("ml_schema") from pyspark.sql import functions as F # Interactive fallback: jobs pass the resolved personal schema; running standalone derives # the same per-user name the bundle uses (dev__fraud) so runs stay isolated. -if not user_schema: +if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - user_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud" # Labels come from the shared gold table; features come from the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" -card_feature_table = f"{catalog_name}.{user_schema}.card_features" -client_feature_table = f"{catalog_name}.{user_schema}.client_features" +card_feature_table = f"{catalog_name}.{ml_schema}.card_features" +client_feature_table = f"{catalog_name}.{ml_schema}.client_features" # Sensible defaults so the notebook is runnable interactively (outside the DAB job), # where the experiment/model widgets may be empty. @@ -52,7 +52,7 @@ experiment_name = ( dbutils.widgets.get("experiment_name") or f"/Users/{current_user}/mlops-workshop-fraud" ) -model_name = dbutils.widgets.get("model_name") or f"{catalog_name}.{user_schema}.fraud_detection" +model_name = dbutils.widgets.get("model_name") or f"{catalog_name}.{ml_schema}.fraud_detection" print(f"Card features: {card_feature_table}") print(f"Client features: {client_feature_table}") @@ -131,7 +131,7 @@ # COMMAND ---------- -schema_fqn = f"{catalog_name}.{user_schema}" +schema_fqn = f"{catalog_name}.{ml_schema}" # Cap the training data at a random sample so the notebook runs in a couple of minutes. # Fraud is extremely rare, so a random sample (not the first N rows) is what keeps enough