Fix/deployment catalog widget

README.md

@@ -436,7 +436,7 @@ The bundle declares the schema with a **base name** in [resources/schemas-resour
 ```yaml
 resources:
   schemas:
-    user_workspace:
+    ml_workspace:
       catalog_name: ${var.catalog_name}
       name: fraud
 ```
@@ -451,17 +451,17 @@ what the bundle created:
 
 ```yaml
 base_parameters:
-  gold_schema: ${var.gold_schema}                      # shared read  -> fraud_gold
-  user_schema: ${resources.schemas.user_workspace.name} # personal write -> dev_<you>_fraud
+  gold_schema: ${var.gold_schema}                      # shared read    -> fraud_gold
+  ml_schema: ${resources.schemas.ml_workspace.name}    # personal write -> dev_<you>_fraud
 ```
 
 and the model name follows the same schema:
-`${var.catalog_name}.${resources.schemas.user_workspace.name}.fraud_detection`.
+`${var.catalog_name}.${resources.schemas.ml_workspace.name}.fraud_detection`.
 
 > **Why reference the resource, not a plain variable?** Development mode prefixes the schema
 > *resource* but not a hand-written variable, so a variable like `${short_name}_fraud` would
 > not match the schema the bundle actually creates. Referencing
-> `${resources.schemas.user_workspace.name}` guarantees notebooks and the provisioned schema
+> `${resources.schemas.ml_workspace.name}` guarantees notebooks and the provisioned schema
 > use the identical name.
 
 ### Attendee flow

databricks.yml

@@ -16,18 +16,28 @@ variables:
     default: /Users/${workspace.current_user.userName}/${bundle.target}-mlops-workshop-fraud
   model_name:
     description: Unity Catalog registered model name (in the per-user workspace schema).
-    default: ${var.catalog_name}.${var.user_schema}.fraud_detection
+    default: ${var.catalog_name}.${var.ml_schema}.fraud_detection
   catalog_name:
     description: Unity Catalog catalog for the environment (dev/staging/prod).
     default: adoption_workshop
-  user_schema:
+  ml_schema:
     description: >-
-      Per-user ML outputs schema. Resolves to the user_workspace schema resource, so it
-      picks up development mode's prefix (base `fraud` -> `dev_<user>_fraud`).
-    default: ${resources.schemas.user_workspace.name}
+      ML outputs schema (feature tables, model, predictions, monitoring). Resolves to the
+      ml_workspace schema resource: development mode prefixes the base `fraud` to
+      `dev_<user>_fraud` (per-user); the clean dev/staging/prod targets use the shared `fraud`.
+    default: ${resources.schemas.ml_workspace.name}
   gold_schema:
     description: Shared, read-only gold schema holding the source consumption tables.
     default: fraud_gold
+  silver_schema:
+    description: Shared silver schema (cleaned, typed, joined). Used by the instructor ingestion job.
+    default: fraud_silver
+  bronze_schema:
+    description: Shared bronze schema (raw ingested tables). Used by the instructor ingestion job.
+    default: fraud_bronze
+  landing_schema:
+    description: Landing schema holding the raw_data volume of source files. Instructor ingestion job.
+    default: fraud_landing
   auto_approve:
     description: >-
       Whether the deployment job's Approval_Check auto-approves (dev) or requires a human
@@ -67,11 +77,11 @@ include:
 # landing volume). Declared ONLY on the clean (non-development) targets so that:
 #   * a `bundle deploy -t personal` (development mode) NEVER creates them. Attendees read
 #     the shared, instructor-seeded schemas and write only to their own
-#     `dev_<user>_fraud` (user_workspace). No empty per-user copies.
+#     `dev_<user>_fraud` (ml_workspace). No empty per-user copies.
 #   * a clean deploy (`-t dev` / `-t staging` / `-t prod`, no name-prefixing) provisions
 #     them literally as the shared `fraud_landing` / `fraud_bronze` / `fraud_silver` /
 #     `fraud_gold` + volume in that target's catalog.
-# `fraud_gold` is the shared source that everyone reads; the per-user `user_workspace`
+# `fraud_gold` is the shared source that everyone reads; the per-user `ml_workspace`
 # schema is separately named `fraud` (ML outputs), so there is no collision on the clean
 # targets.
 # These resources only create the empty schemas/volume; the actual data load

resources/batch-inference-workflow.yml

@@ -14,4 +14,4 @@ resources:
               model_name: ${var.model_name}
               catalog_name: ${var.catalog_name}
               gold_schema: ${var.gold_schema}
-              user_schema: ${var.user_schema}
+              ml_schema: ${var.ml_schema}

resources/data-ingestion-workflow.yml

@@ -13,9 +13,9 @@ resources:
             notebook_path: ../${var.code_root}/data_ingestion/notebooks/load_landing_to_bronze.py
             base_parameters:
               catalog: ${var.catalog_name}
-              bronze_schema: fraud_bronze
+              bronze_schema: ${var.bronze_schema}
               landing_catalog: ${var.catalog_name}
-              landing_schema: fraud_landing
+              landing_schema: ${var.landing_schema}
               landing_volume: raw_data
         - task_key: bronze_to_silver
           depends_on:
@@ -24,9 +24,9 @@ resources:
             notebook_path: ../${var.code_root}/data_ingestion/notebooks/load_bronze_to_silver.py
             base_parameters:
               catalog: ${var.catalog_name}
-              bronze_schema: fraud_bronze
-              silver_schema: fraud_silver
-              gold_schema: fraud_gold
+              bronze_schema: ${var.bronze_schema}
+              silver_schema: ${var.silver_schema}
+              gold_schema: ${var.gold_schema}
       # In production the medallion pipeline runs on a schedule (or a file-arrival trigger)
       # so bronze/silver/gold stay fresh as new transactions land. It's commented out for
       # the workshop, where the shared data is seeded once against the clean `dev` target

resources/deployment-job-workflow.yml

@@ -34,6 +34,7 @@ resources:
           notebook_task:
             notebook_path: ../${var.code_root}/deployment/model_deployment/notebooks/evaluation.py
             base_parameters:
+              catalog_name: ${var.catalog_name}
               metric: roc_auc
               baseline: "0.65"
 
@@ -57,3 +58,6 @@ resources:
           existing_cluster_id: ${var.shared_cluster_id}
           notebook_task:
             notebook_path: ../${var.code_root}/deployment/model_deployment/notebooks/deployment.py
+            base_parameters:
+              catalog_name: ${var.catalog_name}
+              ml_schema: ${var.ml_schema}

resources/feature-engineering-workflow.yml

@@ -15,4 +15,4 @@ resources:
             base_parameters:
               catalog_name: ${var.catalog_name}
               gold_schema: ${var.gold_schema}
-              user_schema: ${var.user_schema}
+              ml_schema: ${var.ml_schema}

resources/model-training-workflow.yml

@@ -21,7 +21,7 @@ resources:
               model_name: ${var.model_name}
               catalog_name: ${var.catalog_name}
               gold_schema: ${var.gold_schema}
-              user_schema: ${var.user_schema}
+              ml_schema: ${var.ml_schema}
               # Connect this model to its deployment job so registering a new version
               # auto-triggers the governed evaluate -> approve -> deploy pipeline. The
               # validation + serving that used to be separate tasks now live in that job.

resources/monitoring-resource.yml

@@ -25,8 +25,8 @@ resources:
     # Batch path: the weekly fraud report. Labels are joined, so this monitor tracks
     # model quality (accuracy/precision/recall/f1) as well as drift.
     batch_monitor:
-      table_name: ${var.catalog_name}.${resources.schemas.user_workspace.name}.fraud_predictions
-      output_schema_name: ${var.catalog_name}.${resources.schemas.user_workspace.name}
+      table_name: ${var.catalog_name}.${resources.schemas.ml_workspace.name}.fraud_predictions
+      output_schema_name: ${var.catalog_name}.${resources.schemas.ml_workspace.name}
       assets_dir: /Workspace/Users/${workspace.current_user.userName}/lakehouse_monitoring
       inference_log:
         granularities: [1 day]
@@ -42,8 +42,8 @@ resources:
     # Online path: ad-hoc checks captured by the serving endpoint's AI Gateway inference
     # table, unpacked into the same shape. Labels lag, so this leads with drift.
     online_monitor:
-      table_name: ${var.catalog_name}.${resources.schemas.user_workspace.name}.fraud_serving_inference
-      output_schema_name: ${var.catalog_name}.${resources.schemas.user_workspace.name}
+      table_name: ${var.catalog_name}.${resources.schemas.ml_workspace.name}.fraud_serving_inference
+      output_schema_name: ${var.catalog_name}.${resources.schemas.ml_workspace.name}
       assets_dir: /Workspace/Users/${workspace.current_user.userName}/lakehouse_monitoring
       inference_log:
         granularities: [1 day]
@@ -68,7 +68,7 @@ resources:
           notebook_task:
             notebook_path: ../${var.code_root}/monitoring/notebooks/metric_violation_check.py
             base_parameters:
-              table_name_under_monitor: ${var.catalog_name}.${resources.schemas.user_workspace.name}.fraud_predictions
+              table_name_under_monitor: ${var.catalog_name}.${resources.schemas.ml_workspace.name}.fraud_predictions
               metric_to_monitor: f1_score.macro
               metric_violation_threshold: "0.20"
               num_evaluation_windows: "5"
@@ -88,7 +88,7 @@ resources:
           notebook_task:
             notebook_path: ../${var.code_root}/monitoring/notebooks/feature_drift_check.py
             base_parameters:
-              table_name_under_monitor: ${var.catalog_name}.${resources.schemas.user_workspace.name}.fraud_predictions
+              table_name_under_monitor: ${var.catalog_name}.${resources.schemas.ml_workspace.name}.fraud_predictions
               features_to_monitor: amount,credit_score,credit_limit,yearly_income,current_age,num_cards_issued,is_night,is_online,is_high_risk_mcc,amount_to_income_ratio,amount_to_credit_limit_ratio
               drift_metric: population_stability_index
               drift_violation_threshold: "0.20"

resources/schemas-resource.yml

@@ -3,13 +3,13 @@
 # for the ML artifacts they produce (feature tables, model, predictions, monitoring).
 # In production mode (no prefix) it resolves to the shared `fraud`. Reads always come
 # from the shared medallion schemas (fraud_bronze / fraud_silver / fraud_gold). Jobs
-# reference ${resources.schemas.user_workspace.name} so notebooks write to exactly this
+# reference ${resources.schemas.ml_workspace.name} so notebooks write to exactly this
 # schema. The layer ("gold") is intentionally NOT in the name: this schema holds ML
 # outputs (model, predictions, monitoring), not a mirror of the shared gold layer.
 # Docs: Bundle deployment modes (dev prefixing) https://learn.microsoft.com/azure/databricks/dev-tools/bundles/deployment-modes
 resources:
   schemas:
-    user_workspace:
+    ml_workspace:
       catalog_name: ${var.catalog_name}
       name: fraud
       comment: Per-user ML outputs (feature tables, model, predictions, monitoring). Shared in prod.

resources/serving-log-processing-workflow.yml

@@ -18,7 +18,7 @@ resources:
             base_parameters:
               catalog_name: ${var.catalog_name}
               gold_schema: ${var.gold_schema}
-              user_schema: ${var.user_schema}
+              ml_schema: ${var.ml_schema}
       # In production you'd run this on a schedule so the online inference table stays
       # fresh ahead of the monitor refresh. It's commented out for the workshop so nothing
       # auto-runs per attendee; the instructor triggers it on demand with

solution/deployment/batch_inference/notebooks/batch_inference.py

@@ -25,30 +25,30 @@
 
 dbutils.widgets.text("catalog_name", "adoption_workshop")
 dbutils.widgets.text("gold_schema", "fraud_gold")
-dbutils.widgets.text("user_schema", "")
+dbutils.widgets.text("ml_schema", "")
 dbutils.widgets.text("model_name", "")
 dbutils.widgets.text("model_alias", "champion")
 
 catalog_name = dbutils.widgets.get("catalog_name")
 gold_schema = dbutils.widgets.get("gold_schema")
-user_schema = dbutils.widgets.get("user_schema")
+ml_schema = dbutils.widgets.get("ml_schema")
 
 from pyspark.sql import functions as F
 
 # Interactive fallback: jobs pass the resolved personal schema; running standalone derives
 # the same per-user name the bundle uses (dev_<short>_fraud) so runs stay isolated.
-if not user_schema:
+if not ml_schema:
     _user = spark.range(1).select(F.current_user()).first()[0]
     _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0])
-    user_schema = f"dev_{_short}_fraud"
+    ml_schema = f"dev_{_short}_fraud"
 
 model_alias = dbutils.widgets.get("model_alias")
 # The batch-inference job passes the full three-level model name; fall back for interactive runs.
-model_name = dbutils.widgets.get("model_name") or f"{catalog_name}.{user_schema}.fraud_detection"
+model_name = dbutils.widgets.get("model_name") or f"{catalog_name}.{ml_schema}.fraud_detection"
 
 # Read the transactions to score from the shared gold source; write predictions to the personal schema.
 source_table = f"{catalog_name}.{gold_schema}.transactions_enriched"
-predictions_table = f"{catalog_name}.{user_schema}.fraud_predictions"
+predictions_table = f"{catalog_name}.{ml_schema}.fraud_predictions"
 
 print(f"Model:       {model_name}@{model_alias}")
 print(f"Score from:  {source_table}")

solution/deployment/model_deployment/notebooks/deployment.py

@@ -21,19 +21,30 @@
 # The deployment job injects model_name + model_version as JOB-level parameters.
 dbutils.widgets.text("model_name", "")
 dbutils.widgets.text("model_version", "")
+dbutils.widgets.text("catalog_name", "adoption_workshop")
+dbutils.widgets.text("ml_schema", "")
 
 model_name = dbutils.widgets.get("model_name")
 model_version = dbutils.widgets.get("model_version")
+catalog_name = dbutils.widgets.get("catalog_name")
+ml_schema = dbutils.widgets.get("ml_schema")
 assert model_name and model_version, (
     "model_name and model_version are injected by the deployment job."
 )
 
-# The three-level model name carries the catalog + personal schema.
-catalog_name, user_schema, _ = model_name.split(".")
-endpoint_name = f"{user_schema}_fraud"
+from pyspark.sql import functions as F
+
+# Interactive fallback: jobs pass the resolved personal schema; running standalone derives
+# the same per-user name the bundle uses (dev_<short>_fraud) so runs stay isolated.
+if not ml_schema:
+    _user = spark.range(1).select(F.current_user()).first()[0]
+    _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0])
+    ml_schema = f"dev_{_short}_fraud"
+
+endpoint_name = f"{ml_schema}_fraud"
 online_store_name = "fraud-workshop-online"
-card_feature_table = f"{catalog_name}.{user_schema}.card_features"
-client_feature_table = f"{catalog_name}.{user_schema}.client_features"
+card_feature_table = f"{catalog_name}.{ml_schema}.card_features"
+client_feature_table = f"{catalog_name}.{ml_schema}.client_features"
 
 print(f"Deploying {model_name} v{model_version} -> endpoint {endpoint_name}")
 
@@ -128,7 +139,7 @@ def wait_online_store(name, minutes=20):
 ]
 inference_table = AiGatewayInferenceTableConfig(
     catalog_name=catalog_name,
-    schema_name=user_schema,
+    schema_name=ml_schema,
     table_name_prefix="fraud_serving",
     enabled=True,
 )

solution/deployment/model_deployment/notebooks/evaluation.py

@@ -24,11 +24,13 @@
 # The deployment job injects model_name + model_version as JOB-level parameters.
 dbutils.widgets.text("model_name", "")
 dbutils.widgets.text("model_version", "")
+dbutils.widgets.text("catalog_name", "adoption_workshop")
 dbutils.widgets.text("metric", "roc_auc")
 dbutils.widgets.text("baseline", "0.65")
 
 model_name = dbutils.widgets.get("model_name")
 model_version = dbutils.widgets.get("model_version")
+catalog_name = dbutils.widgets.get("catalog_name")
 metric = dbutils.widgets.get("metric")
 baseline = float(dbutils.widgets.get("baseline"))
 
@@ -54,7 +56,6 @@
 # lookups + on-demand functions recorded at training, and the served model returns a fraud
 # probability, so a real ROC AUC can be computed. In production this holdout would be a
 # curated, leakage-controlled evaluation table; here it is sampled from the shared gold source.
-catalog_name = model_name.split(".")[0]
 gold_table = f"{catalog_name}.fraud_gold.transactions_enriched"
 
 eval_spine = (

solution/deployment/model_deployment/notebooks/query_endpoint.py

@@ -21,24 +21,24 @@
 
 dbutils.widgets.text("catalog_name", "adoption_workshop")
 dbutils.widgets.text("gold_schema", "fraud_gold")
-dbutils.widgets.text("user_schema", "")
+dbutils.widgets.text("ml_schema", "")
 dbutils.widgets.text("num_requests", "50")
 
 catalog_name = dbutils.widgets.get("catalog_name")
 gold_schema = dbutils.widgets.get("gold_schema")
-user_schema = dbutils.widgets.get("user_schema")
+ml_schema = dbutils.widgets.get("ml_schema")
 num_requests = int(dbutils.widgets.get("num_requests"))
 
 from pyspark.sql import functions as F
 
 # Interactive fallback: derive the same per-user name the bundle uses (dev_<short>_fraud).
-if not user_schema:
+if not ml_schema:
     _user = spark.range(1).select(F.current_user()).first()[0]
     _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0])
-    user_schema = f"dev_{_short}_fraud"
+    ml_schema = f"dev_{_short}_fraud"
 
 # One endpoint per user, named after the personal schema (matches the deployment job).
-endpoint_name = f"{user_schema}_fraud"
+endpoint_name = f"{ml_schema}_fraud"
 source_table = f"{catalog_name}.{gold_schema}.transactions_enriched"
 print(f"Endpoint: {endpoint_name}")
 print(f"Sample:   {num_requests} rows from {source_table}")

solution/feature_engineering/notebooks/build_features.py

@@ -26,25 +26,25 @@
 
 dbutils.widgets.text("catalog_name", "adoption_workshop")
 dbutils.widgets.text("gold_schema", "fraud_gold")
-dbutils.widgets.text("user_schema", "")
+dbutils.widgets.text("ml_schema", "")
 
 catalog_name = dbutils.widgets.get("catalog_name")
 gold_schema = dbutils.widgets.get("gold_schema")
-user_schema = dbutils.widgets.get("user_schema")
+ml_schema = dbutils.widgets.get("ml_schema")
 
 from pyspark.sql import functions as F
 
 # Interactive fallback: jobs pass the resolved personal schema; running standalone derives
 # the same per-user name the bundle uses (dev_<short>_fraud) so runs stay isolated.
-if not user_schema:
+if not ml_schema:
     _user = spark.range(1).select(F.current_user()).first()[0]
     _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0])
-    user_schema = f"dev_{_short}_fraud"
+    ml_schema = f"dev_{_short}_fraud"
 
 # Read from the shared gold source; write to the personal schema.
 source_table = f"{catalog_name}.{gold_schema}.transactions_enriched"
-card_feature_table = f"{catalog_name}.{user_schema}.card_features"
-client_feature_table = f"{catalog_name}.{user_schema}.client_features"
+card_feature_table = f"{catalog_name}.{ml_schema}.card_features"
+client_feature_table = f"{catalog_name}.{ml_schema}.client_features"
 
 # COMMAND ----------
 
@@ -140,7 +140,7 @@ def publish(name, df, key, description):
 # On-demand feature functions are UC Python UDFs. Training and serving call the same
 # functions, so there is no train/serve skew. Four are provided; the first (ff_is_night) is
 # the exercise.
-schema_fqn = f"{catalog_name}.{user_schema}"
+schema_fqn = f"{catalog_name}.{ml_schema}"
 
 # TODO-BEGIN: register the ff_is_night on-demand feature function as a UC Python UDF
 # HINT: CREATE OR REPLACE FUNCTION {schema_fqn}.ff_is_night(transaction_hour INT) RETURNS BOOLEAN