diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 3e1eca9..381cbdb 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -107,6 +107,15 @@ jobs: DATABRICKS_AUTH_TYPE: github-oidc DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }} DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID }} + # staging is mode: production, so its jobs run_as the service principal. Feed the SP + # application id (same client id used to authenticate) to the bundle variable. + # Casing is deliberate: the Databricks CLI reads a bundle variable from an env var named + # `BUNDLE_VAR_` where `` must match the variable EXACTLY. Our variable is the + # lowercase `cicd_service_principal` (in databricks.yml), so the env var is the fixed + # uppercase prefix `BUNDLE_VAR_` + the lowercase name. Upper-casing the tail would make the + # CLI look for a variable `CICD_SERVICE_PRINCIPAL`, which does not exist, and the value + # would be ignored (staging's run_as would then fail to resolve). + BUNDLE_VAR_cicd_service_principal: ${{ secrets.DATABRICKS_CLIENT_ID }} steps: - uses: actions/checkout@v4 - uses: databricks/setup-cli@v0.9.0 diff --git a/databricks.yml b/databricks.yml index c63b540..7d76584 100644 --- a/databricks.yml +++ b/databricks.yml @@ -7,10 +7,6 @@ bundle: engine: direct variables: - shared_cluster_id: - description: Existing shared cluster used by workshop jobs. - lookup: - cluster: "Shared Compute - ML Workshop" experiment_name: description: MLflow experiment for model training runs (per-user path). default: /Users/${workspace.current_user.userName}/${bundle.target}-mlops-workshop-fraud @@ -21,7 +17,7 @@ variables: default: fraud_detection catalog_name: description: Unity Catalog catalog for the environment (dev/staging/prod). - default: adoption_workshop + default: dev ml_schema: description: >- ML outputs schema (feature tables, model, predictions, monitoring). Resolves to the @@ -54,6 +50,12 @@ variables: notebook_path is `../${var.code_root}/...`, so the two trees must mirror each other (the generator guarantees this). default: src + cicd_service_principal: + description: >- + Application (client) ID of the CI/CD service principal that deploys and runs the clean + shared targets. CI supplies the real value (via --var or BUNDLE_VAR_cicd_service_principal) + when deploying staging/prod. Empty by default so the personal target is unaffected. + default: "" include: # All DAB resources (jobs, models, experiments, monitors) live in ./resources. @@ -102,8 +104,7 @@ targets: mode: development default: true variables: - # Temporary override to adoption_workshop for the workshop; revert to dev later. - catalog_name: adoption_workshop + catalog_name: dev # workspace: # host: https://adb-xxxxxxxxxxxxxxxx.x.azuredatabricks.net # add your workspace URL if deploying from a local CLI @@ -140,13 +141,20 @@ targets: # host: https://adb-xxxxxxxxxxxxxxxx.x.azuredatabricks.net # add your workspace URL if deploying from a local CLI staging: + mode: production + run_as: + service_principal_name: ${var.cicd_service_principal} variables: catalog_name: staging auto_approve: "false" code_root: solution resources: *shared_infra - # workspace: - # host: https://adb-xxxxxxxxxxxxxxxx.x.azuredatabricks.net # add your workspace URL if deploying from a local CLI + workspace: + # host: https://adb-xxxxxxxxxxxxxxxx.x.azuredatabricks.net # add your workspace URL if deploying from a local CLI + # Production mode needs a fixed, non-user-scoped root_path so exactly one copy is + # deployed and it is not tied to the deploying user. CI deploys and runs this target as + # the cicd_service_principal. + root_path: /Workspace/mlops-workshop/.bundle/${bundle.name}/${bundle.target} prod: mode: production diff --git a/resources/batch-inference-workflow.yml b/resources/batch-inference-workflow.yml index 1a8e0cc..b0d226f 100644 --- a/resources/batch-inference-workflow.yml +++ b/resources/batch-inference-workflow.yml @@ -7,7 +7,6 @@ resources: name: ${bundle.target}-mlops-workshop-batch-inference tasks: - task_key: batch_inference - existing_cluster_id: ${var.shared_cluster_id} notebook_task: notebook_path: ../${var.code_root}/deployment/batch_inference/notebooks/batch_inference.py base_parameters: diff --git a/resources/deployment-job-workflow.yml b/resources/deployment-job-workflow.yml index a4ebea6..fa6d745 100644 --- a/resources/deployment-job-workflow.yml +++ b/resources/deployment-job-workflow.yml @@ -29,7 +29,6 @@ resources: default: "" tasks: - task_key: Evaluation - existing_cluster_id: ${var.shared_cluster_id} max_retries: 0 notebook_task: notebook_path: ../${var.code_root}/deployment/model_deployment/notebooks/evaluation.py @@ -43,7 +42,6 @@ resources: depends_on: - task_key: Evaluation max_retries: 0 - existing_cluster_id: ${var.shared_cluster_id} notebook_task: notebook_path: ../${var.code_root}/deployment/model_deployment/notebooks/approval.py base_parameters: @@ -56,7 +54,6 @@ resources: depends_on: - task_key: Approval_Check max_retries: 0 - existing_cluster_id: ${var.shared_cluster_id} notebook_task: notebook_path: ../${var.code_root}/deployment/model_deployment/notebooks/deployment.py base_parameters: diff --git a/resources/feature-engineering-workflow.yml b/resources/feature-engineering-workflow.yml index ae843de..b34abad 100644 --- a/resources/feature-engineering-workflow.yml +++ b/resources/feature-engineering-workflow.yml @@ -9,7 +9,6 @@ resources: name: ${bundle.target}-mlops-workshop-feature-engineering tasks: - task_key: build_features - existing_cluster_id: ${var.shared_cluster_id} notebook_task: notebook_path: ../${var.code_root}/feature_engineering/notebooks/build_features.py base_parameters: diff --git a/resources/model-training-workflow.yml b/resources/model-training-workflow.yml index 8037ef3..e5a3425 100644 --- a/resources/model-training-workflow.yml +++ b/resources/model-training-workflow.yml @@ -8,7 +8,6 @@ resources: name: ${bundle.target}-mlops-workshop-training tasks: - task_key: train - existing_cluster_id: ${var.shared_cluster_id} notebook_task: notebook_path: ../${var.code_root}/training/notebooks/training.py base_parameters: diff --git a/resources/monitoring-resource.yml b/resources/monitoring-resource.yml index 6cea809..afd5852 100644 --- a/resources/monitoring-resource.yml +++ b/resources/monitoring-resource.yml @@ -64,7 +64,6 @@ resources: name: ${bundle.target}-mlops-workshop-retraining tasks: - task_key: monitored_metric_violation_check - existing_cluster_id: ${var.shared_cluster_id} notebook_task: notebook_path: ../${var.code_root}/monitoring/notebooks/metric_violation_check.py base_parameters: diff --git a/resources/schemas-resource.yml b/resources/schemas-resource.yml index 4e5e452..7d207c2 100644 --- a/resources/schemas-resource.yml +++ b/resources/schemas-resource.yml @@ -1,7 +1,7 @@ -# Per-user ML workspace schema. Declared with a BASE name (`fraud`); in development -# mode the bundle prefixes it to `dev__fraud`, giving each user their own schema +# Per-user ML workspace schema. Declared with a BASE name (`fraud_ml`); in development +# mode the bundle prefixes it to `dev__fraud_ml`, giving each user their own schema # for the ML artifacts they produce (feature tables, model, predictions, monitoring). -# In production mode (no prefix) it resolves to the shared `fraud`. Reads always come +# In production mode (no prefix) it resolves to the shared `fraud_ml`. Reads always come # from the shared medallion schemas (fraud_bronze / fraud_silver / fraud_gold). Jobs # reference ${resources.schemas.ml_workspace.name} so notebooks write to exactly this # schema. The layer ("gold") is intentionally NOT in the name: this schema holds ML @@ -11,5 +11,5 @@ resources: schemas: ml_workspace: catalog_name: ${var.catalog_name} - name: fraud + name: fraud_ml comment: Per-user ML outputs (feature tables, model, predictions, monitoring). Shared in prod. diff --git a/resources/serving-log-processing-workflow.yml b/resources/serving-log-processing-workflow.yml index 7a0ddc5..e6dd85e 100644 --- a/resources/serving-log-processing-workflow.yml +++ b/resources/serving-log-processing-workflow.yml @@ -12,7 +12,6 @@ resources: name: ${bundle.target}-mlops-workshop-serving-log-processing tasks: - task_key: process_serving_logs - existing_cluster_id: ${var.shared_cluster_id} notebook_task: notebook_path: ../${var.code_root}/monitoring/notebooks/process_serving_logs.py base_parameters: diff --git a/solution/data_ingestion/notebooks/load_bronze_to_silver.py b/solution/data_ingestion/notebooks/load_bronze_to_silver.py index 9c8e5d2..78f0bdd 100644 --- a/solution/data_ingestion/notebooks/load_bronze_to_silver.py +++ b/solution/data_ingestion/notebooks/load_bronze_to_silver.py @@ -29,7 +29,7 @@ # COMMAND ---------- -dbutils.widgets.text("catalog", "adoption_workshop") +dbutils.widgets.text("catalog", "dev") dbutils.widgets.text("bronze_schema", "fraud_bronze") dbutils.widgets.text("silver_schema", "fraud_silver") dbutils.widgets.text("gold_schema", "fraud_gold") diff --git a/solution/data_ingestion/notebooks/load_landing_to_bronze.py b/solution/data_ingestion/notebooks/load_landing_to_bronze.py index 712bc0f..dcfd536 100644 --- a/solution/data_ingestion/notebooks/load_landing_to_bronze.py +++ b/solution/data_ingestion/notebooks/load_landing_to_bronze.py @@ -28,7 +28,7 @@ # COMMAND ---------- -dbutils.widgets.text("catalog", "adoption_workshop") +dbutils.widgets.text("catalog", "dev") dbutils.widgets.text("bronze_schema", "fraud_bronze") dbutils.widgets.text("landing_catalog", "dev") dbutils.widgets.text("landing_schema", "fraud_landing") diff --git a/solution/deployment/batch_inference/notebooks/batch_inference.py b/solution/deployment/batch_inference/notebooks/batch_inference.py index 699c7ca..2dd3b27 100644 --- a/solution/deployment/batch_inference/notebooks/batch_inference.py +++ b/solution/deployment/batch_inference/notebooks/batch_inference.py @@ -23,7 +23,23 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +# MAGIC %md +# MAGIC ## Setup +# MAGIC Install the Feature Engineering client so `score_batch` can resolve the feature +# MAGIC lookups the model was trained with (no train/serve skew). + +# COMMAND ---------- + +# MAGIC %pip install -q "mlflow>=3.0" --upgrade +# MAGIC %pip install -q databricks-feature-engineering + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("gold_schema", "fraud_gold") dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("model_name", "fraud_detection") @@ -40,7 +56,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" model_alias = dbutils.widgets.get("model_alias") model_name = dbutils.widgets.get("model_name") diff --git a/solution/deployment/model_deployment/notebooks/approval.py b/solution/deployment/model_deployment/notebooks/approval.py index 6346aff..09114b0 100644 --- a/solution/deployment/model_deployment/notebooks/approval.py +++ b/solution/deployment/model_deployment/notebooks/approval.py @@ -20,12 +20,20 @@ # COMMAND ---------- +# MAGIC %pip install -q "mlflow>=3.0" --upgrade + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + # The deployment job injects the FULL three-level model name (catalog.schema.model) into # `model_name`, plus the `model_version` it is gating. Both are empty when this notebook is # run interactively, so the fallback below rebuilds them. dbutils.widgets.text("model_name", "") dbutils.widgets.text("model_version", "") -dbutils.widgets.text("catalog_name", "adoption_workshop") +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("approval_tag_name", "") dbutils.widgets.text("auto_approve", "false") @@ -54,7 +62,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" if not uc_model_name: uc_model_name = f"{catalog_name}.{ml_schema}.fraud_detection" if not model_version: diff --git a/solution/deployment/model_deployment/notebooks/deployment.py b/solution/deployment/model_deployment/notebooks/deployment.py index 0eed33c..cf35597 100644 --- a/solution/deployment/model_deployment/notebooks/deployment.py +++ b/solution/deployment/model_deployment/notebooks/deployment.py @@ -18,12 +18,22 @@ # COMMAND ---------- +# MAGIC %pip install -q "mlflow>=3.0" --upgrade +# MAGIC %pip install -q --upgrade databricks-sdk +# MAGIC %pip install -q databricks-feature-engineering + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + # The deployment job injects the FULL three-level model name (catalog.schema.model) into # `model_name`, plus the `model_version` that triggered it. Both are empty when this notebook # is run interactively, so the fallback below rebuilds them. dbutils.widgets.text("model_name", "") dbutils.widgets.text("model_version", "") -dbutils.widgets.text("catalog_name", "adoption_workshop") +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("ml_schema", "") uc_model_name = dbutils.widgets.get("model_name").strip() @@ -47,7 +57,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" if not uc_model_name: uc_model_name = f"{catalog_name}.{ml_schema}.fraud_detection" if not model_version: diff --git a/solution/deployment/model_deployment/notebooks/evaluation.py b/solution/deployment/model_deployment/notebooks/evaluation.py index 493150f..d07f40a 100644 --- a/solution/deployment/model_deployment/notebooks/evaluation.py +++ b/solution/deployment/model_deployment/notebooks/evaluation.py @@ -21,12 +21,21 @@ # COMMAND ---------- +# MAGIC %pip install -q "mlflow>=3.0" --upgrade +# MAGIC %pip install -q databricks-feature-engineering + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + # The deployment job injects the FULL three-level model name (catalog.schema.model) into # `model_name`, plus the `model_version` that triggered it. Both are empty when this notebook # is run interactively, so the fallback below rebuilds them. dbutils.widgets.text("model_name", "") dbutils.widgets.text("model_version", "") -dbutils.widgets.text("catalog_name", "adoption_workshop") +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("metric", "roc_auc") dbutils.widgets.text("baseline", "0.65") @@ -54,7 +63,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" if not uc_model_name: uc_model_name = f"{catalog_name}.{ml_schema}.fraud_detection" if not model_version: @@ -76,12 +85,22 @@ # COMMAND ---------- +import os + import mlflow from databricks.feature_engineering import FeatureEngineeringClient from mlflow.tracking import MlflowClient from pyspark.sql import functions as F from sklearn.metrics import roc_auc_score +# Serverless workaround: fe.score_batch runs the model via mlflow.pyfunc.spark_udf, which on +# serverless ships the model through the DBConnect addArtifact path and then version-checks the +# UDF sandbox with Version(runtime_version). The serverless sandbox reports a non-PEP440 tag +# (e.g. '18.x-photon-scala2'), so that parse throws InvalidVersion. This mlflow flag skips the +# addArtifact path and has each executor pull the model straight from the artifact store, +# avoiding the broken check. Remove once the serverless sandbox reports a parseable version. +os.environ["_MLFLOW_SPARK_UDF_SERVERLESS_SKIP_DBCONNECT_ARTIFACT"] = "true" + mlflow.set_registry_uri("databricks-uc") client = MlflowClient() fe = FeatureEngineeringClient() diff --git a/solution/deployment/model_deployment/notebooks/query_endpoint.py b/solution/deployment/model_deployment/notebooks/query_endpoint.py index 9bf1ccb..a2b55fd 100644 --- a/solution/deployment/model_deployment/notebooks/query_endpoint.py +++ b/solution/deployment/model_deployment/notebooks/query_endpoint.py @@ -19,7 +19,7 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("gold_schema", "fraud_gold") dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("num_requests", "50") @@ -35,7 +35,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" # One endpoint per user, named after the personal schema (matches the deployment job). endpoint_name = f"{ml_schema}_fraud" diff --git a/solution/exploration/notebooks/eda.py b/solution/exploration/notebooks/eda.py index 2621cac..0459ca2 100644 --- a/solution/exploration/notebooks/eda.py +++ b/solution/exploration/notebooks/eda.py @@ -15,7 +15,7 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("gold_schema", "fraud_gold") catalog_name = dbutils.widgets.get("catalog_name") diff --git a/solution/feature_engineering/notebooks/build_features.py b/solution/feature_engineering/notebooks/build_features.py index fec875f..22577d8 100644 --- a/solution/feature_engineering/notebooks/build_features.py +++ b/solution/feature_engineering/notebooks/build_features.py @@ -24,7 +24,22 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +# MAGIC %md +# MAGIC ## Setup +# MAGIC Install the Feature Engineering client. On serverless this package is not +# MAGIC pre-installed, so we add it and restart Python. (No-op on ML Runtime clusters.) + +# COMMAND ---------- + +# MAGIC %pip install -q databricks-feature-engineering + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("gold_schema", "fraud_gold") dbutils.widgets.text("ml_schema", "") @@ -39,7 +54,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" # Read from the shared gold source; write to the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" diff --git a/solution/monitoring/notebooks/monitoring.py b/solution/monitoring/notebooks/monitoring.py index 601bddf..49acbca 100644 --- a/solution/monitoring/notebooks/monitoring.py +++ b/solution/monitoring/notebooks/monitoring.py @@ -23,7 +23,16 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +# MAGIC %pip install -q --upgrade databricks-sdk +# MAGIC + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("ml_schema", "") catalog_name = dbutils.widgets.get("catalog_name") @@ -36,7 +45,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" batch_table = f"{catalog_name}.{ml_schema}.fraud_predictions" online_table = f"{catalog_name}.{ml_schema}.fraud_serving_inference" diff --git a/solution/monitoring/notebooks/process_serving_logs.py b/solution/monitoring/notebooks/process_serving_logs.py index 819de39..6a74f08 100644 --- a/solution/monitoring/notebooks/process_serving_logs.py +++ b/solution/monitoring/notebooks/process_serving_logs.py @@ -31,7 +31,7 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("gold_schema", "fraud_gold") dbutils.widgets.text("ml_schema", "") @@ -46,7 +46,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" payload_table = f"{catalog_name}.{ml_schema}.fraud_serving_payload" online_table = f"{catalog_name}.{ml_schema}.fraud_serving_inference" diff --git a/solution/monitoring/notebooks/seed_drift.py b/solution/monitoring/notebooks/seed_drift.py index d0d6530..142c2f7 100644 --- a/solution/monitoring/notebooks/seed_drift.py +++ b/solution/monitoring/notebooks/seed_drift.py @@ -23,7 +23,15 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +# MAGIC %pip install -q --upgrade databricks-sdk + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("amount_multiplier", "6.0", label="Scale amount by this to force drift") dbutils.widgets.text("num_rows", "5000", label="How many drifted rows to append") @@ -39,7 +47,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" amount_multiplier = float(dbutils.widgets.get("amount_multiplier")) num_rows = int(dbutils.widgets.get("num_rows")) diff --git a/solution/training/notebooks/training.py b/solution/training/notebooks/training.py index b97d89a..a5d7469 100644 --- a/solution/training/notebooks/training.py +++ b/solution/training/notebooks/training.py @@ -21,7 +21,25 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +# MAGIC %md +# MAGIC ## Setup +# MAGIC Upgrade MLflow to 3.x for the new registry/logged-model features. Install the +# MAGIC Feature Engineering client **without `--upgrade`** so it stays at the runtime version +# MAGIC that created the feature table (mismatched client versions can't resolve the feature +# MAGIC table). On ML Runtime the client is pre-installed; on serverless this adds it. + +# COMMAND ---------- + +# MAGIC %pip install -q "mlflow>=3.0" --upgrade +# MAGIC %pip install -q databricks-feature-engineering + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("gold_schema", "fraud_gold") dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("experiment_name", "") @@ -39,7 +57,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" # Labels come from the shared gold table; features come from the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" diff --git a/src/data_ingestion/notebooks/load_bronze_to_silver.py b/src/data_ingestion/notebooks/load_bronze_to_silver.py index 9c8e5d2..78f0bdd 100644 --- a/src/data_ingestion/notebooks/load_bronze_to_silver.py +++ b/src/data_ingestion/notebooks/load_bronze_to_silver.py @@ -29,7 +29,7 @@ # COMMAND ---------- -dbutils.widgets.text("catalog", "adoption_workshop") +dbutils.widgets.text("catalog", "dev") dbutils.widgets.text("bronze_schema", "fraud_bronze") dbutils.widgets.text("silver_schema", "fraud_silver") dbutils.widgets.text("gold_schema", "fraud_gold") diff --git a/src/data_ingestion/notebooks/load_landing_to_bronze.py b/src/data_ingestion/notebooks/load_landing_to_bronze.py index 712bc0f..dcfd536 100644 --- a/src/data_ingestion/notebooks/load_landing_to_bronze.py +++ b/src/data_ingestion/notebooks/load_landing_to_bronze.py @@ -28,7 +28,7 @@ # COMMAND ---------- -dbutils.widgets.text("catalog", "adoption_workshop") +dbutils.widgets.text("catalog", "dev") dbutils.widgets.text("bronze_schema", "fraud_bronze") dbutils.widgets.text("landing_catalog", "dev") dbutils.widgets.text("landing_schema", "fraud_landing") diff --git a/src/deployment/batch_inference/notebooks/batch_inference.py b/src/deployment/batch_inference/notebooks/batch_inference.py index 53f2309..4f2847e 100644 --- a/src/deployment/batch_inference/notebooks/batch_inference.py +++ b/src/deployment/batch_inference/notebooks/batch_inference.py @@ -23,7 +23,23 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +# MAGIC %md +# MAGIC ## Setup +# MAGIC Install the Feature Engineering client so `score_batch` can resolve the feature +# MAGIC lookups the model was trained with (no train/serve skew). + +# COMMAND ---------- + +# MAGIC %pip install -q "mlflow>=3.0" --upgrade +# MAGIC %pip install -q databricks-feature-engineering + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("gold_schema", "fraud_gold") dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("model_name", "fraud_detection") @@ -40,7 +56,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" model_alias = dbutils.widgets.get("model_alias") model_name = dbutils.widgets.get("model_name") diff --git a/src/deployment/model_deployment/notebooks/approval.py b/src/deployment/model_deployment/notebooks/approval.py index 6346aff..09114b0 100644 --- a/src/deployment/model_deployment/notebooks/approval.py +++ b/src/deployment/model_deployment/notebooks/approval.py @@ -20,12 +20,20 @@ # COMMAND ---------- +# MAGIC %pip install -q "mlflow>=3.0" --upgrade + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + # The deployment job injects the FULL three-level model name (catalog.schema.model) into # `model_name`, plus the `model_version` it is gating. Both are empty when this notebook is # run interactively, so the fallback below rebuilds them. dbutils.widgets.text("model_name", "") dbutils.widgets.text("model_version", "") -dbutils.widgets.text("catalog_name", "adoption_workshop") +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("approval_tag_name", "") dbutils.widgets.text("auto_approve", "false") @@ -54,7 +62,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" if not uc_model_name: uc_model_name = f"{catalog_name}.{ml_schema}.fraud_detection" if not model_version: diff --git a/src/deployment/model_deployment/notebooks/deployment.py b/src/deployment/model_deployment/notebooks/deployment.py index 0eed33c..cf35597 100644 --- a/src/deployment/model_deployment/notebooks/deployment.py +++ b/src/deployment/model_deployment/notebooks/deployment.py @@ -18,12 +18,22 @@ # COMMAND ---------- +# MAGIC %pip install -q "mlflow>=3.0" --upgrade +# MAGIC %pip install -q --upgrade databricks-sdk +# MAGIC %pip install -q databricks-feature-engineering + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + # The deployment job injects the FULL three-level model name (catalog.schema.model) into # `model_name`, plus the `model_version` that triggered it. Both are empty when this notebook # is run interactively, so the fallback below rebuilds them. dbutils.widgets.text("model_name", "") dbutils.widgets.text("model_version", "") -dbutils.widgets.text("catalog_name", "adoption_workshop") +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("ml_schema", "") uc_model_name = dbutils.widgets.get("model_name").strip() @@ -47,7 +57,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" if not uc_model_name: uc_model_name = f"{catalog_name}.{ml_schema}.fraud_detection" if not model_version: diff --git a/src/deployment/model_deployment/notebooks/evaluation.py b/src/deployment/model_deployment/notebooks/evaluation.py index 493150f..d07f40a 100644 --- a/src/deployment/model_deployment/notebooks/evaluation.py +++ b/src/deployment/model_deployment/notebooks/evaluation.py @@ -21,12 +21,21 @@ # COMMAND ---------- +# MAGIC %pip install -q "mlflow>=3.0" --upgrade +# MAGIC %pip install -q databricks-feature-engineering + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + # The deployment job injects the FULL three-level model name (catalog.schema.model) into # `model_name`, plus the `model_version` that triggered it. Both are empty when this notebook # is run interactively, so the fallback below rebuilds them. dbutils.widgets.text("model_name", "") dbutils.widgets.text("model_version", "") -dbutils.widgets.text("catalog_name", "adoption_workshop") +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("metric", "roc_auc") dbutils.widgets.text("baseline", "0.65") @@ -54,7 +63,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" if not uc_model_name: uc_model_name = f"{catalog_name}.{ml_schema}.fraud_detection" if not model_version: @@ -76,12 +85,22 @@ # COMMAND ---------- +import os + import mlflow from databricks.feature_engineering import FeatureEngineeringClient from mlflow.tracking import MlflowClient from pyspark.sql import functions as F from sklearn.metrics import roc_auc_score +# Serverless workaround: fe.score_batch runs the model via mlflow.pyfunc.spark_udf, which on +# serverless ships the model through the DBConnect addArtifact path and then version-checks the +# UDF sandbox with Version(runtime_version). The serverless sandbox reports a non-PEP440 tag +# (e.g. '18.x-photon-scala2'), so that parse throws InvalidVersion. This mlflow flag skips the +# addArtifact path and has each executor pull the model straight from the artifact store, +# avoiding the broken check. Remove once the serverless sandbox reports a parseable version. +os.environ["_MLFLOW_SPARK_UDF_SERVERLESS_SKIP_DBCONNECT_ARTIFACT"] = "true" + mlflow.set_registry_uri("databricks-uc") client = MlflowClient() fe = FeatureEngineeringClient() diff --git a/src/deployment/model_deployment/notebooks/query_endpoint.py b/src/deployment/model_deployment/notebooks/query_endpoint.py index 9bf1ccb..a2b55fd 100644 --- a/src/deployment/model_deployment/notebooks/query_endpoint.py +++ b/src/deployment/model_deployment/notebooks/query_endpoint.py @@ -19,7 +19,7 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("gold_schema", "fraud_gold") dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("num_requests", "50") @@ -35,7 +35,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" # One endpoint per user, named after the personal schema (matches the deployment job). endpoint_name = f"{ml_schema}_fraud" diff --git a/src/exploration/notebooks/eda.py b/src/exploration/notebooks/eda.py index 2621cac..0459ca2 100644 --- a/src/exploration/notebooks/eda.py +++ b/src/exploration/notebooks/eda.py @@ -15,7 +15,7 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("gold_schema", "fraud_gold") catalog_name = dbutils.widgets.get("catalog_name") diff --git a/src/feature_engineering/notebooks/build_features.py b/src/feature_engineering/notebooks/build_features.py index b5a39cd..158044e 100644 --- a/src/feature_engineering/notebooks/build_features.py +++ b/src/feature_engineering/notebooks/build_features.py @@ -24,7 +24,22 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +# MAGIC %md +# MAGIC ## Setup +# MAGIC Install the Feature Engineering client. On serverless this package is not +# MAGIC pre-installed, so we add it and restart Python. (No-op on ML Runtime clusters.) + +# COMMAND ---------- + +# MAGIC %pip install -q databricks-feature-engineering + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("gold_schema", "fraud_gold") dbutils.widgets.text("ml_schema", "") @@ -39,7 +54,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" # Read from the shared gold source; write to the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched" diff --git a/src/monitoring/notebooks/monitoring.py b/src/monitoring/notebooks/monitoring.py index 6fc202d..31f9315 100644 --- a/src/monitoring/notebooks/monitoring.py +++ b/src/monitoring/notebooks/monitoring.py @@ -23,7 +23,16 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +# MAGIC %pip install -q --upgrade databricks-sdk +# MAGIC + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("ml_schema", "") catalog_name = dbutils.widgets.get("catalog_name") @@ -36,7 +45,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" batch_table = f"{catalog_name}.{ml_schema}.fraud_predictions" online_table = f"{catalog_name}.{ml_schema}.fraud_serving_inference" diff --git a/src/monitoring/notebooks/process_serving_logs.py b/src/monitoring/notebooks/process_serving_logs.py index 00bc9b0..deedb3a 100644 --- a/src/monitoring/notebooks/process_serving_logs.py +++ b/src/monitoring/notebooks/process_serving_logs.py @@ -31,7 +31,7 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("gold_schema", "fraud_gold") dbutils.widgets.text("ml_schema", "") @@ -46,7 +46,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" payload_table = f"{catalog_name}.{ml_schema}.fraud_serving_payload" online_table = f"{catalog_name}.{ml_schema}.fraud_serving_inference" diff --git a/src/monitoring/notebooks/seed_drift.py b/src/monitoring/notebooks/seed_drift.py index d0d6530..142c2f7 100644 --- a/src/monitoring/notebooks/seed_drift.py +++ b/src/monitoring/notebooks/seed_drift.py @@ -23,7 +23,15 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +# MAGIC %pip install -q --upgrade databricks-sdk + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("amount_multiplier", "6.0", label="Scale amount by this to force drift") dbutils.widgets.text("num_rows", "5000", label="How many drifted rows to append") @@ -39,7 +47,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" amount_multiplier = float(dbutils.widgets.get("amount_multiplier")) num_rows = int(dbutils.widgets.get("num_rows")) diff --git a/src/training/notebooks/training.py b/src/training/notebooks/training.py index ac55125..84ec3cc 100644 --- a/src/training/notebooks/training.py +++ b/src/training/notebooks/training.py @@ -21,7 +21,25 @@ # COMMAND ---------- -dbutils.widgets.text("catalog_name", "adoption_workshop") +# MAGIC %md +# MAGIC ## Setup +# MAGIC Upgrade MLflow to 3.x for the new registry/logged-model features. Install the +# MAGIC Feature Engineering client **without `--upgrade`** so it stays at the runtime version +# MAGIC that created the feature table (mismatched client versions can't resolve the feature +# MAGIC table). On ML Runtime the client is pre-installed; on serverless this adds it. + +# COMMAND ---------- + +# MAGIC %pip install -q "mlflow>=3.0" --upgrade +# MAGIC %pip install -q databricks-feature-engineering + +# COMMAND ---------- + +dbutils.library.restartPython() + +# COMMAND ---------- + +dbutils.widgets.text("catalog_name", "dev") dbutils.widgets.text("gold_schema", "fraud_gold") dbutils.widgets.text("ml_schema", "") dbutils.widgets.text("experiment_name", "") @@ -39,7 +57,7 @@ if not ml_schema: _user = spark.range(1).select(F.current_user()).first()[0] _short = "".join(c if c.isalnum() else "_" for c in _user.split("@")[0]) - ml_schema = f"dev_{_short}_fraud" + ml_schema = f"dev_{_short}_fraud_ml" # Labels come from the shared gold table; features come from the personal schema. source_table = f"{catalog_name}.{gold_schema}.transactions_enriched"