From 78e00201ff7200a93a5bc2f9b9e326cf2d7c6aac Mon Sep 17 00:00:00 2001 From: Shiyi Zheng Date: Sun, 12 Jul 2026 17:16:48 +0800 Subject: [PATCH 1/6] Add ASR fp16 recipe for MahmoudAshraf/mms-300m-1130-forced-aligner Standard Wav2Vec2ForCTC model. Stock 'winml build' auto-config loads ASR via AutoModelForSpeechSeq2Seq, which rejects Wav2Vec2Config and fails. This recipe forces model_class=AutoModelForCTC (via 'winml config --model-class'), making the model buildable. Validated on CPU: fp16 build exit 0 (250.6s), 426 FLOAT16/0 FLOAT32 inits, output logits[1,49,31] CTC frames, L1 perf P50 133.9ms. --- ...omatic-speech-recognition_fp16_config.json | 59 +++++++++++++++++++ examples/recipes/README.md | 3 +- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 examples/recipes/MahmoudAshraf_mms-300m-1130-forced-aligner/automatic-speech-recognition_fp16_config.json diff --git a/examples/recipes/MahmoudAshraf_mms-300m-1130-forced-aligner/automatic-speech-recognition_fp16_config.json b/examples/recipes/MahmoudAshraf_mms-300m-1130-forced-aligner/automatic-speech-recognition_fp16_config.json new file mode 100644 index 000000000..f729900c5 --- /dev/null +++ b/examples/recipes/MahmoudAshraf_mms-300m-1130-forced-aligner/automatic-speech-recognition_fp16_config.json @@ -0,0 +1,59 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "input_values", + "dtype": "float32", + "shape": [ + 1, + 16000 + ], + "value_range": [ + -1, + 1 + ] + } + ], + "output_tensors": [ + { + "name": "logits" + } + ] + }, + "optim": {}, + "quant": { + "mode": "fp16", + "samples": 10, + "calibration_method": "minmax", + "weight_type": "uint8", + "activation_type": "uint8", + "per_channel": false, + "symmetric": false, + "weight_symmetric": null, + "activation_symmetric": null, + "save_calibration": false, + "distribution": "uniform", + "seed": null, + "calibration_load_path": null, + "calibration_save_path": null, + "op_types_to_quantize": null, + "nodes_to_exclude": null, + "fp16_keep_io_types": true, + "fp16_op_block_list": null + }, + "compile": null, + "loader": { + "task": "automatic-speech-recognition", + "model_class": "AutoModelForCTC", + "model_type": "wav2vec2" + } +} diff --git a/examples/recipes/README.md b/examples/recipes/README.md index 1077e4f74..0743ec731 100644 --- a/examples/recipes/README.md +++ b/examples/recipes/README.md @@ -14,7 +14,7 @@ Each *(model, task)* includes: ## Models -Total: **75** (model, task) tuples that pass fp16 eval on all 10 (EP, device) buckets. +Total: **76** (model, task) tuples that pass fp16 eval on all 10 (EP, device) buckets. | Model | Task | |---|---| @@ -32,6 +32,7 @@ Total: **75** (model, task) tuples that pass fp16 eval on all 10 (EP, device) bu | Intel/bert-base-uncased-mrpc | feature-extraction | | Intel/bert-base-uncased-mrpc | text-classification | | Isotonic/distilbert_finetuned_ai4privacy_v2 | token-classification | +| MahmoudAshraf/mms-300m-1130-forced-aligner | automatic-speech-recognition | | ProsusAI/finbert | text-classification | | Salesforce/blip-image-captioning-base | image-to-text | | StanfordAIMI/dinov2-base-xray-224 | image-feature-extraction | From 39302944fda8bcb72ea3dc36cbd520bb919aea26 Mon Sep 17 00:00:00 2001 From: Shiyi Zheng Date: Mon, 13 Jul 2026 13:47:18 +0800 Subject: [PATCH 2/6] recipe(mms-forced-aligner): nest under cpu/cpu/ per tested-EP layout (_meta-058) --- .../{ => cpu/cpu}/automatic-speech-recognition_fp16_config.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/recipes/MahmoudAshraf_mms-300m-1130-forced-aligner/{ => cpu/cpu}/automatic-speech-recognition_fp16_config.json (100%) diff --git a/examples/recipes/MahmoudAshraf_mms-300m-1130-forced-aligner/automatic-speech-recognition_fp16_config.json b/examples/recipes/MahmoudAshraf_mms-300m-1130-forced-aligner/cpu/cpu/automatic-speech-recognition_fp16_config.json similarity index 100% rename from examples/recipes/MahmoudAshraf_mms-300m-1130-forced-aligner/automatic-speech-recognition_fp16_config.json rename to examples/recipes/MahmoudAshraf_mms-300m-1130-forced-aligner/cpu/cpu/automatic-speech-recognition_fp16_config.json From 981119d3eae80b4d6c3fa27f9432c40dbbf8868b Mon Sep 17 00:00:00 2001 From: Shiyi Zheng Date: Mon, 13 Jul 2026 20:12:53 +0800 Subject: [PATCH 3/6] recipe: drop README index-row change (recipe-only PR; not certified for all-10-bucket README) --- examples/recipes/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/recipes/README.md b/examples/recipes/README.md index 0743ec731..1077e4f74 100644 --- a/examples/recipes/README.md +++ b/examples/recipes/README.md @@ -14,7 +14,7 @@ Each *(model, task)* includes: ## Models -Total: **76** (model, task) tuples that pass fp16 eval on all 10 (EP, device) buckets. +Total: **75** (model, task) tuples that pass fp16 eval on all 10 (EP, device) buckets. | Model | Task | |---|---| @@ -32,7 +32,6 @@ Total: **76** (model, task) tuples that pass fp16 eval on all 10 (EP, device) bu | Intel/bert-base-uncased-mrpc | feature-extraction | | Intel/bert-base-uncased-mrpc | text-classification | | Isotonic/distilbert_finetuned_ai4privacy_v2 | token-classification | -| MahmoudAshraf/mms-300m-1130-forced-aligner | automatic-speech-recognition | | ProsusAI/finbert | text-classification | | Salesforce/blip-image-captioning-base | image-to-text | | StanfordAIMI/dinov2-base-xray-224 | image-feature-extraction | From 0976f77637f88a0ccca203a313801fa85b878ccb Mon Sep 17 00:00:00 2001 From: Shiyi Zheng Date: Wed, 15 Jul 2026 15:00:44 +0800 Subject: [PATCH 4/6] fix: resolve CTC loaders from architecture metadata --- src/winml/modelkit/loader/resolution.py | 35 +++++++++++++++++++++ tests/unit/loader/test_config_resolution.py | 28 +++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/src/winml/modelkit/loader/resolution.py b/src/winml/modelkit/loader/resolution.py index 871c7c817..d931ce52c 100644 --- a/src/winml/modelkit/loader/resolution.py +++ b/src/winml/modelkit/loader/resolution.py @@ -132,6 +132,37 @@ class from the ``transformers`` package. ) from e +# Some pipeline tasks span incompatible model families whose AutoModel classes +# cannot load one another. Keep the mapping architecture-driven: the checkpoint's +# published ``architectures`` metadata selects the specialized AutoModel, never a +# model-id allowlist. +_ARCHITECTURE_AUTO_CLASS_OVERRIDES: dict[tuple[str, str], str] = { + ("automatic-speech-recognition", "ForCTC"): "AutoModelForCTC", +} + + +def _resolve_architecture_auto_class(config: PretrainedConfig, task: str) -> type | None: + """Return an architecture-specific AutoModel class when a task is ambiguous. + + ``automatic-speech-recognition`` covers both encoder-only CTC models and + encoder-decoder speech models. Optimum's task-level default is + ``AutoModelForSpeechSeq2Seq``, which cannot load a ``Wav2Vec2Config``. A + published ``*ForCTC`` architecture is an architecture-wide signal that the + correct loader is ``AutoModelForCTC``; seq2seq architectures continue to the + existing TasksManager fallback. + """ + architectures = getattr(config, "architectures", None) + if not architectures: + return None + + architecture = architectures[0] + for (mapped_task, suffix), auto_class_name in _ARCHITECTURE_AUTO_CLASS_OVERRIDES.items(): + if task == mapped_task and architecture.endswith(suffix): + transformers_module = importlib.import_module("transformers") + return cast("type", getattr(transformers_module, auto_class_name)) + return None + + def _detect_task_from_model_class(model_class: type) -> str: """Detect task from a model class via TasksManager. @@ -515,6 +546,8 @@ def resolve_task( resolved = _get_custom_model_class( model_type_norm, original ) or _get_custom_model_class(model_type_norm, normalized) + if resolved is None: + resolved = _resolve_architecture_auto_class(config, normalized) if resolved is None: try: resolved = TasksManager.get_model_class_for_task(normalized, framework="pt") @@ -580,6 +613,8 @@ def resolve_task( # --- Stage 2: model class (if not already resolved in 1b) ------------- if resolved is None: resolved = _get_custom_model_class(model_type_norm, opt_task) + if resolved is None: + resolved = _resolve_architecture_auto_class(config, opt_task) if resolved is None: try: resolved = TasksManager.get_model_class_for_task(opt_task, framework="pt") diff --git a/tests/unit/loader/test_config_resolution.py b/tests/unit/loader/test_config_resolution.py index f11b89492..00a2bea2b 100644 --- a/tests/unit/loader/test_config_resolution.py +++ b/tests/unit/loader/test_config_resolution.py @@ -84,6 +84,34 @@ def test_auto_detect_new_architectures( assert hasattr(r.model_class, "__name__") +class TestResolveAmbiguousASRModelClass: + """Resolve ASR's CTC and seq2seq families from architecture metadata.""" + + @pytest.mark.parametrize("explicit_task", [False, True], ids=["auto-task", "explicit-task"]) + def test_ctc_architecture_uses_auto_model_for_ctc( + self, + explicit_task: bool, + make_mock_config, + ) -> None: + config = make_mock_config("wav2vec2", ["Wav2Vec2ForCTC"]) + + r = resolve_task( + config, + task="automatic-speech-recognition" if explicit_task else None, + ) + + assert r.task == "automatic-speech-recognition" + assert r.model_class.__name__ == "AutoModelForCTC" + + def test_seq2seq_architecture_keeps_speech_seq2seq_default(self, make_mock_config) -> None: + config = make_mock_config("whisper", ["WhisperForConditionalGeneration"]) + + r = resolve_task(config) + + assert r.task == "automatic-speech-recognition" + assert r.model_class.__name__ == "AutoModelForSpeechSeq2Seq" + + class TestResolveTaskAliasPreservation: """User-task path: original task is returned, not the normalized form. From 7e972a0222d35b624190df5d0cdf6dd6fab7cc8c Mon Sep 17 00:00:00 2001 From: Shiyi Zheng Date: Wed, 15 Jul 2026 15:04:50 +0800 Subject: [PATCH 5/6] recipe: rely on automatic CTC loader resolution --- .../cpu/cpu/automatic-speech-recognition_fp16_config.json | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/recipes/MahmoudAshraf_mms-300m-1130-forced-aligner/cpu/cpu/automatic-speech-recognition_fp16_config.json b/examples/recipes/MahmoudAshraf_mms-300m-1130-forced-aligner/cpu/cpu/automatic-speech-recognition_fp16_config.json index f729900c5..cc15d0ecd 100644 --- a/examples/recipes/MahmoudAshraf_mms-300m-1130-forced-aligner/cpu/cpu/automatic-speech-recognition_fp16_config.json +++ b/examples/recipes/MahmoudAshraf_mms-300m-1130-forced-aligner/cpu/cpu/automatic-speech-recognition_fp16_config.json @@ -53,7 +53,6 @@ "compile": null, "loader": { "task": "automatic-speech-recognition", - "model_class": "AutoModelForCTC", "model_type": "wav2vec2" } } From 6eebeb9c6891ed4024d814a0709378ab8dbc1568 Mon Sep 17 00:00:00 2001 From: "Shiyi Zheng (from Dev Box)" Date: Thu, 16 Jul 2026 10:39:16 +0800 Subject: [PATCH 6/6] docs(loader): clarify ambiguous-task override map --- src/winml/modelkit/loader/resolution.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/winml/modelkit/loader/resolution.py b/src/winml/modelkit/loader/resolution.py index d931ce52c..78e46357c 100644 --- a/src/winml/modelkit/loader/resolution.py +++ b/src/winml/modelkit/loader/resolution.py @@ -135,7 +135,8 @@ class from the ``transformers`` package. # Some pipeline tasks span incompatible model families whose AutoModel classes # cannot load one another. Keep the mapping architecture-driven: the checkpoint's # published ``architectures`` metadata selects the specialized AutoModel, never a -# model-id allowlist. +# model-id allowlist. Add one entry for each future ambiguous task family rather +# than broadening this into a global preference over TasksManager defaults. _ARCHITECTURE_AUTO_CLASS_OVERRIDES: dict[tuple[str, str], str] = { ("automatic-speech-recognition", "ForCTC"): "AutoModelForCTC", }