Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions src/winml/modelkit/loader/resolution.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

from .task import (
HF_TASK_DEFAULTS,
KNOWN_TASKS,
get_default_task_for_model_id,
get_supported_tasks,
normalize_task,
Expand Down Expand Up @@ -253,6 +254,7 @@ class TaskSource(str, Enum):
SENTINEL_DEFAULT = "sentinel-default" # (model_type, None) sentinel
TASKS_MANAGER = "tasks-manager" # Optimum inference (incl. fill-mask upgrade)
WRAPPED_LIBRARY = "wrapped-library" # no architectures -> first supported task
PIPELINE_TAG = "pipeline-tag" # Hub pipeline_tag fallback
HF_TASK_DEFAULT = "hf-task-default" # last-resort default


Expand Down Expand Up @@ -440,8 +442,8 @@ def resolve_task(
"""Resolve a single model's task + class from an HF config.

Stages: 0 user override -> 1 detect (override / no-architectures /
TasksManager / default) -> 2 model class -> 3 modality upgrade
(detection path only) -> 4 composite tag.
TasksManager / pipeline-tag / default) -> 2 model class -> 3 modality
upgrade (detection path only) -> 4 composite tag.

``model_type_override`` lets a caller drive resolution with a build variant
(e.g. ``qwen3_transformer_only``) without mutating the loaded HF config; when
Expand Down Expand Up @@ -572,6 +574,17 @@ def resolve_task(
except ValueError:
opt_task = None

# 1e. Hub pipeline_tag fallback

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: this block is labeled 1e but executes before 1d (last-resort default), so the code reads 1c -> 1e -> 1d. The docstring already lists them in execution order ("TasksManager / pipeline-tag / default"); renaming pipeline-tag to 1d and the default to 1e would make the inline labels match.

if opt_task is None and model_id:
Comment thread
xieofxie marked this conversation as resolved.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This puts a Hub round-trip (model_info) on resolve_task, which is shared by build / eval / inference too, not just inspect — those callers reach here whenever --task isn't passed and arch detection fails. Task resolution has been deliberately offline/local up to now, so worth confirming we want the network reach on all of those paths vs. scoping it to inspect. Offline mode degrades fine (the except returns None) and it's guarded to the fallback, so this is more "is this intended" than a bug. Minor: model_info retries transient errors, so worst-case latency can exceed the 10s timeout.

from ..utils.hub_utils import get_pipeline_tag

tag = get_pipeline_tag(model_id)
if tag:
normalized_tag = normalize_task(tag)
if normalized_tag in KNOWN_TASKS:

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

KNOWN_TASKS is the display/validation taxonomy and includes tasks that have no ONNX export path (text-to-image, reinforcement-learning, time-series-forecasting, any-to-any, inpainting, ...). I checked, and sentence-similarity is safe because optimum's map_from_synonym collapses it to feature-extraction inside normalize_task — but those others pass through unchanged. So for a model whose pipeline_tag is one of them and whose Stage 1c failed, we'd now set opt_task to a non-exportable task instead of the old feature-extraction default. For inspect that's arguably more accurate, but on the build/eval path it turns a usable default into a hard failure at export / model-class resolution. Consider gating on the exportable set (e.g. intersect with get_supported_tasks / what TasksManager can resolve) rather than the full KNOWN_TASKS.

opt_task = normalized_tag
source = TaskSource.PIPELINE_TAG

# 1d. last-resort default
if opt_task is None:
opt_task = next(iter(HF_TASK_DEFAULTS))
Expand Down
22 changes: 22 additions & 0 deletions src/winml/modelkit/utils/hub_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,28 @@ def is_hub_model(model_name_or_path: str) -> tuple[bool, dict]:
return False, {"type": "local", "path": model_name_or_path}


_PIPELINE_TAG_TIMEOUT = 10 # seconds

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

make it smaller ?



def get_pipeline_tag(model_id: str) -> str | None:
"""Return the Hub ``pipeline_tag`` for *model_id*, or ``None``.

Lightweight helper that skips the full metadata extraction of
``is_hub_model``. Returns ``None`` (never raises) when *model_id* is a
local path, the Hub is unreachable, or the model has no tag.
"""
if _is_local_path(model_id):
return None
try:
from huggingface_hub import HfApi

info = HfApi().model_info(model_id, timeout=_PIPELINE_TAG_TIMEOUT)
return getattr(info, "pipeline_tag", None)
except Exception:
logger.debug("pipeline_tag lookup failed for '%s'", model_id, exc_info=True)
return None


def inject_hub_metadata(onnx_model: Any, model_name_or_path: str, metadata: dict) -> None:
"""Inject HuggingFace Hub metadata into ONNX model.

Expand Down
55 changes: 55 additions & 0 deletions tests/unit/loader/test_detect_task.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,3 +228,58 @@ def test_resolve_task_case1_surfaces_modality_aware_task() -> None:
r = resolve_task(cfg)
assert r.task == "image-feature-extraction"
assert r.optimum_task == "feature-extraction"


# =============================================================================
# Stage 1e — Hub pipeline_tag fallback
# =============================================================================

_GET_PIPELINE_TAG = "winml.modelkit.utils.hub_utils.get_pipeline_tag"


def test_resolve_task_uses_pipeline_tag_when_architecture_fails() -> None:
"""When config.architectures contains a generic name (e.g. 'Model') that
TasksManager cannot resolve, the Hub pipeline_tag is used as fallback."""
cfg = _FakeConfig("faketype", name_or_path="audeering/wav2vec2-large-robust-24-ft-age-gender")
with (
patch(_INFER, side_effect=ValueError("unknown arch")),
patch(_GET_PIPELINE_TAG, return_value="audio-classification"),
):
r = resolve_task(cfg)
assert r.task == "audio-classification"
assert r.source == TaskSource.PIPELINE_TAG


def test_resolve_task_pipeline_tag_skips_invalid_task() -> None:
"""When pipeline_tag is not a recognized task, falls through to last-resort default."""
cfg = _FakeConfig("faketype", name_or_path="someone/some-model")
with (
patch(_INFER, side_effect=ValueError("unknown arch")),
patch(_GET_PIPELINE_TAG, return_value="not-a-real-task"),
):
r = resolve_task(cfg)
assert r.source == TaskSource.HF_TASK_DEFAULT


def test_resolve_task_pipeline_tag_handles_api_failure() -> None:
"""When the Hub API call fails (network error, etc.), get_pipeline_tag returns None."""
cfg = _FakeConfig("faketype", name_or_path="someone/some-model")
with (
patch(_INFER, side_effect=ValueError("unknown arch")),
patch(_GET_PIPELINE_TAG, return_value=None),
):
r = resolve_task(cfg)
assert r.source == TaskSource.HF_TASK_DEFAULT


def test_resolve_task_pipeline_tag_skips_local_path() -> None:

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test mocks get_pipeline_tag, so it never actually exercises the local-path skip — that logic lives inside get_pipeline_tag, which is patched out here. The assertion only confirms the helper is called with "./local-model" and returns the mocked None, so the name/docstring overclaim what's covered: the real _is_local_path rejection and the except Exception fallthrough have no direct test. Suggest a focused unit test on get_pipeline_tag itself (local path -> None without hitting the network; model_info raising -> None).

"""When model_id is a local path, get_pipeline_tag is still called but returns None
(local-path rejection is internal to get_pipeline_tag)."""
cfg = _FakeConfig("faketype", name_or_path="./local-model")
with (
patch(_INFER, side_effect=ValueError("unknown arch")),
patch(_GET_PIPELINE_TAG, return_value=None) as mock_tag,
):
r = resolve_task(cfg)
mock_tag.assert_called_once_with("./local-model")
assert r.source == TaskSource.HF_TASK_DEFAULT
Loading