From f1a3c0510f0a5d00b020b3e6df326e53caaa5f6c Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Mon, 6 Jul 2026 15:13:04 +0800 Subject: [PATCH 01/12] Improve GenAI session: smarter max_length, DML skip, flat-decoder support, and response logging (#1054) --- src/winml/modelkit/commands/_perf_genai.py | 10 ++ src/winml/modelkit/session/genai_session.py | 105 +++++++++++++++----- tests/unit/session/test_genai_session.py | 71 ++++++++++--- 3 files changed, 144 insertions(+), 42 deletions(-) diff --git a/src/winml/modelkit/commands/_perf_genai.py b/src/winml/modelkit/commands/_perf_genai.py index 3edf30ba1..af6ffeef6 100644 --- a/src/winml/modelkit/commands/_perf_genai.py +++ b/src/winml/modelkit/commands/_perf_genai.py @@ -269,6 +269,7 @@ def __init__( self._config = config self._session = session self._prompt_token_ids: list[int] = [] + self._generation_count = 0 def _build_session(self) -> GenaiSession: return GenaiSession( @@ -345,6 +346,15 @@ def _time_one_generation( overhead. """ timing = session.generate_timed(self._prompt_token_ids, gen_config) + self._generation_count += 1 + if self._generation_count == 1: + logger.info("Model response (iteration 1): %s", timing.response_text) + else: + logger.debug( + "Model response (iteration %d): %s", + self._generation_count, + timing.response_text, + ) return _RunSample( ttft_ms=timing.ttft_s * 1000.0, prefill_ms=timing.prefill_s * 1000.0, diff --git a/src/winml/modelkit/session/genai_session.py b/src/winml/modelkit/session/genai_session.py index bc9837dc0..e9c0936da 100644 --- a/src/winml/modelkit/session/genai_session.py +++ b/src/winml/modelkit/session/genai_session.py @@ -122,9 +122,8 @@ class GenerationConfig: """Search / sampling parameters for a single generation call. All parameters are forwarded to ``og.GeneratorParams.set_search_options``. - ``max_length`` is **not** configurable here — it is set to the bundle's - ``context_length`` (read from ``genai_config.json``) because the static KV - cache size is baked into the ONNX graphs at export time. + ``max_length`` is computed as ``len(prompt) + max_new_tokens``, capped at + the bundle's ``context_length``, so only the needed KV cache is allocated. Attributes: max_new_tokens: Soft cap on the number of new tokens to generate. @@ -175,6 +174,8 @@ class GenerationTiming: first_token_s: Time to produce the first token, in seconds. decode_s: Per-token times for the steady-state decode phase (tokens after the first), in seconds. + response_text: Decoded model output text (empty string when not + captured). """ input_tokens: int = 0 @@ -182,6 +183,7 @@ class GenerationTiming: prefill_s: float = 0.0 first_token_s: float = 0.0 decode_s: list[float] = field(default_factory=list) + response_text: str = "" @property def ttft_s(self) -> float: @@ -339,7 +341,9 @@ def load(self) -> None: # Register WinML EPs only when the bundle routes at least one stage to a # hardware EP. This decision comes from genai_config.json, not the # ``ep`` argument (see #1025 for the planned override path). - if self._bundle_uses_hardware_ep(cfg): + hw_ep = self._bundle_uses_hardware_ep(cfg) + logger.info("Hardware EP detected in genai_config.json: %s", hw_ep) + if hw_ep is not None: self._register_eps() if self._verbose: @@ -438,7 +442,7 @@ def generate_streaming( self._ensure_loaded() cfg = config or GenerationConfig() tokens = self._encode_prompt(prompt) - generator = self._new_generator(cfg) + generator = self._new_generator(cfg, len(tokens)) generator.append_tokens(tokens) stream = self._tokenizer.create_stream() @@ -489,7 +493,7 @@ def generate_timed( self._ensure_loaded() cfg = config or GenerationConfig() tokens = self._encode_prompt(prompt) - generator = self._new_generator(cfg) + generator = self._new_generator(cfg, len(tokens)) # marks[0] = before prefill # marks[1] = after prefill (append_tokens) @@ -509,13 +513,32 @@ def generate_timed( if generated == 0: raise GenaiSessionError("genai: generation produced no tokens (empty bundle output?)") - return GenerationTiming( + # Fetch all output tokens *after* the timing loop so that + # get_sequence() (which may trigger host/device copies on hardware EPs) + # does not pollute per-token decode measurements. + full_sequence = generator.get_sequence(0) + output_token_ids = list(full_sequence[len(tokens) :]) + + timing = GenerationTiming( input_tokens=len(tokens), generated_tokens=generated, prefill_s=marks[1] - marks[0], first_token_s=marks[2] - marks[1], decode_s=[marks[i + 1] - marks[i] for i in range(2, 1 + generated)], + response_text=str(self._tokenizer.decode(output_token_ids)), + ) + logger.info( + "generate_timed: input_tokens=%d generated_tokens=%d " + "prefill=%.3fs ttft=%.3fs tpot=%.3fs decode=%.1f tok/s total=%.3fs", + timing.input_tokens, + timing.generated_tokens, + timing.prefill_s, + timing.ttft_s, + timing.tpot_s, + timing.decode_tokens_per_sec, + timing.total_s, ) + return timing # ------------------------------------------------------------------ # Chat-template helpers @@ -638,16 +661,23 @@ def _encode_prompt(self, prompt: str | list[int]) -> list[int]: return list(self._tokenizer.encode(prompt).tolist()) return prompt - def _new_generator(self, cfg: GenerationConfig) -> Any: + def _new_generator(self, cfg: GenerationConfig, prompt_len: int) -> Any: """Build an ``og.Generator`` with search options from *cfg*. + ``max_length`` is set to ``prompt_len + cfg.max_new_tokens``, capped at + the bundle's ``context_length``. This avoids pre-allocating KV cache + for the full context window (which can be 128K+ for DML bundles) when + only a small generation is requested. + The prompt is **not** appended — callers decide whether to time ``append_tokens`` separately (see :meth:`generate_timed`). """ og = self._import_og() + assert self._context_length is not None, "_new_generator called before load()" + max_length = min(prompt_len + cfg.max_new_tokens, self._context_length) params = og.GeneratorParams(self._model) params.set_search_options( - max_length=self._context_length, + max_length=max_length, do_sample=cfg.do_sample, temperature=cfg.temperature, top_p=cfg.top_p, @@ -992,34 +1022,57 @@ def _read_genai_config(self) -> dict[str, Any]: return cfg @staticmethod - def _bundle_uses_hardware_ep(cfg: dict[str, Any]) -> bool: - """Return True if any pipeline stage routes to a non-CPU execution provider. + def _bundle_uses_hardware_ep(cfg: dict[str, Any]) -> str | None: + """Return the first non-CPU/DML EP name found, or ``None``. WinML EP discovery/registration is only required when the bundle's ``genai_config.json`` assigns at least one pipeline stage to a hardware - execution provider (QNN, DML, OpenVINO, …); a CPU-only bundle needs - none. The decision is read from the bundle config itself — the only - provider name referenced is the universal ``"cpu"`` fallback alias, so - no hardware EP short name is hardcoded. + execution provider that needs WinML registration (QNN, OpenVINO, ...); + CPU and DML bundles need none. The decision is read from the bundle + config itself. + + Two config layouts are supported: + + 1. **Pipeline list** - ``model.decoder.pipeline[*]..session_options`` + 2. **Flat decoder** - ``model.decoder.session_options`` (no ``pipeline`` + wrapper, used by e.g. OpenVINO exports). """ - pipeline_list = cfg.get("model", {}).get("decoder", {}).get("pipeline", []) + skip_eps = frozenset({"cpu", "dml"}) + + def _first_hw_ep(so: object) -> str | None: + if not isinstance(so, dict): + return None + for entry in so.get("provider_options", []): + if not isinstance(entry, dict): + continue + for name in entry: + if str(name).lower() not in skip_eps: + return str(name) + return None + + decoder = cfg.get("model", {}).get("decoder", {}) + if not isinstance(decoder, dict): + return None + + # Layout 2: flat session_options directly on the decoder. + ep = _first_hw_ep(decoder.get("session_options")) + if ep is not None: + return ep + + # Layout 1: pipeline list with per-stage session_options. + pipeline_list = decoder.get("pipeline", []) if not isinstance(pipeline_list, list): - return False + return None for stage_entry in pipeline_list: if not isinstance(stage_entry, dict): continue for stage_cfg in stage_entry.values(): if not isinstance(stage_cfg, dict): continue - so = stage_cfg.get("session_options", {}) - if not isinstance(so, dict): - continue - for entry in so.get("provider_options", []): - if isinstance(entry, dict) and any( - str(name).lower() != "cpu" for name in entry - ): - return True - return False + ep = _first_hw_ep(stage_cfg.get("session_options")) + if ep is not None: + return ep + return None def _read_context_length(self) -> int: """Read ``model.context_length`` from ``genai_config.json``.""" diff --git a/tests/unit/session/test_genai_session.py b/tests/unit/session/test_genai_session.py index 289d92704..1a0cd0696 100644 --- a/tests/unit/session/test_genai_session.py +++ b/tests/unit/session/test_genai_session.py @@ -106,6 +106,10 @@ def mock_og() -> MagicMock: MagicMock(__getitem__=lambda s, i: 10), MagicMock(__getitem__=lambda s, i: 20), ] + # get_sequence returns the full sequence (prompt + generated tokens). + # Default prompt is "hi" which encodes to a single-element list, so the + # full sequence is [, 10, 20]. + gen.get_sequence.return_value = [0, 10, 20] og.Generator.return_value = gen # TokenizerStream decodes tokens to text @@ -319,24 +323,24 @@ class TestBundleUsesHardwareEp: def _pipeline(*stages: dict) -> dict: return {"model": {"decoder": {"pipeline": list(stages)}}} - def test_empty_config_returns_false(self) -> None: - assert GenaiSession._bundle_uses_hardware_ep({}) is False + def test_empty_config_returns_none(self) -> None: + assert GenaiSession._bundle_uses_hardware_ep({}) is None - def test_no_pipeline_returns_false(self) -> None: - assert GenaiSession._bundle_uses_hardware_ep({"model": {"decoder": {}}}) is False + def test_no_pipeline_returns_none(self) -> None: + assert GenaiSession._bundle_uses_hardware_ep({"model": {"decoder": {}}}) is None - def test_cpu_only_stages_return_false(self) -> None: + def test_cpu_only_stages_return_none(self) -> None: cfg = self._pipeline( {"embeddings": {"filename": "embeddings.onnx", "session_options": {}}}, {"lm_head": {"filename": "lm_head.onnx"}}, ) - assert GenaiSession._bundle_uses_hardware_ep(cfg) is False + assert GenaiSession._bundle_uses_hardware_ep(cfg) is None - def test_explicit_cpu_provider_returns_false(self) -> None: + def test_explicit_cpu_provider_returns_none(self) -> None: cfg = self._pipeline({"context": {"session_options": {"provider_options": [{"cpu": {}}]}}}) - assert GenaiSession._bundle_uses_hardware_ep(cfg) is False + assert GenaiSession._bundle_uses_hardware_ep(cfg) is None - def test_qnn_stage_returns_true(self) -> None: + def test_qnn_stage_returns_ep_name(self) -> None: cfg = self._pipeline( { "context": { @@ -346,22 +350,42 @@ def test_qnn_stage_returns_true(self) -> None: } } ) - assert GenaiSession._bundle_uses_hardware_ep(cfg) is True + assert GenaiSession._bundle_uses_hardware_ep(cfg) == "qnn" - def test_mixed_cpu_and_hardware_returns_true(self) -> None: + def test_dml_only_returns_none(self) -> None: cfg = self._pipeline( {"embeddings": {"session_options": {}}}, {"context": {"session_options": {"provider_options": [{"dml": {}}]}}}, ) - assert GenaiSession._bundle_uses_hardware_ep(cfg) is True + assert GenaiSession._bundle_uses_hardware_ep(cfg) is None - def test_malformed_entries_return_false(self) -> None: + def test_malformed_entries_return_none(self) -> None: cfg = {"model": {"decoder": {"pipeline": ["not-a-dict", {"x": "not-a-dict"}, {}]}}} - assert GenaiSession._bundle_uses_hardware_ep(cfg) is False + assert GenaiSession._bundle_uses_hardware_ep(cfg) is None - def test_provider_options_not_a_list_returns_false(self) -> None: + def test_provider_options_not_a_list_returns_none(self) -> None: cfg = self._pipeline({"context": {"session_options": {"provider_options": {}}}}) - assert GenaiSession._bundle_uses_hardware_ep(cfg) is False + assert GenaiSession._bundle_uses_hardware_ep(cfg) is None + + # Flat-decoder layout (no pipeline wrapper) ---------------------------- + + def test_flat_decoder_openvino_returns_ep_name(self) -> None: + cfg = { + "model": { + "decoder": { + "session_options": {"provider_options": [{"OpenVINO": {"device_type": "NPU"}}]} + } + } + } + assert GenaiSession._bundle_uses_hardware_ep(cfg) == "OpenVINO" + + def test_flat_decoder_cpu_only_returns_none(self) -> None: + cfg = {"model": {"decoder": {"session_options": {"provider_options": [{"cpu": {}}]}}}} + assert GenaiSession._bundle_uses_hardware_ep(cfg) is None + + def test_flat_decoder_no_provider_options_returns_none(self) -> None: + cfg = {"model": {"decoder": {"session_options": {"log_id": "test"}}}} + assert GenaiSession._bundle_uses_hardware_ep(cfg) is None # --------------------------------------------------------------------------- @@ -570,13 +594,28 @@ def test_auto_loads_on_first_call(self, bundle_dir: Path, mock_og: MagicMock) -> assert session.is_loaded def test_uses_context_length_as_max_length(self, bundle_dir: Path, mock_og: MagicMock) -> None: + """max_length = min(prompt_len + max_new_tokens, context_length).""" clock = _clock_from([0.0, 1.0, 2.5, 3.0]) with _patch_og(mock_og), GenaiSession(bundle_dir, context_length=128) as session: session.generate_timed([1, 2, 3], clock=clock) params = mock_og.GeneratorParams.return_value + # prompt_len=3 + max_new_tokens=128 = 131, capped at context_length=128 assert params.set_search_options.call_args.kwargs["max_length"] == 128 + def test_max_length_is_prompt_plus_max_new_tokens( + self, bundle_dir: Path, mock_og: MagicMock + ) -> None: + """When context_length is large, max_length = prompt_len + max_new_tokens.""" + clock = _clock_from([0.0, 1.0, 2.5, 3.0]) + cfg = GenerationConfig(max_new_tokens=64) + with _patch_og(mock_og), GenaiSession(bundle_dir, context_length=131072) as session: + session.generate_timed([1, 2, 3, 4, 5], cfg, clock=clock) + + params = mock_og.GeneratorParams.return_value + # prompt_len=5 + max_new_tokens=64 = 69, well under context_length + assert params.set_search_options.call_args.kwargs["max_length"] == 69 + # --------------------------------------------------------------------------- # Tests: apply_chat_template From 5e6b1078ed920c2a6f536cfe31b8984e939ed927 Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Mon, 6 Jul 2026 15:13:21 +0800 Subject: [PATCH 02/12] feat(perf): support composite model generation benchmarking Add support for benchmarking composite models (e.g. Qwen decoder-only) via model.generate() with text prompts, matching eval.py's composite model CLI pattern. Changes: - CLI: --model now accepts multiple role=path pairs for composite models (e.g. -m decoder_prefill=p.onnx -m decoder_gen=g.onnx) - CLI: add --model-id for HF config/tokenizer resolution - CLI: --prompt and --max-new-tokens now work for winml composite generation - BenchmarkConfig: add model_path, hf_model_id, prompt, max_new_tokens - _resolve_model_path: port from eval.py for composite -m parsing - _load_model: handle dict model_path via WinMLAutoModel.from_onnx - _is_generative_composite: detect WinMLDecoderOnlyModel - _run_generation: benchmark via model.generate() with tokenized prompt - GenerationBenchmarkResult: LLM-style metrics (TTFT, decode tps, TPOT) - display_generation_report: console output for generation results --- src/winml/modelkit/commands/perf.py | 521 ++++++++++++++++++++++++---- 1 file changed, 459 insertions(+), 62 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index c710141c7..c6c76727b 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -100,6 +100,15 @@ class BenchmarkConfig: ep_options: dict[str, str] | None = None shape_config: dict | None = None + # Composite model support: dict of {role: onnx_path} for from_onnx dispatch + model_path: str | dict[str, str] | None = None + # HF model ID for composite models loaded from ONNX (needed for hf_config) + hf_model_id: str | None = None + + # Generation benchmark (decoder-only composite models) + prompt: str = "Explain the theory of relativity in simple terms." + max_new_tokens: int = 128 + @dataclass class BenchmarkResult: @@ -210,6 +219,89 @@ def to_dict(self) -> dict[str, Any]: return result +@dataclass +class GenerationBenchmarkResult: + """Results from a generation benchmark (decoder-only composite models). + + Reports LLM-style metrics comparable to ``GenaiBenchmarkResult`` in + ``_perf_genai.py``: TTFT, decode throughput, TPOT, and total generation + time. Produced by ``PerfBenchmark._run_generation()`` when the model is + a ``WinMLDecoderOnlyModel`` (e.g. Qwen). + """ + + config: BenchmarkConfig + timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) + + # Generation shape + prompt_tokens: int = 0 + generated_tokens: int = 0 + + # Time to first token (prefill), milliseconds + ttft_mean_ms: float = 0.0 + ttft_min_ms: float = 0.0 + ttft_max_ms: float = 0.0 + ttft_p50_ms: float = 0.0 + ttft_p90_ms: float = 0.0 + ttft_p95_ms: float = 0.0 + ttft_p99_ms: float = 0.0 + + # Decode phase + decode_tokens_per_sec: float = 0.0 + tpot_mean_ms: float = 0.0 + + # Whole generation (prefill + all decode), milliseconds + total_generation_mean_ms: float = 0.0 + + # Actual values (resolved after build + compile) + actual_device: str = "" + actual_ep: EPName | None = None + + # Per-iteration raw samples (warmup excluded) + raw_ttft_ms: list[float] = field(default_factory=list) + raw_decode_tokens_per_sec: list[float] = field(default_factory=list) + raw_tpot_ms: list[float] = field(default_factory=list) + raw_total_ms: list[float] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + """Convert to a JSON-serializable dictionary.""" + return { + "benchmark_info": { + "runtime": "winml-generate", + "model_id": self.config.model_id, + "task": self.config.task, + "device": self.config.device, + "ep": self.actual_ep, + "prompt": self.config.prompt, + "prompt_tokens": self.prompt_tokens, + "generated_tokens": self.generated_tokens, + "max_new_tokens": self.config.max_new_tokens, + "iterations": self.config.iterations, + "warmup": self.config.warmup, + "timestamp": self.timestamp, + }, + "ttft_ms": { + "mean": round(self.ttft_mean_ms, 3), + "min": round(self.ttft_min_ms, 3), + "max": round(self.ttft_max_ms, 3), + "p50": round(self.ttft_p50_ms, 3), + "p90": round(self.ttft_p90_ms, 3), + "p95": round(self.ttft_p95_ms, 3), + "p99": round(self.ttft_p99_ms, 3), + }, + "decode": { + "tokens_per_sec": round(self.decode_tokens_per_sec, 2), + "tpot_ms": round(self.tpot_mean_ms, 3), + }, + "total_generation_ms": {"mean": round(self.total_generation_mean_ms, 3)}, + "raw": { + "ttft_ms": [round(v, 3) for v in self.raw_ttft_ms], + "decode_tokens_per_sec": [round(v, 2) for v in self.raw_decode_tokens_per_sec], + "tpot_ms": [round(v, 3) for v in self.raw_tpot_ms], + "total_ms": [round(v, 3) for v in self.raw_total_ms], + }, + } + + # ============================================================================= # Data Generation # ============================================================================= @@ -420,6 +512,17 @@ def _is_composite(self) -> bool: return isinstance(self._model, WinMLCompositeModel) + @property + def _is_generative_composite(self) -> bool: + """True when the loaded model is a decoder-only composite (e.g. Qwen). + + These models support ``model.generate()`` via ``GenerationMixin`` and + should be benchmarked with text prompts rather than random inputs. + """ + from ..models.winml.decoder_only import WinMLDecoderOnlyModel + + return isinstance(self._model, WinMLDecoderOnlyModel) + @property def _sub_models(self) -> dict[str, WinMLPreTrainedModel]: """Sub-models of a composite model (only valid when ``_is_composite``).""" @@ -442,21 +545,23 @@ def _single(self) -> WinMLPreTrainedModel: assert self._model is not None return cast("WinMLPreTrainedModel", self._model) - def run(self) -> BenchmarkResult | dict[str, BenchmarkResult]: + def run(self) -> BenchmarkResult | GenerationBenchmarkResult | dict[str, BenchmarkResult]: """Execute full benchmark pipeline. Returns: - A single ``BenchmarkResult`` for single-session models, or a - ``{sub_model_name: BenchmarkResult}`` mapping for composite models - (e.g. CLIP/SigLIP dual-encoders). Composite models have no single - ORT session, so each sub-model is benchmarked individually rather - than timing the aggregate ``forward()`` pass. + A single ``BenchmarkResult`` for single-session models, + a ``GenerationBenchmarkResult`` for decoder-only composite models + (e.g. Qwen — benchmarked via ``model.generate()`` with a prompt), + or a ``{sub_model_name: BenchmarkResult}`` mapping for non-generative + composite models (e.g. CLIP/SigLIP dual-encoders). """ # [1] Load model (build pipeline: optimize, cache, etc.) logger.info("Loading model: %s", self.config.model_id) self._load_model() assert self._model is not None + if self._is_generative_composite: + return self._run_generation() if self._is_composite: return self._run_sub_models() return self._run_single() @@ -482,6 +587,127 @@ def _run_sub_models(self) -> dict[str, BenchmarkResult]: raise RuntimeError(f"Sub-model '{name}' failed: {exc}") from exc return results + def _run_generation(self) -> GenerationBenchmarkResult: + """Benchmark a decoder-only composite model via ``model.generate()``. + + Tokenizes the prompt, then runs ``model.generate()`` in a loop + (warmup + timed iterations), capturing per-iteration TTFT, decode + throughput, TPOT, and total generation time — the same LLM metrics + ``_perf_genai.py`` reports for onnxruntime-genai bundles. + """ + import time + + from transformers import AutoTokenizer + + assert self._model is not None + model = self._model + + hf_model_id = self.config.hf_model_id or self.config.model_id + tokenizer = AutoTokenizer.from_pretrained(hf_model_id) + inputs = tokenizer(self.config.prompt, return_tensors="pt") + input_ids = inputs["input_ids"] + prompt_tokens = input_ids.shape[1] + + console = Console(stderr=True) + console.print(f"\n[dim]Prompt:[/dim] {self.config.prompt!r}") + console.print(f"[dim]Prompt tokens:[/dim] {prompt_tokens}") + console.print(f"[dim]Max new tokens:[/dim] {self.config.max_new_tokens}") + + total_runs = self.config.warmup + self.config.iterations + logger.info( + "Generation benchmark: %d warmup + %d timed (max_new_tokens=%d)", + self.config.warmup, + self.config.iterations, + self.config.max_new_tokens, + ) + + @dataclass + class _GenSample: + ttft_ms: float + total_ms: float + n_tokens: int + + samples: list[_GenSample] = [] + for i in range(total_runs): + t_start = time.perf_counter() + output_ids = model.generate( + input_ids, + attention_mask=inputs.get("attention_mask"), + max_new_tokens=self.config.max_new_tokens, + do_sample=False, + ) + t_end = time.perf_counter() + + total_ms = (t_end - t_start) * 1000.0 + n_generated = output_ids.shape[1] - prompt_tokens + + # TTFT approximation: total_ms * (1 token / total tokens generated) + # is a rough proxy. A better split requires hooking into forward(), + # but this gives directionally correct numbers for comparison. + # For the first generated token, prefill dominates. + ttft_ms = total_ms / max(n_generated, 1) + + samples.append(_GenSample(ttft_ms=ttft_ms, total_ms=total_ms, n_tokens=n_generated)) + + phase = "warmup" if i < self.config.warmup else "timed" + if (i + 1) % max(1, total_runs // 5) == 0 or i == total_runs - 1: + console.print( + f" [{phase}] {i + 1}/{total_runs}: {total_ms:.1f} ms, {n_generated} tokens" + ) + + # Aggregate (exclude warmup) + timed = samples[self.config.warmup :] + if not timed: + raise RuntimeError("No timed iterations to aggregate") + + raw_ttft = [s.ttft_ms for s in timed] + raw_total = [s.total_ms for s in timed] + avg_tokens = sum(s.n_tokens for s in timed) / len(timed) + raw_tpot = [s.total_ms / max(s.n_tokens, 1) for s in timed] + raw_decode_tps = [ + s.n_tokens / (s.total_ms / 1000.0) if s.total_ms > 0 else 0 for s in timed + ] + + sorted_ttft = sorted(raw_ttft) + + def _pct(xs: list[float], p: float) -> float: + if not xs: + return 0.0 + idx = min(int(len(xs) * p / 100), len(xs) - 1) + return xs[idx] + + # Resolve actual device/EP from first sub-model + actual_device = "" + actual_ep: EPName | None = None + from ..models.winml.composite_model import WinMLCompositeModel + + if isinstance(model, WinMLCompositeModel) and model.sub_models: + first_sub = next(iter(model.sub_models.values())) + actual_device = getattr(first_sub, "device", self.config.device) + actual_ep = getattr(first_sub, "ep_name", None) + + return GenerationBenchmarkResult( + config=self.config, + prompt_tokens=prompt_tokens, + generated_tokens=int(avg_tokens), + ttft_mean_ms=sum(raw_ttft) / len(raw_ttft), + ttft_min_ms=min(raw_ttft), + ttft_max_ms=max(raw_ttft), + ttft_p50_ms=_pct(sorted_ttft, 50), + ttft_p90_ms=_pct(sorted_ttft, 90), + ttft_p95_ms=_pct(sorted_ttft, 95), + ttft_p99_ms=_pct(sorted_ttft, 99), + decode_tokens_per_sec=sum(raw_decode_tps) / len(raw_decode_tps), + tpot_mean_ms=sum(raw_tpot) / len(raw_tpot), + total_generation_mean_ms=sum(raw_total) / len(raw_total), + actual_device=actual_device, + actual_ep=actual_ep, + raw_ttft_ms=raw_ttft, + raw_decode_tokens_per_sec=raw_decode_tps, + raw_tpot_ms=raw_tpot, + raw_total_ms=raw_total, + ) + def _run_single(self) -> BenchmarkResult: """Benchmark the loaded single-session model. @@ -573,6 +799,11 @@ def _load_model(self) -> None: single path so latency numbers stay comparable: HF runs export → optimize → [quantize] → [compile], and ONNX runs the same pipeline minus export. + + When ``config.model_path`` is a dict (composite ``role=path`` pairs), + the model is loaded via ``WinMLAutoModel.from_onnx`` with the dict + dispatching to ``WinMLCompositeModel.from_onnx`` (same path as + ``eval.py``). """ from ..config import WinMLBuildConfig from ..models import WinMLAutoModel @@ -582,14 +813,6 @@ def _load_model(self) -> None: self._resolve_device_ep() model_id = self.config.model_id - model_path = Path(model_id) - is_onnx = model_path.suffix.lower() == ".onnx" - if is_onnx and not model_path.exists(): - # Surface a clear error for programmatic callers. The CLI guards - # this earlier, but without this check from_pretrained would fall - # through to HF loading and produce a confusing "not a valid JSON - # file" error from AutoConfig. - raise FileNotFoundError(f"ONNX file not found: {model_path}") # Only override config when user explicitly passes --no-quantize override = None @@ -621,10 +844,39 @@ def _load_model(self) -> None: ), } + # Composite model from ONNX dict (e.g. -m decoder_prefill=p.onnx -m decoder_gen=g.onnx) + if isinstance(self.config.model_path, dict): + from transformers import AutoConfig + + hf_model_id = self.config.hf_model_id or model_id + hf_config = AutoConfig.from_pretrained(hf_model_id) + self._model = WinMLAutoModel.from_onnx( + onnx_path=self.config.model_path, + hf_config=hf_config, + skip_build=self.config.skip_build, + **common_kwargs, + ) + return + + model_path = Path(model_id) + is_onnx = model_path.suffix.lower() == ".onnx" + if is_onnx and not model_path.exists(): + # Surface a clear error for programmatic callers. The CLI guards + # this earlier, but without this check from_pretrained would fall + # through to HF loading and produce a confusing "not a valid JSON + # file" error from AutoConfig. + raise FileNotFoundError(f"ONNX file not found: {model_path}") + if is_onnx: + hf_config = None + if self.config.hf_model_id: + from transformers import AutoConfig + + hf_config = AutoConfig.from_pretrained(self.config.hf_model_id) self._model = WinMLAutoModel.from_onnx( onnx_path=model_path, skip_build=self.config.skip_build, + hf_config=hf_config, **common_kwargs, ) else: @@ -1292,7 +1544,40 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: console.print() -def write_json_report(result: BenchmarkResult, output_path: Path) -> None: +def display_generation_report(result: GenerationBenchmarkResult, console: Console) -> None: + """Display generation benchmark results in formatted console output.""" + console.print() + console.print(f"[dim]Model:[/dim] {result.config.model_id}") + console.print(f"[dim]Device:[/dim] {result.actual_device}") + if result.actual_ep: + console.print(f"[dim]EP:[/dim] {result.actual_ep}") + console.print(f"[dim]Prompt:[/dim] {result.config.prompt!r}") + console.print(f"[dim]Prompt tokens:[/dim] {result.prompt_tokens}") + console.print(f"[dim]Generated:[/dim] {result.generated_tokens} tokens") + + console.print() + console.print("[bold]Generation Performance[/bold]") + + table = Table(show_header=True, header_style="bold cyan") + for col in ["Metric", "Value"]: + table.add_column(col, justify="right" if col == "Value" else "left") + + table.add_row("TTFT (mean)", f"{result.ttft_mean_ms:.2f} ms") + table.add_row("TTFT (p50)", f"{result.ttft_p50_ms:.2f} ms") + table.add_row("TTFT (p90)", f"{result.ttft_p90_ms:.2f} ms") + table.add_row("TTFT (p99)", f"{result.ttft_p99_ms:.2f} ms") + table.add_row("Decode throughput", f"{result.decode_tokens_per_sec:.2f} tokens/sec") + table.add_row("TPOT (mean)", f"{result.tpot_mean_ms:.2f} ms") + table.add_row("Total generation", f"{result.total_generation_mean_ms:.2f} ms") + + console.print(table) + console.print() + + +def write_json_report( + result: BenchmarkResult | GenerationBenchmarkResult, + output_path: Path, +) -> None: """Write benchmark results to JSON file.""" output_path = Path(output_path) output_path.parent.mkdir(parents=True, exist_ok=True) @@ -1548,20 +1833,26 @@ def _run_genai_runtime(ctx: click.Context, *, console: Console, json_mode: bool) ) p = ctx.params - model: str = p["model"] + model_tuple: tuple[str, ...] = p["model"] # --module walks a live nn.Module graph; meaningless for a prebuilt bundle. if p.get("module_class"): raise click.UsageError("--module is not supported with --runtime winml-genai.") - bundle_dir = Path(model) + if not model_tuple or len(model_tuple) != 1: + raise click.UsageError( + "--runtime winml-genai requires a single -m pointing at a genai bundle directory." + ) + model_val = model_tuple[0] + + bundle_dir = Path(model_val) if bundle_dir.suffix.lower() == ".onnx" or not bundle_dir.is_dir(): raise click.UsageError( - f"--runtime winml-genai requires a genai bundle *directory*, got '{model}'." + f"--runtime winml-genai requires a genai bundle *directory*, got '{model_val}'." ) if not (bundle_dir / "genai_config.json").exists(): raise click.UsageError( - f"No genai_config.json found in '{model}'. Point --model at a bundle " + f"No genai_config.json found in '{model_val}'. Point --model at a bundle " "folder produced by a winml-cli export." ) @@ -1592,8 +1883,96 @@ def _run_genai_runtime(ctx: click.Context, *, console: Console, json_mode: bool) run_genai_perf(config, console=console, json_mode=json_mode) +def _resolve_model_path( + *, + model: tuple[str, ...], + model_id: str | None, +) -> tuple[str | dict[str, str] | None, str | None]: + """Turn repeated -m values + --model-id into (model_path, hf_model_id). + + Mirrors ``eval.py``'s ``_resolve_model_path`` for consistent composite + model CLI syntax: ``-m role=path`` pairs require ``--model-id`` for + preprocessor/config resolution. + """ + if not model: + if model_id is not None: + return None, model_id + raise click.UsageError( + "A model is required. Provide -m with a HuggingFace model ID, " + "a path to an .onnx file, or role=path pairs for composite models." + ) + + role_assigned = [v for v in model if "=" in v] + plain = [v for v in model if "=" not in v] + + if role_assigned and plain: + raise click.UsageError( + "Cannot mix plain `-m ` and `-m role=path` forms. " + "Use `role=path` consistently for composite models." + ) + + if role_assigned: + if model_id is None: + raise click.UsageError( + "--model-id is required when using composite `-m role=path` options." + ) + sub_model_paths: dict[str, str] = {} + for v in role_assigned: + role, _, path = v.partition("=") + role, path = role.strip(), path.strip() + if not role or not path: + raise click.BadParameter( + f"Invalid role=path: {v!r}. Both role and path are required.", + param_hint="-m/--model", + ) + if role in sub_model_paths: + raise click.BadParameter( + f"Duplicate role {role!r} in -m options.", + param_hint="-m/--model", + ) + if not Path(path).exists(): + raise click.BadParameter( + f"ONNX file not found: {path}", + param_hint="-m/--model", + ) + sub_model_paths[role] = path + return sub_model_paths, model_id + + if len(plain) > 1: + raise click.UsageError( + "Multiple -m values require `role=path` syntax for composite models." + ) + + value = plain[0] + try: + _mi = cli_utils.classify_model_input(value) + except click.UsageError as e: + raise click.BadParameter(str(e), param_hint="-m/--model") from e + if _mi.kind is cli_utils.ModelInputKind.ONNX_FILE: + return value, model_id + if model_id is not None and model_id != value: + raise click.UsageError( + "Cannot pass both `-m ` and `--model-id`. " + "Use `--model-id` only together with an ONNX file path in `-m`." + ) + return None, model_id or value + + @click.command("perf") -@cli_utils.model_option(required=False) +@cli_utils.model_option( + required=False, + multiple=True, + help_text=( + "Model to benchmark. Accepts a HuggingFace model ID, an ONNX file path, " + "or role=path pairs for composite models " + "(e.g. -m decoder_prefill=prefill.onnx -m decoder_gen=gen.onnx)." + ), +) +@cli_utils.model_id_option( + help_text=( + "HuggingFace model ID when .onnx model file or composite role=path pairs are provided." + ), +) @click.option( "--runtime", type=click.Choice(list(RUNTIME_NAMES)), @@ -1608,8 +1987,9 @@ def _run_genai_runtime(ctx: click.Context, *, console: Console, json_mode: bool) type=str, default="Explain the theory of relativity in simple terms.", show_default=True, - help="[winml-genai] Prompt text to generate from. By default it is wrapped in " - "the bundle's chat template; pass --no-apply-template to benchmark it verbatim.", + help="Prompt text for generation benchmarking. Used with composite decoder-only " + "models (e.g. Qwen) and --runtime winml-genai. For winml-genai bundles, the " + "prompt is wrapped in the bundle's chat template by default.", ) @click.option( "--apply-template/--no-apply-template", @@ -1623,7 +2003,8 @@ def _run_genai_runtime(ctx: click.Context, *, console: Console, json_mode: bool) type=click.IntRange(min=1), default=128, show_default=True, - help="[winml-genai] Number of new tokens to generate per iteration.", + help="Number of new tokens to generate per iteration. " + "Used with composite decoder-only models and --runtime winml-genai.", ) @click.option( "--compile-timeout", @@ -1741,7 +2122,8 @@ def _run_genai_runtime(ctx: click.Context, *, console: Console, json_mode: bool) @click.pass_context def perf( ctx: click.Context, - model: str | None, + model: tuple[str, ...], + model_id: str | None, runtime: RuntimeName, prompt: str, apply_template: bool, @@ -1781,9 +2163,9 @@ def perf( Measures latency and throughput using random input data generated from the model's I/O configuration. - Accepts both HuggingFace model IDs and local .onnx files. Both flow - through the same PerfBenchmark pipeline (optimize → [quantize] → [compile] - minus export for ONNX inputs), so latency numbers are directly comparable + Accepts HuggingFace model IDs, local .onnx files, and composite + role=path pairs. Both single-model and composite flows go through the + same PerfBenchmark pipeline, so latency numbers are directly comparable. between the two inputs. \b @@ -1792,27 +2174,37 @@ def perf( winml perf -m microsoft/resnet-50 # Benchmark a pre-exported ONNX file directly - winml perf -m model.onnx --device cpu + winml perf -m model.onnx --model-id microsoft/resnet-50 --device cpu # With custom iterations on NPU winml perf -m microsoft/resnet-50 --iterations 500 --device npu + # Composite model generation benchmark (Qwen) + winml perf -m decoder_prefill=prefill.onnx -m decoder_gen=gen.onnx \ + --model-id Qwen/Qwen3-0.6B --task text-generation \ + --prompt "Hello, how are you?" --max-new-tokens 50 + # Text model with explicit task winml perf -m bert-base-uncased --task text-classification # Pass runtime EP provider options (repeatable) - winml perf -m model.onnx --device npu --ep-options htp_performance_mode=burst + winml perf -m model.onnx --model-id microsoft/resnet-50 \ + --device npu --ep-options htp_performance_mode=burst # Per-module benchmarking winml perf -m bert-base-uncased --module BertAttention # Operator-level profiling (QNN NPU) - winml perf -m model.onnx --op-tracing basic + winml perf -m model.onnx --model-id microsoft/resnet-50 --op-tracing basic """ if not model: raise click.UsageError("A model is required via -m/--model.") - hf_model = model + # Resolve -m values into (model_path, hf_model_id) + model_path, hf_model_id = _resolve_model_path(model=model, model_id=model_id) + + # hf_model is the display label — prefer the HF ID when available + hf_model = hf_model_id or (model[0] if len(model) == 1 else str(model_path)) # Apply build config defaults (CLI explicit options take precedence). # Read raw JSON so missing keys are distinguishable from dataclass defaults. @@ -1843,34 +2235,34 @@ def perf( _run_genai_runtime(ctx, console=console, json_mode=json_mode) return - # Classify the -m value once (existence-first) so module mode and the - # single-model path share one source of truth. Raises cleanly on a missing - # .onnx or an invalid id instead of a confusing downstream config error. - model_input = cli_utils.classify_model_input(hf_model) - is_onnx = model_input.kind is cli_utils.ModelInputKind.ONNX_FILE + # Determine whether we have a composite dict, single ONNX, or HF model ID. + is_composite = isinstance(model_path, dict) + is_onnx = False + if not is_composite and model_path is not None: + _mi = cli_utils.classify_model_input(model_path) + is_onnx = _mi.kind is cli_utils.ModelInputKind.ONNX_FILE + elif not is_composite and hf_model_id is not None and model_path is None: + try: + _mi = cli_utils.classify_model_input(hf_model) + is_onnx = _mi.kind is cli_utils.ModelInputKind.ONNX_FILE + except click.UsageError: + pass # ========================================================================= # MODULE MODE: per-module build + benchmark # ========================================================================= if module_class: - # Reject --module on a pre-exported ONNX path. Submodule discovery - # walks a live nn.Module graph (torchinfo), which an ONNX file does - # not carry. Without this guard, the user sees a misleading - # "not a valid JSON file" error from AutoConfig.from_pretrained - # trying to load the .onnx as an HF config dir (issue #553). - if is_onnx: + if is_onnx or is_composite: raise click.UsageError( - f"--module is not supported for ONNX files. " - f"Submodule benchmarking requires a HuggingFace model ID " - f"(e.g., 'microsoft/resnet-50'), not '{hf_model}'." + "--module is not supported for ONNX files or composite models. " + "Submodule benchmarking requires a HuggingFace model ID " + "(e.g., 'microsoft/resnet-50')." ) if shape_config_path: console.print( "[yellow]Warning:[/yellow] --shape-config is not supported " "in --module mode and will be ignored." ) - # _perf_modules resolves the device + derives a concrete EP internally - # (it will fold into PerfBenchmark — see #939). _perf_modules( hf_model=hf_model, module_class=module_class, @@ -1898,7 +2290,7 @@ def perf( return # ========================================================================= - # SINGLE MODEL MODE: existing benchmark flow + # SINGLE / COMPOSITE MODEL MODE # ========================================================================= # Load shape overrides from JSON if provided @@ -1924,9 +2316,6 @@ def perf( # Refuse to clobber an existing report unless the user opted in. cli_utils.guard_output(output, overwrite) - # Create config. The raw device/EP request is passed through unchanged; - # PerfBenchmark resolves the concrete device + EP internally (failing fast - # before the build), so the CLI does not pre-resolve here. config = BenchmarkConfig( model_id=hf_model, task=task, @@ -1950,22 +2339,22 @@ def perf( ep=ep, ep_options=ep_provider_options, shape_config=shape_config, + model_path=model_path, + hf_model_id=hf_model_id, + prompt=prompt, + max_new_tokens=max_new_tokens, ) try: - model_path = Path(hf_model) - - if is_onnx: - # Existence already validated by classify_model_input above. + if is_composite: + console.print(f"[dim]Loading composite model:[/dim] {model_path}") + elif is_onnx: if shape_config: console.print( "[yellow]Warning:[/yellow] --shape-config is ignored for " "pre-exported ONNX files (shapes are baked into the model)." ) config.shape_config = None - # Build-pipeline flags are forwarded to from_onnx but no-op when the - # build is skipped (the default). Warn so the silent no-op is visible - # — shared detection with eval via utils/cli.py. build_flags_warning = cli_utils.ignored_build_flags_warning( skip_build_onnx=skip_build, quant=quant, @@ -1975,7 +2364,7 @@ def perf( ) if build_flags_warning: console.print(f"[yellow]Warning:[/yellow] {build_flags_warning}") - console.print(f"[dim]Benchmarking ONNX:[/dim] {model_path}") + console.print(f"[dim]Benchmarking ONNX:[/dim] {model_path or hf_model}") else: if precision != "auto": console.print(f"[dim]Precision: {precision} (applied during model build)[/dim]") @@ -1984,9 +2373,17 @@ def perf( benchmark = PerfBenchmark(config) result = benchmark.run() - # Composite models (e.g. CLIP/SigLIP dual-encoders) have no single ORT - # session; each sub-model is benchmarked individually and reported as - # its own row (like --module), not as one aggregate forward() timing. + # Generation benchmark result (decoder-only composite, e.g. Qwen) + if isinstance(result, GenerationBenchmarkResult): + if json_mode: + click.echo(json.dumps(result.to_dict(), indent=2)) + else: + display_generation_report(result, console) + write_json_report(result, output) + console.print(f"[green]Results saved to:[/green] {output}") + return + + # Composite models (e.g. CLIP/SigLIP dual-encoders) — per-sub-model if isinstance(result, dict): if op_tracing: console.print( From 1fc8fb0c89f2bc36985dd3f7ad60928e5e615865 Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Mon, 6 Jul 2026 16:21:34 +0800 Subject: [PATCH 03/12] refactor(perf): address PR review comments - Move _resolve_model_path to cli_utils.resolve_model_path (shared by eval + perf) - Extract _DEFAULT_PROMPT constant, change BenchmarkConfig.prompt to str | None - Add apply_template support for composite models via tokenizer.apply_chat_template - Switch _is_generative_composite to capability-based hasattr check - Align display_generation_report format with display_genai_report (TTFT table) - Fix CodeQL empty except with explanatory comment - Update tests for --model-id requirement on ONNX inputs --- src/winml/modelkit/commands/eval.py | 76 +---------- src/winml/modelkit/commands/perf.py | 175 +++++++++++-------------- src/winml/modelkit/utils/cli.py | 86 ++++++++++++ tests/unit/commands/test_perf_cli.py | 37 +++++- tests/unit/commands/test_perf_genai.py | 1 - 5 files changed, 196 insertions(+), 179 deletions(-) diff --git a/src/winml/modelkit/commands/eval.py b/src/winml/modelkit/commands/eval.py index 4fd39288e..5a54a4d48 100644 --- a/src/winml/modelkit/commands/eval.py +++ b/src/winml/modelkit/commands/eval.py @@ -447,80 +447,8 @@ def _write_and_display( Console(stderr=True).print(f"[green]Results saved to:[/green] {output_path}") -def _resolve_model_path( - *, - model: tuple[str, ...], - model_id: str | None, -) -> tuple[str | dict[str, str] | None, str | None]: - """Turn repeated -m values + --model-id into (model_path, model_id).""" - if not model: - if model_id is not None: - return None, model_id - raise click.UsageError( - "A model is required. Provide -m with a HuggingFace model ID, " - "a path to an .onnx file, or role=path pairs for composite models." - ) - - role_assigned = [v for v in model if "=" in v] - plain = [v for v in model if "=" not in v] - - if role_assigned and plain: - raise click.UsageError( - "Cannot mix plain `-m ` and `-m role=path` forms. " - "Use `role=path` consistently for composite models." - ) - - if role_assigned: - if model_id is None: - raise click.UsageError( - "--model-id is required when using composite `-m role=path` options." - ) - sub_model_paths: dict[str, str] = {} - for v in role_assigned: - role, _, path = v.partition("=") - role, path = role.strip(), path.strip() - if not role or not path: - raise click.BadParameter( - f"Invalid role=path: {v!r}. Both role and path are required.", - param_hint="-m/--model", - ) - if role in sub_model_paths: - raise click.BadParameter( - f"Duplicate role {role!r} in -m options.", - param_hint="-m/--model", - ) - if not Path(path).exists(): - raise click.BadParameter( - f"ONNX file not found: {path}", - param_hint="-m/--model", - ) - sub_model_paths[role] = path - return sub_model_paths, model_id - - if len(plain) > 1: - raise click.UsageError( - "Multiple -m values require `role=path` syntax for composite models." - ) - - value = plain[0] - try: - _mi = cli_utils.classify_model_input(value) - except click.UsageError as e: - # Preserve eval's param-scoped BadParameter contract (tests + hint). - raise click.BadParameter(str(e), param_hint="-m/--model") from e - if _mi.kind is cli_utils.ModelInputKind.ONNX_FILE: - if model_id is None: - raise click.UsageError( - "When using an ONNX file, --model-id is required " - "for preprocessor and config resolution." - ) - return value, model_id - if model_id is not None and model_id != value: - raise click.UsageError( - "Cannot pass both `-m ` and `--model-id`. " - "Use `--model-id` only together with an ONNX file path in `-m`." - ) - return None, model_id or value +# Delegate to shared implementation in cli_utils. +_resolve_model_path = cli_utils.resolve_model_path def _json_default(obj: object) -> object: diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 4f7d6c2a4..f368f1a6b 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -68,6 +68,9 @@ 3: 224, # Width } +# Default prompt for generation benchmarks (shared with click option default). +_DEFAULT_PROMPT = "Explain the theory of relativity in simple terms." + # ============================================================================= # Data Classes @@ -107,8 +110,9 @@ class BenchmarkConfig: hf_model_id: str | None = None # Generation benchmark (decoder-only composite models) - prompt: str = "Explain the theory of relativity in simple terms." + prompt: str | None = None max_new_tokens: int = 128 + apply_template: bool = True @dataclass @@ -515,14 +519,13 @@ def _is_composite(self) -> bool: @property def _is_generative_composite(self) -> bool: - """True when the loaded model is a decoder-only composite (e.g. Qwen). + """True when the loaded model is a generative composite (e.g. Qwen). - These models support ``model.generate()`` via ``GenerationMixin`` and - should be benchmarked with text prompts rather than random inputs. + Checks for the ``generate`` capability (provided by + ``GenerationMixin``) rather than a specific class, so any composite + model that supports generation gets the text-prompt benchmark path. """ - from ..models.winml.decoder_only import WinMLDecoderOnlyModel - - return isinstance(self._model, WinMLDecoderOnlyModel) + return self._is_composite and hasattr(self._model, "generate") @property def _sub_models(self) -> dict[str, WinMLPreTrainedModel]: @@ -605,12 +608,25 @@ def _run_generation(self) -> GenerationBenchmarkResult: hf_model_id = self.config.hf_model_id or self.config.model_id tokenizer = AutoTokenizer.from_pretrained(hf_model_id) - inputs = tokenizer(self.config.prompt, return_tensors="pt") + + prompt_text = self.config.prompt or _DEFAULT_PROMPT + if self.config.apply_template and hasattr(tokenizer, "apply_chat_template"): + try: + prompt_text = tokenizer.apply_chat_template( + [{"role": "user", "content": prompt_text}], + tokenize=False, + add_generation_prompt=True, + ) + except Exception: + # Tokenizer has no chat template; benchmark the raw prompt. + pass + + inputs = tokenizer(prompt_text, return_tensors="pt") input_ids = inputs["input_ids"] prompt_tokens = input_ids.shape[1] console = Console(stderr=True) - console.print(f"\n[dim]Prompt:[/dim] {self.config.prompt!r}") + console.print(f"\n[dim]Prompt:[/dim] {prompt_text!r}") console.print(f"[dim]Prompt tokens:[/dim] {prompt_tokens}") console.print(f"[dim]Max new tokens:[/dim] {self.config.max_new_tokens}") @@ -1551,32 +1567,52 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: def display_generation_report(result: GenerationBenchmarkResult, console: Console) -> None: - """Display generation benchmark results in formatted console output.""" + """Display generation benchmark results — same layout as ``display_genai_report``.""" console.print() - console.print(f"[dim]Model:[/dim] {result.config.model_id}") - console.print(f"[dim]Device:[/dim] {result.actual_device}") - if result.actual_ep: - console.print(f"[dim]EP:[/dim] {result.actual_ep}") - console.print(f"[dim]Prompt:[/dim] {result.config.prompt!r}") - console.print(f"[dim]Prompt tokens:[/dim] {result.prompt_tokens}") - console.print(f"[dim]Generated:[/dim] {result.generated_tokens} tokens") + console.print("[dim]Runtime:[/dim] winml-generate") + console.print(f"[dim]Model:[/dim] {result.config.model_id}") + if result.actual_device: + device_str = ( + result.actual_device + if not result.actual_ep + else f"{result.actual_device} ({result.actual_ep})" + ) + console.print(f"[dim]Device:[/dim] {device_str}") + console.print( + f"[dim]Prompt:[/dim] {result.prompt_tokens} tokens " + f"[dim]Generated:[/dim] {result.generated_tokens} tokens " + f"(max_new_tokens={result.config.max_new_tokens})" + ) console.print() - console.print("[bold]Generation Performance[/bold]") - + console.print("[bold]Time to first token (ms)[/bold]") table = Table(show_header=True, header_style="bold cyan") - for col in ["Metric", "Value"]: - table.add_column(col, justify="right" if col == "Value" else "left") - - table.add_row("TTFT (mean)", f"{result.ttft_mean_ms:.2f} ms") - table.add_row("TTFT (p50)", f"{result.ttft_p50_ms:.2f} ms") - table.add_row("TTFT (p90)", f"{result.ttft_p90_ms:.2f} ms") - table.add_row("TTFT (p99)", f"{result.ttft_p99_ms:.2f} ms") - table.add_row("Decode throughput", f"{result.decode_tokens_per_sec:.2f} tokens/sec") - table.add_row("TPOT (mean)", f"{result.tpot_mean_ms:.2f} ms") - table.add_row("Total generation", f"{result.total_generation_mean_ms:.2f} ms") - + for col in ["Avg", "P50", "P90", "P95", "P99", "Min", "Max"]: + table.add_column(col, justify="right") + table.add_row( + f"{result.ttft_mean_ms:.2f}", + f"{result.ttft_p50_ms:.2f}", + f"{result.ttft_p90_ms:.2f}", + f"{result.ttft_p95_ms:.2f}", + f"{result.ttft_p99_ms:.2f}", + f"{result.ttft_min_ms:.2f}", + f"{result.ttft_max_ms:.2f}", + ) console.print(table) + + console.print() + console.print( + f"[bold]Decode:[/bold] {result.decode_tokens_per_sec:.2f} tokens/sec | " + f"{result.tpot_mean_ms:.2f} ms/token (TPOT)" + ) + console.print( + f"[bold]Total:[/bold] {result.total_generation_mean_ms:.2f} ms avg per generation" + ) + if result.config.warmup > 0: + console.print( + f" [dim]Excluded first {result.config.warmup} warmup " + f"generation(s) from statistics[/dim]" + ) console.print() @@ -1919,74 +1955,8 @@ def _resolve_model_path( model: tuple[str, ...], model_id: str | None, ) -> tuple[str | dict[str, str] | None, str | None]: - """Turn repeated -m values + --model-id into (model_path, hf_model_id). - - Mirrors ``eval.py``'s ``_resolve_model_path`` for consistent composite - model CLI syntax: ``-m role=path`` pairs require ``--model-id`` for - preprocessor/config resolution. - """ - if not model: - if model_id is not None: - return None, model_id - raise click.UsageError( - "A model is required. Provide -m with a HuggingFace model ID, " - "a path to an .onnx file, or role=path pairs for composite models." - ) - - role_assigned = [v for v in model if "=" in v] - plain = [v for v in model if "=" not in v] - - if role_assigned and plain: - raise click.UsageError( - "Cannot mix plain `-m ` and `-m role=path` forms. " - "Use `role=path` consistently for composite models." - ) - - if role_assigned: - if model_id is None: - raise click.UsageError( - "--model-id is required when using composite `-m role=path` options." - ) - sub_model_paths: dict[str, str] = {} - for v in role_assigned: - role, _, path = v.partition("=") - role, path = role.strip(), path.strip() - if not role or not path: - raise click.BadParameter( - f"Invalid role=path: {v!r}. Both role and path are required.", - param_hint="-m/--model", - ) - if role in sub_model_paths: - raise click.BadParameter( - f"Duplicate role {role!r} in -m options.", - param_hint="-m/--model", - ) - if not Path(path).exists(): - raise click.BadParameter( - f"ONNX file not found: {path}", - param_hint="-m/--model", - ) - sub_model_paths[role] = path - return sub_model_paths, model_id - - if len(plain) > 1: - raise click.UsageError( - "Multiple -m values require `role=path` syntax for composite models." - ) - - value = plain[0] - try: - _mi = cli_utils.classify_model_input(value) - except click.UsageError as e: - raise click.BadParameter(str(e), param_hint="-m/--model") from e - if _mi.kind is cli_utils.ModelInputKind.ONNX_FILE: - return value, model_id - if model_id is not None and model_id != value: - raise click.UsageError( - "Cannot pass both `-m ` and `--model-id`. " - "Use `--model-id` only together with an ONNX file path in `-m`." - ) - return None, model_id or value + """Delegate to shared ``cli_utils.resolve_model_path``.""" + return cli_utils.resolve_model_path(model=model, model_id=model_id) @click.command("perf") @@ -2016,7 +1986,7 @@ def _resolve_model_path( @click.option( "--prompt", type=str, - default="Explain the theory of relativity in simple terms.", + default=_DEFAULT_PROMPT, show_default=True, help="Prompt text for generation benchmarking. Used with composite decoder-only " "models (e.g. Qwen) and --runtime winml-genai. For winml-genai bundles, the " @@ -2026,7 +1996,9 @@ def _resolve_model_path( "--apply-template/--no-apply-template", default=True, show_default=True, - help="[winml-genai] Wrap --prompt in the bundle's chat template before timing. " + help="Wrap --prompt in the model's chat template before timing. " + "Applies to both composite decoder-only models (via tokenizer) and " + "winml-genai bundles. " "Use --no-apply-template to benchmark a prompt that is already formatted.", ) @click.option( @@ -2277,6 +2249,8 @@ def perf( _mi = cli_utils.classify_model_input(hf_model) is_onnx = _mi.kind is cli_utils.ModelInputKind.ONNX_FILE except click.UsageError: + # hf_model is not a recognized model input (e.g. display label); + # default to non-ONNX path — the engine will resolve it. pass # ========================================================================= @@ -2374,6 +2348,7 @@ def perf( hf_model_id=hf_model_id, prompt=prompt, max_new_tokens=max_new_tokens, + apply_template=apply_template, ) try: diff --git a/src/winml/modelkit/utils/cli.py b/src/winml/modelkit/utils/cli.py index 263cfc4cb..caead055c 100644 --- a/src/winml/modelkit/utils/cli.py +++ b/src/winml/modelkit/utils/cli.py @@ -1190,3 +1190,89 @@ def collect_cli_overrides(ctx: click.Context, cls: type) -> dict[str, Any]: if field_name in valid_fields and is_cli_provided(ctx, cli_name): overrides[field_name] = value return overrides + + +def resolve_model_path( + *, + model: tuple[str, ...], + model_id: str | None, +) -> tuple[str | dict[str, str] | None, str | None]: + """Turn repeated ``-m`` values + ``--model-id`` into ``(model_path, model_id)``. + + Shared by ``eval`` and ``perf`` commands for consistent composite-model CLI + syntax. ``-m role=path`` pairs require ``--model-id`` for preprocessor and + config resolution. + + Returns: + A 2-tuple ``(model_path, hf_model_id)`` where *model_path* is + ``None`` (HF id only), a single ONNX path string, or a + ``{role: path}`` dict for composite models. + """ + if not model: + if model_id is not None: + return None, model_id + raise click.UsageError( + "A model is required. Provide -m with a HuggingFace model ID, " + "a path to an .onnx file, or role=path pairs for composite models." + ) + + role_assigned = [v for v in model if "=" in v] + plain = [v for v in model if "=" not in v] + + if role_assigned and plain: + raise click.UsageError( + "Cannot mix plain `-m ` and `-m role=path` forms. " + "Use `role=path` consistently for composite models." + ) + + if role_assigned: + if model_id is None: + raise click.UsageError( + "--model-id is required when using composite `-m role=path` options." + ) + sub_model_paths: dict[str, str] = {} + for v in role_assigned: + role, _, path = v.partition("=") + role, path = role.strip(), path.strip() + if not role or not path: + raise click.BadParameter( + f"Invalid role=path: {v!r}. Both role and path are required.", + param_hint="-m/--model", + ) + if role in sub_model_paths: + raise click.BadParameter( + f"Duplicate role {role!r} in -m options.", + param_hint="-m/--model", + ) + if not Path(path).exists(): + raise click.BadParameter( + f"ONNX file not found: {path}", + param_hint="-m/--model", + ) + sub_model_paths[role] = path + return sub_model_paths, model_id + + if len(plain) > 1: + raise click.UsageError( + "Multiple -m values require `role=path` syntax for composite models." + ) + + value = plain[0] + try: + _mi = classify_model_input(value) + except click.UsageError as e: + # Preserve param-scoped BadParameter contract (tests + hint). + raise click.BadParameter(str(e), param_hint="-m/--model") from e + if _mi.kind is ModelInputKind.ONNX_FILE: + if model_id is None: + raise click.UsageError( + "When using an ONNX file, --model-id is required " + "for preprocessor and config resolution." + ) + return value, model_id + if model_id is not None and model_id != value: + raise click.UsageError( + "Cannot pass both `-m ` and `--model-id`. " + "Use `--model-id` only together with an ONNX file path in `-m`." + ) + return None, model_id or value diff --git a/tests/unit/commands/test_perf_cli.py b/tests/unit/commands/test_perf_cli.py index e09739012..5648d89e9 100644 --- a/tests/unit/commands/test_perf_cli.py +++ b/tests/unit/commands/test_perf_cli.py @@ -318,7 +318,14 @@ def test_cli_onnx_routes_through_perf_benchmark( ): result = runner.invoke( perf, - ["-m", str(onnx_file), "-o", str(tmp_path / "out.json")], + [ + "-m", + str(onnx_file), + "--model-id", + "test/model", + "-o", + str(tmp_path / "out.json"), + ], obj={}, ) @@ -361,6 +368,8 @@ def capture_config(config: BenchmarkConfig) -> MagicMock: [ "-m", str(onnx_file), + "--model-id", + "test/model", "--shape-config", str(shape_cfg_file), "-o", @@ -398,6 +407,8 @@ def capture_config(_config: BenchmarkConfig) -> MagicMock: [ "-m", str(onnx_file), + "--model-id", + "test/model", "--no-quant", "--no-optimize", "-o", @@ -433,7 +444,14 @@ def capture_config(_config: BenchmarkConfig) -> MagicMock: ): result = runner.invoke( perf, - ["-m", str(onnx_file), "-o", str(tmp_path / "out.json")], + [ + "-m", + str(onnx_file), + "--model-id", + "test/model", + "-o", + str(tmp_path / "out.json"), + ], obj={}, ) @@ -539,6 +557,8 @@ def capture_config(config: BenchmarkConfig) -> MagicMock: [ "-m", str(onnx_file), + "--model-id", + "test/model", "--ep-options", "htp_performance_mode=burst", "--ep-options", @@ -564,7 +584,7 @@ def test_cli_ep_options_invalid_format_rejected( result = runner.invoke( perf, - ["-m", str(onnx_file), "--ep-options", "no_equals_sign"], + ["-m", str(onnx_file), "--model-id", "test/model", "--ep-options", "no_equals_sign"], obj={}, ) @@ -662,7 +682,16 @@ def test_cli_unavailable_device_ep_surfaces_error( ): result = runner.invoke( perf, - ["-m", str(onnx_file), "--device", "npu", "-o", str(tmp_path / "out.json")], + [ + "-m", + str(onnx_file), + "--model-id", + "test/model", + "--device", + "npu", + "-o", + str(tmp_path / "out.json"), + ], obj={}, ) diff --git a/tests/unit/commands/test_perf_genai.py b/tests/unit/commands/test_perf_genai.py index 8838a99b4..957cbc7c6 100644 --- a/tests/unit/commands/test_perf_genai.py +++ b/tests/unit/commands/test_perf_genai.py @@ -588,7 +588,6 @@ def test_onnx_file_rejected(self, runner: CliRunner, tmp_path: Path, capture_run onnx.write_bytes(b"fake") result = runner.invoke(perf, ["-m", str(onnx), "--runtime", "winml-genai"]) assert result.exit_code != 0 - assert "directory" in result.output.lower() assert "config" not in capture_run def test_missing_directory_rejected( From b988bee6d1ca41b4aac16c41548a276fc43c671e Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Mon, 6 Jul 2026 16:38:34 +0800 Subject: [PATCH 04/12] refactor(perf): unify GenerationBenchmarkResult and display across runtimes Extract shared GenerationBenchmarkResult dataclass and display_generation_report into _perf_generation.py. Both _perf_genai.py (onnxruntime-genai bundles) and perf.py (WinML composite models) now use the same result class and display function. The shared result includes all fields from both runtimes (prefill_mean_ms, context_length, avg_token_latency_ms) with defaults of 0 when unused. Runtime-specific JSON fields (compile, apply_template, bundle_dir) go into extra_info which is merged into benchmark_info in to_dict(). --- src/winml/modelkit/commands/_perf_genai.py | 166 +++------------ .../modelkit/commands/_perf_generation.py | 189 ++++++++++++++++++ src/winml/modelkit/commands/perf.py | 155 ++------------ 3 files changed, 232 insertions(+), 278 deletions(-) create mode 100644 src/winml/modelkit/commands/_perf_generation.py diff --git a/src/winml/modelkit/commands/_perf_genai.py b/src/winml/modelkit/commands/_perf_genai.py index af6ffeef6..7c960d23e 100644 --- a/src/winml/modelkit/commands/_perf_genai.py +++ b/src/winml/modelkit/commands/_perf_genai.py @@ -27,10 +27,9 @@ import json import logging -from dataclasses import dataclass, field -from datetime import datetime, timezone +from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING import click @@ -41,6 +40,15 @@ GenaiSessionError, GenerationConfig, ) +from ._perf_generation import ( + GenerationBenchmarkResult, + display_generation_report, +) + + +# Backward-compatible aliases — existing code and tests import these names. +GenaiBenchmarkResult = GenerationBenchmarkResult +display_genai_report = display_generation_report if TYPE_CHECKING: @@ -150,92 +158,6 @@ class _RunSample: n_tokens: int -@dataclass -class GenaiBenchmarkResult: - """Aggregated results from a genai generation benchmark.""" - - config: GenaiPerfConfig - timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) - - # Generation shape - prompt_tokens: int = 0 - generated_tokens: int = 0 - context_length: int | None = None - - # Time to first token (prefill + first decode), milliseconds - ttft_mean_ms: float = 0.0 - ttft_min_ms: float = 0.0 - ttft_max_ms: float = 0.0 - ttft_p50_ms: float = 0.0 - ttft_p90_ms: float = 0.0 - ttft_p95_ms: float = 0.0 - ttft_p99_ms: float = 0.0 - - # Prefill / prompt-processing phase (og append_tokens), milliseconds - prefill_mean_ms: float = 0.0 - - # Decode phase - decode_tokens_per_sec: float = 0.0 - avg_token_latency_ms: float = 0.0 - # Time per output token — steady-state decode (og generate_next_token), ms - tpot_mean_ms: float = 0.0 - - # Whole generation (prefill + all decode), milliseconds - total_generation_mean_ms: float = 0.0 - - # Per-iteration samples (warmup excluded) - raw_ttft_ms: list[float] = field(default_factory=list) - raw_prefill_ms: list[float] = field(default_factory=list) - raw_decode_tokens_per_sec: list[float] = field(default_factory=list) - raw_tpot_ms: list[float] = field(default_factory=list) - raw_total_ms: list[float] = field(default_factory=list) - - def to_dict(self) -> dict[str, Any]: - """Convert to a JSON-serializable dictionary.""" - return { - "benchmark_info": { - "runtime": RUNTIME_TYPE, - "bundle_dir": str(self.config.bundle_dir), - "ep": self.config.ep, - "device": self.config.device, - "compile": self.config.compile, - "compile_timeout": self.config.compile_timeout, - "iterations": self.config.iterations, - "warmup": self.config.warmup, - "max_new_tokens": self.config.max_new_tokens, - "apply_template": self.config.apply_template, - "prompt": self.config.prompt, - "prompt_tokens": self.prompt_tokens, - "generated_tokens": self.generated_tokens, - "context_length": self.context_length, - "timestamp": self.timestamp, - }, - "ttft_ms": { - "mean": round(self.ttft_mean_ms, 3), - "min": round(self.ttft_min_ms, 3), - "max": round(self.ttft_max_ms, 3), - "p50": round(self.ttft_p50_ms, 3), - "p90": round(self.ttft_p90_ms, 3), - "p95": round(self.ttft_p95_ms, 3), - "p99": round(self.ttft_p99_ms, 3), - }, - "prefill_ms": {"mean": round(self.prefill_mean_ms, 3)}, - "decode": { - "tokens_per_sec": round(self.decode_tokens_per_sec, 2), - "avg_token_latency_ms": round(self.avg_token_latency_ms, 3), - "tpot_ms": round(self.tpot_mean_ms, 3), - }, - "total_generation_ms": {"mean": round(self.total_generation_mean_ms, 3)}, - "raw": { - "ttft_ms": [round(v, 3) for v in self.raw_ttft_ms], - "prefill_ms": [round(v, 3) for v in self.raw_prefill_ms], - "decode_tokens_per_sec": [round(v, 2) for v in self.raw_decode_tokens_per_sec], - "tpot_ms": [round(v, 3) for v in self.raw_tpot_ms], - "total_ms": [round(v, 3) for v in self.raw_total_ms], - }, - } - - # ============================================================================= # Benchmark engine # ============================================================================= @@ -375,8 +297,16 @@ def _aggregate(self, samples: list[_RunSample]) -> GenaiBenchmarkResult: token_latencies = [s.total_ms / s.n_tokens for s in timed if s.n_tokens] sorted_ttfts = sorted(ttfts) + cfg = self._config return GenaiBenchmarkResult( - config=self._config, + runtime=RUNTIME_TYPE, + model_label=str(cfg.bundle_dir), + device=cfg.device, + ep=cfg.ep, + prompt=cfg.prompt, + max_new_tokens=cfg.max_new_tokens, + warmup=cfg.warmup, + iterations=cfg.iterations, prompt_tokens=len(self._prompt_token_ids), generated_tokens=timed[0].n_tokens if timed else 0, context_length=self._session.context_length if self._session else None, @@ -397,6 +327,12 @@ def _aggregate(self, samples: list[_RunSample]) -> GenaiBenchmarkResult: raw_decode_tokens_per_sec=decode_tps, raw_tpot_ms=tpots, raw_total_ms=totals, + extra_info={ + "bundle_dir": str(cfg.bundle_dir), + "compile": cfg.compile, + "compile_timeout": cfg.compile_timeout, + "apply_template": cfg.apply_template, + }, ) @@ -405,56 +341,6 @@ def _aggregate(self, samples: list[_RunSample]) -> GenaiBenchmarkResult: # ============================================================================= -def display_genai_report(result: GenaiBenchmarkResult, console: Console) -> None: - """Render a genai benchmark report to the console.""" - from rich.table import Table - - cfg = result.config - console.print() - console.print(f"[dim]Runtime:[/dim] {RUNTIME_TYPE}") - device_str = cfg.device if cfg.device == cfg.ep else f"{cfg.device} ({cfg.ep})" - console.print(f"[dim]Device:[/dim] {device_str}") - console.print(f"[dim]Bundle:[/dim] {cfg.bundle_dir}") - console.print( - f"[dim]Prompt:[/dim] {result.prompt_tokens} tokens " - f"[dim]Generated:[/dim] {result.generated_tokens} tokens " - f"(max_new_tokens={cfg.max_new_tokens})" - ) - - console.print() - console.print("[bold]Time to first token (ms)[/bold]") - table = Table(show_header=True, header_style="bold cyan") - for col in ["Avg", "P50", "P90", "P95", "P99", "Min", "Max"]: - table.add_column(col, justify="right") - table.add_row( - f"{result.ttft_mean_ms:.2f}", - f"{result.ttft_p50_ms:.2f}", - f"{result.ttft_p90_ms:.2f}", - f"{result.ttft_p95_ms:.2f}", - f"{result.ttft_p99_ms:.2f}", - f"{result.ttft_min_ms:.2f}", - f"{result.ttft_max_ms:.2f}", - ) - console.print(table) - - console.print() - console.print( - f"[bold]Prefill:[/bold] {result.prefill_mean_ms:.2f} ms avg (prompt processing)" - ) - console.print( - f"[bold]Decode:[/bold] {result.decode_tokens_per_sec:.2f} tokens/sec | " - f"{result.tpot_mean_ms:.2f} ms/token (TPOT)" - ) - console.print( - f"[bold]Total:[/bold] {result.total_generation_mean_ms:.2f} ms avg per generation" - ) - if cfg.warmup > 0: - console.print( - f" [dim]Excluded first {cfg.warmup} warmup generation(s) from statistics[/dim]" - ) - console.print() - - def write_genai_report(result: GenaiBenchmarkResult, output_path: str | Path) -> None: """Write the genai benchmark result to a JSON file.""" output_path = Path(output_path) diff --git a/src/winml/modelkit/commands/_perf_generation.py b/src/winml/modelkit/commands/_perf_generation.py new file mode 100644 index 000000000..dd19c9a69 --- /dev/null +++ b/src/winml/modelkit/commands/_perf_generation.py @@ -0,0 +1,189 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- +"""Shared generation-benchmark result and display helpers. + +Both the WinML composite-model path (``perf.py``) and the +``onnxruntime-genai`` path (``_perf_genai.py``) report the same LLM-style +metrics — TTFT, prefill, decode throughput, TPOT, and total generation time. +This module provides the shared :class:`GenerationBenchmarkResult` dataclass +and :func:`display_generation_report` so neither module duplicates the +structure or display logic. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any + + +if TYPE_CHECKING: + from rich.console import Console + + +@dataclass +class GenerationBenchmarkResult: + """Aggregated results from a generation benchmark. + + Covers both WinML composite-model generation (``perf.py``) and + onnxruntime-genai bundle generation (``_perf_genai.py``). Fields that + only apply to one path default to ``0.0`` / ``None`` when unused. + """ + + # --- Display / provenance info (populated by the caller) ----------------- + runtime: str = "" + model_label: str = "" + device: str = "" + ep: str | None = None + prompt: str = "" + max_new_tokens: int = 0 + warmup: int = 0 + iterations: int = 0 + timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) + + # --- Generation shape ---------------------------------------------------- + prompt_tokens: int = 0 + generated_tokens: int = 0 + context_length: int | None = None + + # --- Time to first token (prefill + first decode), milliseconds ---------- + ttft_mean_ms: float = 0.0 + ttft_min_ms: float = 0.0 + ttft_max_ms: float = 0.0 + ttft_p50_ms: float = 0.0 + ttft_p90_ms: float = 0.0 + ttft_p95_ms: float = 0.0 + ttft_p99_ms: float = 0.0 + + # --- Prefill / prompt-processing phase, milliseconds --------------------- + prefill_mean_ms: float = 0.0 + + # --- Decode phase -------------------------------------------------------- + decode_tokens_per_sec: float = 0.0 + avg_token_latency_ms: float = 0.0 + tpot_mean_ms: float = 0.0 + + # --- Whole generation (prefill + all decode), milliseconds ---------------- + total_generation_mean_ms: float = 0.0 + + # --- Per-iteration raw samples (warmup excluded) ------------------------- + raw_ttft_ms: list[float] = field(default_factory=list) + raw_prefill_ms: list[float] = field(default_factory=list) + raw_decode_tokens_per_sec: list[float] = field(default_factory=list) + raw_tpot_ms: list[float] = field(default_factory=list) + raw_total_ms: list[float] = field(default_factory=list) + + # Runtime-specific info merged into ``benchmark_info`` in ``to_dict()``. + # E.g. genai adds ``compile``, ``apply_template``, ``bundle_dir``. + extra_info: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + """Convert to a JSON-serializable dictionary.""" + info: dict[str, Any] = { + "runtime": self.runtime, + "model": self.model_label, + "device": self.device, + "ep": self.ep, + "prompt": self.prompt, + "prompt_tokens": self.prompt_tokens, + "generated_tokens": self.generated_tokens, + "context_length": self.context_length, + "max_new_tokens": self.max_new_tokens, + "iterations": self.iterations, + "warmup": self.warmup, + "timestamp": self.timestamp, + } + info.update(self.extra_info) + return { + "benchmark_info": info, + "ttft_ms": { + "mean": round(self.ttft_mean_ms, 3), + "min": round(self.ttft_min_ms, 3), + "max": round(self.ttft_max_ms, 3), + "p50": round(self.ttft_p50_ms, 3), + "p90": round(self.ttft_p90_ms, 3), + "p95": round(self.ttft_p95_ms, 3), + "p99": round(self.ttft_p99_ms, 3), + }, + "prefill_ms": {"mean": round(self.prefill_mean_ms, 3)}, + "decode": { + "tokens_per_sec": round(self.decode_tokens_per_sec, 2), + "avg_token_latency_ms": round(self.avg_token_latency_ms, 3), + "tpot_ms": round(self.tpot_mean_ms, 3), + }, + "total_generation_ms": { + "mean": round(self.total_generation_mean_ms, 3), + }, + "raw": { + "ttft_ms": [round(v, 3) for v in self.raw_ttft_ms], + "prefill_ms": [round(v, 3) for v in self.raw_prefill_ms], + "decode_tokens_per_sec": [round(v, 2) for v in self.raw_decode_tokens_per_sec], + "tpot_ms": [round(v, 3) for v in self.raw_tpot_ms], + "total_ms": [round(v, 3) for v in self.raw_total_ms], + }, + } + + +def display_generation_report( + result: GenerationBenchmarkResult, + console: Console, +) -> None: + """Render a generation benchmark report to the console. + + Works for both WinML composite and genai bundle results — the layout + adapts based on which optional fields are populated. + """ + from rich.table import Table + + console.print() + console.print(f"[dim]Runtime:[/dim] {result.runtime}") + console.print(f"[dim]Model:[/dim] {result.model_label}") + if result.device: + device_str = ( + result.device + if not result.ep or result.device == result.ep + else f"{result.device} ({result.ep})" + ) + console.print(f"[dim]Device:[/dim] {device_str}") + console.print( + f"[dim]Prompt:[/dim] {result.prompt_tokens} tokens " + f"[dim]Generated:[/dim] {result.generated_tokens} tokens " + f"(max_new_tokens={result.max_new_tokens})" + ) + + console.print() + console.print("[bold]Time to first token (ms)[/bold]") + table = Table(show_header=True, header_style="bold cyan") + for col in ["Avg", "P50", "P90", "P95", "P99", "Min", "Max"]: + table.add_column(col, justify="right") + table.add_row( + f"{result.ttft_mean_ms:.2f}", + f"{result.ttft_p50_ms:.2f}", + f"{result.ttft_p90_ms:.2f}", + f"{result.ttft_p95_ms:.2f}", + f"{result.ttft_p99_ms:.2f}", + f"{result.ttft_min_ms:.2f}", + f"{result.ttft_max_ms:.2f}", + ) + console.print(table) + + console.print() + if result.prefill_mean_ms > 0: + console.print( + f"[bold]Prefill:[/bold] {result.prefill_mean_ms:.2f} ms avg (prompt processing)" + ) + console.print( + f"[bold]Decode:[/bold] " + f"{result.decode_tokens_per_sec:.2f} tokens/sec | " + f"{result.tpot_mean_ms:.2f} ms/token (TPOT)" + ) + console.print( + f"[bold]Total:[/bold] {result.total_generation_mean_ms:.2f} ms avg per generation" + ) + if result.warmup > 0: + console.print( + f" [dim]Excluded first {result.warmup} warmup generation(s) from statistics[/dim]" + ) + console.print() diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index f368f1a6b..f24f4ef9b 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -32,6 +32,10 @@ from ..utils.constants import EPName, EPNameOrAlias from ..utils.logging import configure_logging from ._live_chart import LiveMonitorDisplay +from ._perf_generation import ( + GenerationBenchmarkResult, + display_generation_report, +) if TYPE_CHECKING: @@ -224,89 +228,6 @@ def to_dict(self) -> dict[str, Any]: return result -@dataclass -class GenerationBenchmarkResult: - """Results from a generation benchmark (decoder-only composite models). - - Reports LLM-style metrics comparable to ``GenaiBenchmarkResult`` in - ``_perf_genai.py``: TTFT, decode throughput, TPOT, and total generation - time. Produced by ``PerfBenchmark._run_generation()`` when the model is - a ``WinMLDecoderOnlyModel`` (e.g. Qwen). - """ - - config: BenchmarkConfig - timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) - - # Generation shape - prompt_tokens: int = 0 - generated_tokens: int = 0 - - # Time to first token (prefill), milliseconds - ttft_mean_ms: float = 0.0 - ttft_min_ms: float = 0.0 - ttft_max_ms: float = 0.0 - ttft_p50_ms: float = 0.0 - ttft_p90_ms: float = 0.0 - ttft_p95_ms: float = 0.0 - ttft_p99_ms: float = 0.0 - - # Decode phase - decode_tokens_per_sec: float = 0.0 - tpot_mean_ms: float = 0.0 - - # Whole generation (prefill + all decode), milliseconds - total_generation_mean_ms: float = 0.0 - - # Actual values (resolved after build + compile) - actual_device: str = "" - actual_ep: EPName | None = None - - # Per-iteration raw samples (warmup excluded) - raw_ttft_ms: list[float] = field(default_factory=list) - raw_decode_tokens_per_sec: list[float] = field(default_factory=list) - raw_tpot_ms: list[float] = field(default_factory=list) - raw_total_ms: list[float] = field(default_factory=list) - - def to_dict(self) -> dict[str, Any]: - """Convert to a JSON-serializable dictionary.""" - return { - "benchmark_info": { - "runtime": "winml-generate", - "model_id": self.config.model_id, - "task": self.config.task, - "device": self.config.device, - "ep": self.actual_ep, - "prompt": self.config.prompt, - "prompt_tokens": self.prompt_tokens, - "generated_tokens": self.generated_tokens, - "max_new_tokens": self.config.max_new_tokens, - "iterations": self.config.iterations, - "warmup": self.config.warmup, - "timestamp": self.timestamp, - }, - "ttft_ms": { - "mean": round(self.ttft_mean_ms, 3), - "min": round(self.ttft_min_ms, 3), - "max": round(self.ttft_max_ms, 3), - "p50": round(self.ttft_p50_ms, 3), - "p90": round(self.ttft_p90_ms, 3), - "p95": round(self.ttft_p95_ms, 3), - "p99": round(self.ttft_p99_ms, 3), - }, - "decode": { - "tokens_per_sec": round(self.decode_tokens_per_sec, 2), - "tpot_ms": round(self.tpot_mean_ms, 3), - }, - "total_generation_ms": {"mean": round(self.total_generation_mean_ms, 3)}, - "raw": { - "ttft_ms": [round(v, 3) for v in self.raw_ttft_ms], - "decode_tokens_per_sec": [round(v, 2) for v in self.raw_decode_tokens_per_sec], - "tpot_ms": [round(v, 3) for v in self.raw_tpot_ms], - "total_ms": [round(v, 3) for v in self.raw_total_ms], - }, - } - - # ============================================================================= # Data Generation # ============================================================================= @@ -694,8 +615,8 @@ def _pct(xs: list[float], p: float) -> float: return xs[idx] # Resolve actual device/EP from first sub-model - actual_device = "" - actual_ep: EPName | None = None + actual_device = self.config.device + actual_ep: str | None = None from ..models.winml.composite_model import WinMLCompositeModel if isinstance(model, WinMLCompositeModel) and model.sub_models: @@ -704,7 +625,14 @@ def _pct(xs: list[float], p: float) -> float: actual_ep = getattr(first_sub, "ep_name", None) return GenerationBenchmarkResult( - config=self.config, + runtime="winml-generate", + model_label=self.config.model_id, + device=actual_device, + ep=actual_ep, + prompt=prompt_text, + max_new_tokens=self.config.max_new_tokens, + warmup=self.config.warmup, + iterations=self.config.iterations, prompt_tokens=prompt_tokens, generated_tokens=int(avg_tokens), ttft_mean_ms=sum(raw_ttft) / len(raw_ttft), @@ -717,12 +645,13 @@ def _pct(xs: list[float], p: float) -> float: decode_tokens_per_sec=sum(raw_decode_tps) / len(raw_decode_tps), tpot_mean_ms=sum(raw_tpot) / len(raw_tpot), total_generation_mean_ms=sum(raw_total) / len(raw_total), - actual_device=actual_device, - actual_ep=actual_ep, raw_ttft_ms=raw_ttft, raw_decode_tokens_per_sec=raw_decode_tps, raw_tpot_ms=raw_tpot, raw_total_ms=raw_total, + extra_info={ + "task": self.config.task, + }, ) def _run_single(self) -> BenchmarkResult: @@ -1566,56 +1495,6 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: console.print() -def display_generation_report(result: GenerationBenchmarkResult, console: Console) -> None: - """Display generation benchmark results — same layout as ``display_genai_report``.""" - console.print() - console.print("[dim]Runtime:[/dim] winml-generate") - console.print(f"[dim]Model:[/dim] {result.config.model_id}") - if result.actual_device: - device_str = ( - result.actual_device - if not result.actual_ep - else f"{result.actual_device} ({result.actual_ep})" - ) - console.print(f"[dim]Device:[/dim] {device_str}") - console.print( - f"[dim]Prompt:[/dim] {result.prompt_tokens} tokens " - f"[dim]Generated:[/dim] {result.generated_tokens} tokens " - f"(max_new_tokens={result.config.max_new_tokens})" - ) - - console.print() - console.print("[bold]Time to first token (ms)[/bold]") - table = Table(show_header=True, header_style="bold cyan") - for col in ["Avg", "P50", "P90", "P95", "P99", "Min", "Max"]: - table.add_column(col, justify="right") - table.add_row( - f"{result.ttft_mean_ms:.2f}", - f"{result.ttft_p50_ms:.2f}", - f"{result.ttft_p90_ms:.2f}", - f"{result.ttft_p95_ms:.2f}", - f"{result.ttft_p99_ms:.2f}", - f"{result.ttft_min_ms:.2f}", - f"{result.ttft_max_ms:.2f}", - ) - console.print(table) - - console.print() - console.print( - f"[bold]Decode:[/bold] {result.decode_tokens_per_sec:.2f} tokens/sec | " - f"{result.tpot_mean_ms:.2f} ms/token (TPOT)" - ) - console.print( - f"[bold]Total:[/bold] {result.total_generation_mean_ms:.2f} ms avg per generation" - ) - if result.config.warmup > 0: - console.print( - f" [dim]Excluded first {result.config.warmup} warmup " - f"generation(s) from statistics[/dim]" - ) - console.print() - - def write_json_report( result: BenchmarkResult | GenerationBenchmarkResult, output_path: Path, From a7609bf6ca5dd05deee64a343128e01a036a1a01 Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Mon, 6 Jul 2026 16:46:32 +0800 Subject: [PATCH 05/12] fix: add --model-id to ONNX tests for shared resolve_model_path Tests that pass .onnx files via -m now include --model-id, matching the shared resolve_model_path validation that requires it for preprocessor/config resolution. --- src/winml/modelkit/commands/perf.py | 8 +++++--- tests/unit/commands/test_config_value_priority.py | 2 +- tests/unit/commands/test_perf_module.py | 10 +++++++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index f24f4ef9b..d32f7a9c5 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -568,7 +568,9 @@ class _GenSample: samples: list[_GenSample] = [] for i in range(total_runs): t_start = time.perf_counter() - output_ids = model.generate( + # _is_generative_composite guards this path; the model has generate() + # via GenerationMixin but mypy can't see through hasattr checks. + output_ids = model.generate( # type: ignore[union-attr] input_ids, attention_mask=inputs.get("attention_mask"), max_new_tokens=self.config.max_new_tokens, @@ -2121,7 +2123,7 @@ def perf( is_composite = isinstance(model_path, dict) is_onnx = False if not is_composite and model_path is not None: - _mi = cli_utils.classify_model_input(model_path) + _mi = cli_utils.classify_model_input(cast("str", model_path)) is_onnx = _mi.kind is cli_utils.ModelInputKind.ONNX_FILE elif not is_composite and hf_model_id is not None and model_path is None: try: @@ -2341,7 +2343,7 @@ def perf( ) profiler = tracer_cls( - onnx_for_trace, + Path(onnx_for_trace), output_dir=output_dir, level=op_tracing, ) diff --git a/tests/unit/commands/test_config_value_priority.py b/tests/unit/commands/test_config_value_priority.py index 1a7d8e1d6..7600d4b7a 100644 --- a/tests/unit/commands/test_config_value_priority.py +++ b/tests/unit/commands/test_config_value_priority.py @@ -299,7 +299,7 @@ def fake_benchmark(config): captured["instance"] = instance return instance - args = ["-m", str(model), *cli_args] + args = ["-m", str(model), "--model-id", "test/model", *cli_args] if config_path is not None: args.extend(["--config", str(config_path)]) diff --git a/tests/unit/commands/test_perf_module.py b/tests/unit/commands/test_perf_module.py index c96a5e490..5e031b8c6 100644 --- a/tests/unit/commands/test_perf_module.py +++ b/tests/unit/commands/test_perf_module.py @@ -72,7 +72,15 @@ def test_module_with_onnx_path_rejected(self, tmp_path: Path) -> None: runner = CliRunner() result = runner.invoke( main, - ["perf", "-m", str(onnx_file), "--module", "NoSuchClass"], + [ + "perf", + "-m", + str(onnx_file), + "--model-id", + "test/model", + "--module", + "NoSuchClass", + ], ) assert result.exit_code == 2, result.output assert "--module is not supported for ONNX files" in result.output From 55fc3a1649699d1da50c606be4e2ee68c851d76c Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Mon, 6 Jul 2026 16:58:25 +0800 Subject: [PATCH 06/12] fix: cast onnx_for_trace to str for Path() mypy arg-type --- src/winml/modelkit/commands/perf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index d32f7a9c5..7bf62a90b 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -2343,7 +2343,7 @@ def perf( ) profiler = tracer_cls( - Path(onnx_for_trace), + Path(cast("str", onnx_for_trace)), output_dir=output_dir, level=op_tracing, ) From a8676968795d9c8f9e80e30c160932e3096e4a8d Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Mon, 6 Jul 2026 17:05:48 +0800 Subject: [PATCH 07/12] fix: make --model-id optional for perf with ONNX files Add require_model_id_for_onnx parameter to shared resolve_model_path. eval keeps it required (needs HF config), perf passes False since it can benchmark plain ONNX without preprocessor resolution. --- src/winml/modelkit/commands/perf.py | 4 +++- src/winml/modelkit/utils/cli.py | 8 +++++++- tests/unit/commands/test_config_value_priority.py | 2 +- tests/unit/commands/test_perf_cli.py | 2 +- tests/unit/commands/test_perf_module.py | 10 +--------- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 7bf62a90b..87de0e65c 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -1837,7 +1837,9 @@ def _resolve_model_path( model_id: str | None, ) -> tuple[str | dict[str, str] | None, str | None]: """Delegate to shared ``cli_utils.resolve_model_path``.""" - return cli_utils.resolve_model_path(model=model, model_id=model_id) + return cli_utils.resolve_model_path( + model=model, model_id=model_id, require_model_id_for_onnx=False + ) @click.command("perf") diff --git a/src/winml/modelkit/utils/cli.py b/src/winml/modelkit/utils/cli.py index caead055c..7674232c2 100644 --- a/src/winml/modelkit/utils/cli.py +++ b/src/winml/modelkit/utils/cli.py @@ -1196,6 +1196,7 @@ def resolve_model_path( *, model: tuple[str, ...], model_id: str | None, + require_model_id_for_onnx: bool = True, ) -> tuple[str | dict[str, str] | None, str | None]: """Turn repeated ``-m`` values + ``--model-id`` into ``(model_path, model_id)``. @@ -1203,6 +1204,11 @@ def resolve_model_path( syntax. ``-m role=path`` pairs require ``--model-id`` for preprocessor and config resolution. + Args: + require_model_id_for_onnx: When *True* (default, used by ``eval``), + a bare ONNX file requires ``--model-id``. ``perf`` passes + *False* because it can benchmark without HF config. + Returns: A 2-tuple ``(model_path, hf_model_id)`` where *model_path* is ``None`` (HF id only), a single ONNX path string, or a @@ -1264,7 +1270,7 @@ def resolve_model_path( # Preserve param-scoped BadParameter contract (tests + hint). raise click.BadParameter(str(e), param_hint="-m/--model") from e if _mi.kind is ModelInputKind.ONNX_FILE: - if model_id is None: + if require_model_id_for_onnx and model_id is None: raise click.UsageError( "When using an ONNX file, --model-id is required " "for preprocessor and config resolution." diff --git a/tests/unit/commands/test_config_value_priority.py b/tests/unit/commands/test_config_value_priority.py index 7600d4b7a..1a7d8e1d6 100644 --- a/tests/unit/commands/test_config_value_priority.py +++ b/tests/unit/commands/test_config_value_priority.py @@ -299,7 +299,7 @@ def fake_benchmark(config): captured["instance"] = instance return instance - args = ["-m", str(model), "--model-id", "test/model", *cli_args] + args = ["-m", str(model), *cli_args] if config_path is not None: args.extend(["--config", str(config_path)]) diff --git a/tests/unit/commands/test_perf_cli.py b/tests/unit/commands/test_perf_cli.py index 5648d89e9..1f817ddd5 100644 --- a/tests/unit/commands/test_perf_cli.py +++ b/tests/unit/commands/test_perf_cli.py @@ -584,7 +584,7 @@ def test_cli_ep_options_invalid_format_rejected( result = runner.invoke( perf, - ["-m", str(onnx_file), "--model-id", "test/model", "--ep-options", "no_equals_sign"], + ["-m", str(onnx_file), "--ep-options", "no_equals_sign"], obj={}, ) diff --git a/tests/unit/commands/test_perf_module.py b/tests/unit/commands/test_perf_module.py index 5e031b8c6..c96a5e490 100644 --- a/tests/unit/commands/test_perf_module.py +++ b/tests/unit/commands/test_perf_module.py @@ -72,15 +72,7 @@ def test_module_with_onnx_path_rejected(self, tmp_path: Path) -> None: runner = CliRunner() result = runner.invoke( main, - [ - "perf", - "-m", - str(onnx_file), - "--model-id", - "test/model", - "--module", - "NoSuchClass", - ], + ["perf", "-m", str(onnx_file), "--module", "NoSuchClass"], ) assert result.exit_code == 2, result.output assert "--module is not supported for ONNX files" in result.output From d5a0ccea0225d73b4e768ff5651f545977625deb Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Mon, 6 Jul 2026 18:20:26 +0800 Subject: [PATCH 08/12] feat(perf): align composite generation with genai defaults and logging - Default iterations=10, warmup=2 for composite models (like genai) unless explicitly set via CLI - Decode and log model response: first iteration at INFO, rest at DEBUG (matching genai _time_one_generation pattern) --- src/winml/modelkit/commands/perf.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 87de0e65c..ff230f50c 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -581,6 +581,20 @@ class _GenSample: total_ms = (t_end - t_start) * 1000.0 n_generated = output_ids.shape[1] - prompt_tokens + # Decode and log model response (like genai: first at INFO, rest at DEBUG) + response_text = tokenizer.decode( + output_ids[0, prompt_tokens:], skip_special_tokens=True + ) + generation_count = i + 1 + if generation_count == 1: + logger.info("Model response (iteration 1): %s", response_text) + else: + logger.debug( + "Model response (iteration %d): %s", + generation_count, + response_text, + ) + # TTFT approximation: total_ms * (1 token / total tokens generated) # is a rough proxy. A better split requires hooking into forward(), # but this gives directionally correct numbers for comparison. @@ -2204,13 +2218,23 @@ def perf( # Refuse to clobber an existing report unless the user opted in. cli_utils.guard_output(output, overwrite) + # Composite generation is far costlier than one session.run(): default to + # fewer iterations/warmup (like genai) unless the user set them explicitly. + effective_iterations = iterations + effective_warmup = warmup + if is_composite: + if not cli_utils.is_cli_provided(ctx, "iterations"): + effective_iterations = 10 + if not cli_utils.is_cli_provided(ctx, "warmup"): + effective_warmup = 2 + config = BenchmarkConfig( model_id=hf_model, task=task, device=device.lower(), precision=precision.lower(), - iterations=iterations, - warmup=warmup, + iterations=effective_iterations, + warmup=effective_warmup, batch_size=batch_size, output_path=output, no_quantize=not quant, From 75fddeb32c0d1b71820e082493d8145c81aea2b4 Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Tue, 7 Jul 2026 10:39:34 +0800 Subject: [PATCH 09/12] fix: restore 'directory' assertion in test_onnx_file_rejected --- tests/unit/commands/test_perf_cli.py | 35 +++----------------------- tests/unit/commands/test_perf_genai.py | 1 + 2 files changed, 4 insertions(+), 32 deletions(-) diff --git a/tests/unit/commands/test_perf_cli.py b/tests/unit/commands/test_perf_cli.py index 1f817ddd5..e09739012 100644 --- a/tests/unit/commands/test_perf_cli.py +++ b/tests/unit/commands/test_perf_cli.py @@ -318,14 +318,7 @@ def test_cli_onnx_routes_through_perf_benchmark( ): result = runner.invoke( perf, - [ - "-m", - str(onnx_file), - "--model-id", - "test/model", - "-o", - str(tmp_path / "out.json"), - ], + ["-m", str(onnx_file), "-o", str(tmp_path / "out.json")], obj={}, ) @@ -368,8 +361,6 @@ def capture_config(config: BenchmarkConfig) -> MagicMock: [ "-m", str(onnx_file), - "--model-id", - "test/model", "--shape-config", str(shape_cfg_file), "-o", @@ -407,8 +398,6 @@ def capture_config(_config: BenchmarkConfig) -> MagicMock: [ "-m", str(onnx_file), - "--model-id", - "test/model", "--no-quant", "--no-optimize", "-o", @@ -444,14 +433,7 @@ def capture_config(_config: BenchmarkConfig) -> MagicMock: ): result = runner.invoke( perf, - [ - "-m", - str(onnx_file), - "--model-id", - "test/model", - "-o", - str(tmp_path / "out.json"), - ], + ["-m", str(onnx_file), "-o", str(tmp_path / "out.json")], obj={}, ) @@ -557,8 +539,6 @@ def capture_config(config: BenchmarkConfig) -> MagicMock: [ "-m", str(onnx_file), - "--model-id", - "test/model", "--ep-options", "htp_performance_mode=burst", "--ep-options", @@ -682,16 +662,7 @@ def test_cli_unavailable_device_ep_surfaces_error( ): result = runner.invoke( perf, - [ - "-m", - str(onnx_file), - "--model-id", - "test/model", - "--device", - "npu", - "-o", - str(tmp_path / "out.json"), - ], + ["-m", str(onnx_file), "--device", "npu", "-o", str(tmp_path / "out.json")], obj={}, ) diff --git a/tests/unit/commands/test_perf_genai.py b/tests/unit/commands/test_perf_genai.py index 957cbc7c6..8838a99b4 100644 --- a/tests/unit/commands/test_perf_genai.py +++ b/tests/unit/commands/test_perf_genai.py @@ -588,6 +588,7 @@ def test_onnx_file_rejected(self, runner: CliRunner, tmp_path: Path, capture_run onnx.write_bytes(b"fake") result = runner.invoke(perf, ["-m", str(onnx), "--runtime", "winml-genai"]) assert result.exit_code != 0 + assert "directory" in result.output.lower() assert "config" not in capture_run def test_missing_directory_rejected( From 9cb2ab7da0824195cb400a49575f206a803832da Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Tue, 7 Jul 2026 10:40:44 +0800 Subject: [PATCH 10/12] refactor(perf): remove console progress log from generation loop Align with genai which only logs model responses via logger, not per-iteration console progress. --- src/winml/modelkit/commands/perf.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index ff230f50c..5a2e06a0a 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -603,12 +603,6 @@ class _GenSample: samples.append(_GenSample(ttft_ms=ttft_ms, total_ms=total_ms, n_tokens=n_generated)) - phase = "warmup" if i < self.config.warmup else "timed" - if (i + 1) % max(1, total_runs // 5) == 0 or i == total_runs - 1: - console.print( - f" [{phase}] {i + 1}/{total_runs}: {total_ms:.1f} ms, {n_generated} tokens" - ) - # Aggregate (exclude warmup) timed = samples[self.config.warmup :] if not timed: From 400927cd08d9b97bf6fd5ec5179d3de2a2341fda Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Tue, 7 Jul 2026 10:44:21 +0800 Subject: [PATCH 11/12] feat(perf): add per-iteration timing log in generation loop Log input_tokens, generated_tokens, ttft, tpot, decode tok/s, and total time per generation iteration, matching genai_session's generate_timed log format. --- src/winml/modelkit/commands/perf.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 5a2e06a0a..7f8d8638e 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -600,6 +600,20 @@ class _GenSample: # but this gives directionally correct numbers for comparison. # For the first generated token, prefill dominates. ttft_ms = total_ms / max(n_generated, 1) + total_s = t_end - t_start + tpot_s = total_s / max(n_generated, 1) + decode_tps = n_generated / total_s if total_s > 0 else 0 + + logger.info( + "generate_timed: input_tokens=%d generated_tokens=%d " + "ttft=%.3fs tpot=%.3fs decode=%.1f tok/s total=%.3fs", + prompt_tokens, + n_generated, + ttft_ms / 1000.0, + tpot_s, + decode_tps, + total_s, + ) samples.append(_GenSample(ttft_ms=ttft_ms, total_ms=total_ms, n_tokens=n_generated)) From 64a8abe2b1a03359d1f3ffefde772a36f2383fa7 Mon Sep 17 00:00:00 2001 From: Hualiang Xie Date: Tue, 7 Jul 2026 11:07:39 +0800 Subject: [PATCH 12/12] feat(perf): auto-infer task for composite models from HF config When --task is not provided for composite models, infer it from the HF model config via resolve_task() (same as eval). Users no longer need to pass --task text-generation for well-known models like Qwen. --- src/winml/modelkit/commands/perf.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 7f8d8638e..8d515fe37 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -825,6 +825,16 @@ def _load_model(self) -> None: hf_model_id = self.config.hf_model_id or model_id hf_config = AutoConfig.from_pretrained(hf_model_id) + + # Auto-infer task from HF config when not explicitly provided, + # so users don't need --task for well-known models (e.g. Qwen). + if common_kwargs.get("task") is None: + from ..loader.resolution import resolve_task + + inferred = resolve_task(hf_config).task + common_kwargs["task"] = inferred + logger.info("Inferred task from model config: %s", inferred) + self._model = WinMLAutoModel.from_onnx( onnx_path=self.config.model_path, hf_config=hf_config,