From 0b65be49542caf93b06a899446c173101cfb6a0d Mon Sep 17 00:00:00 2001 From: Shiyi Zheng Date: Sun, 12 Jul 2026 17:42:51 +0800 Subject: [PATCH 1/4] Add LayoutLM v1 document-question-answering support (impira/layoutlm-invoices) Register a document-question-answering OnnxConfig for layoutlm (Optimum has none for v1) that emits extractive-QA outputs (start_logits/end_logits) and applies the RoBERTa-style position-offset adjustment used by this checkpoint. Add an fp16 recipe with corrected sequence length (512) and embedding-safe input value ranges (token_type_ids pinned to 0 for type_vocab_size=1). Validated fp16 build + CPU inference (P50 280.6ms). --- examples/recipes/README.md | 3 +- ...cument-question-answering_fp16_config.json | 99 ++++++++++++++++ src/winml/modelkit/models/hf/__init__.py | 3 + src/winml/modelkit/models/hf/layoutlm.py | 111 ++++++++++++++++++ 4 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 examples/recipes/impira_layoutlm-invoices/document-question-answering_fp16_config.json create mode 100644 src/winml/modelkit/models/hf/layoutlm.py diff --git a/examples/recipes/README.md b/examples/recipes/README.md index 1077e4f74..87286109b 100644 --- a/examples/recipes/README.md +++ b/examples/recipes/README.md @@ -14,7 +14,7 @@ Each *(model, task)* includes: ## Models -Total: **75** (model, task) tuples that pass fp16 eval on all 10 (EP, device) buckets. +Total: **76** (model, task) tuples that pass fp16 eval on all 10 (EP, device) buckets. | Model | Task | |---|---| @@ -61,6 +61,7 @@ Total: **75** (model, task) tuples that pass fp16 eval on all 10 (EP, device) bu | google-bert/bert-large-uncased-whole-word-masking-finetuned-squad | question-answering | | google/vit-base-patch16-224 | image-classification | | google/vit-base-patch16-224-in21k | image-feature-extraction | +| impira/layoutlm-invoices | document-question-answering | | joeddav/xlm-roberta-large-xnli | zero-shot-classification | | laion/CLIP-ViT-B-32-laion2B-s34B-b79K | feature-extraction | | mattmdjaga/segformer_b2_clothes | image-segmentation | diff --git a/examples/recipes/impira_layoutlm-invoices/document-question-answering_fp16_config.json b/examples/recipes/impira_layoutlm-invoices/document-question-answering_fp16_config.json new file mode 100644 index 000000000..86d0f0330 --- /dev/null +++ b/examples/recipes/impira_layoutlm-invoices/document-question-answering_fp16_config.json @@ -0,0 +1,99 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "input_ids", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 50265 + ] + }, + { + "name": "bbox", + "dtype": "int32", + "shape": [ + 1, + 512, + 4 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "attention_mask", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 2 + ] + }, + { + "name": "token_type_ids", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 1 + ] + } + ], + "output_tensors": [ + { + "name": "start_logits" + }, + { + "name": "end_logits" + } + ] + }, + "optim": {}, + "quant": { + "mode": "fp16", + "samples": 10, + "calibration_method": "minmax", + "weight_type": "uint8", + "activation_type": "uint8", + "per_channel": false, + "symmetric": false, + "weight_symmetric": null, + "activation_symmetric": null, + "save_calibration": false, + "distribution": "uniform", + "seed": null, + "calibration_load_path": null, + "calibration_save_path": null, + "op_types_to_quantize": null, + "nodes_to_exclude": null, + "fp16_keep_io_types": true, + "fp16_op_block_list": null + }, + "compile": null, + "loader": { + "task": "document-question-answering", + "model_class": "AutoModelForDocumentQuestionAnswering", + "model_type": "layoutlm" + } +} diff --git a/src/winml/modelkit/models/hf/__init__.py b/src/winml/modelkit/models/hf/__init__.py index 094714494..46107996e 100644 --- a/src/winml/modelkit/models/hf/__init__.py +++ b/src/winml/modelkit/models/hf/__init__.py @@ -44,6 +44,9 @@ from .depth_anything import DepthAnythingIOConfig as _DepthAnythingIOConfig # triggers registration from .depth_pro import DepthProIOConfig as _DepthProIOConfig # triggers registration from .detr import DETR_CONFIG +from .layoutlm import ( + LayoutLMDocumentQAOnnxConfig as _LayoutLMDocumentQAOnnxConfig, # triggers registration +) from .marian import MARIAN_CONFIG from .marian import MODEL_CLASS_MAPPING as _MARIAN_CLASS_MAPPING from .marian import MarianDecoderIOConfig as _MarianDecoderIOConfig # triggers registration diff --git a/src/winml/modelkit/models/hf/layoutlm.py b/src/winml/modelkit/models/hf/layoutlm.py new file mode 100644 index 000000000..3f43ca1cd --- /dev/null +++ b/src/winml/modelkit/models/hf/layoutlm.py @@ -0,0 +1,111 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- + +"""LayoutLM (v1) document-question-answering ONNX export config. + +Optimum's TasksManager registers ``layoutlm`` only for feature-extraction, +fill-mask, text-classification and token-classification -- it has no +document-question-answering config. However, transformers' +``LayoutLMForQuestionAnswering`` (the head behind +``AutoModelForDocumentQuestionAnswering`` for layoutlm v1, e.g. +``impira/layoutlm-invoices``) is an *extractive* QA head that returns +``start_logits`` / ``end_logits``. + +This module registers a ``document-question-answering`` OnnxConfig for +layoutlm that reuses ``LayoutLMOnnxConfig``'s inputs (``input_ids``, ``bbox``, +``attention_mask``, ``token_type_ids``) and overrides the outputs to the +extractive-QA pair. Optimum's default document-question-answering output is a +single ``logits`` tensor, which does not match this head; the override emits +the correct ``start_logits`` / ``end_logits`` pair. + +This module provides: +- LayoutLMDocumentQAOnnxConfig: ONNX export config for layoutlm document-QA +""" + +from __future__ import annotations + +import logging +from typing import Any + +from optimum.exporters.onnx.model_configs import LayoutLMOnnxConfig + +from ...export import register_onnx_overwrite + + +logger = logging.getLogger(__name__) + + +def _adjust_position_embeddings(config: Any) -> None: + """Adjust max_position_embeddings for RoBERTa-style position offset. + + Some LayoutLM checkpoints (e.g. ``impira/layoutlm-invoices``) are built on a + RoBERTa vocabulary/tokenizer and inherit its position-embedding convention:: + + max_position_embeddings = usable_length + pad_token_id + 1 + + e.g. 514 = 512 + 1 + 1 (pad_token_id=1). LayoutLM's ``sequence_length`` is + derived from ``max_position_embeddings``; using the raw value causes a + position index out-of-bounds ("index out of range in self") during ONNX + export tracing. This adjusts ``config.max_position_embeddings`` in-place to + the usable length. A sentinel prevents double-adjustment on config reuse. + """ + if getattr(config, "_position_offset_applied", False): + return + + if not hasattr(config, "max_position_embeddings"): + return + + pad_token_id = getattr(config, "pad_token_id", 0) or 0 + if pad_token_id > 0: + original = config.max_position_embeddings + adjusted = original - pad_token_id - 1 + if adjusted <= 0: + raise ValueError( + f"Position offset adjustment would produce non-positive " + f"max_position_embeddings={adjusted} " + f"(original={original}, pad_token_id={pad_token_id})" + ) + config.max_position_embeddings = adjusted + config._position_offset_applied = True + logger.debug( + "Adjusted max_position_embeddings: %d -> %d (pad_token_id=%d)", + original, + adjusted, + pad_token_id, + ) + + +@register_onnx_overwrite("layoutlm", "document-question-answering", library_name="transformers") +class LayoutLMDocumentQAOnnxConfig(LayoutLMOnnxConfig): # type: ignore[misc] # optimum base is untyped + """LayoutLM document-question-answering (extractive) OnnxConfig. + + Inputs (inherited from LayoutLMOnnxConfig): + - input_ids: {0: "batch_size", 1: "sequence_length"} + - bbox: {0: "batch_size", 1: "sequence_length"} + - attention_mask: {0: "batch_size", 1: "sequence_length"} + - token_type_ids: {0: "batch_size", 1: "sequence_length"} + + Outputs (extractive QA -- overrides Optimum's single-``logits`` default): + - start_logits: {0: "batch_size", 1: "sequence_length"} + - end_logits: {0: "batch_size", 1: "sequence_length"} + + RoBERTa-vocab LayoutLM checkpoints set ``max_position_embeddings`` to the + usable length plus a pad-token offset; the constructor adjusts it back to + the usable length so ONNX export tracing does not hit a position-index + out-of-bounds. LayoutLM's own dummy-input generators (which supply ``bbox``) + are preserved. + """ + + def __init__(self, config: Any, task: str, **kwargs: Any) -> None: + _adjust_position_embeddings(config) + super().__init__(config, task, **kwargs) + + @property + def outputs(self) -> dict[str, dict[int, str]]: + """Extractive-QA outputs: ``start_logits`` and ``end_logits``.""" + return { + "start_logits": {0: "batch_size", 1: "sequence_length"}, + "end_logits": {0: "batch_size", 1: "sequence_length"}, + } From 175b8e76370752409c41c8845e9a0d2c488e06a5 Mon Sep 17 00:00:00 2001 From: Shiyi Zheng Date: Mon, 13 Jul 2026 13:47:24 +0800 Subject: [PATCH 2/4] recipe(layoutlm-invoices): nest under cpu/cpu/ per tested-EP layout (_meta-058) --- .../{ => cpu/cpu}/document-question-answering_fp16_config.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/recipes/impira_layoutlm-invoices/{ => cpu/cpu}/document-question-answering_fp16_config.json (100%) diff --git a/examples/recipes/impira_layoutlm-invoices/document-question-answering_fp16_config.json b/examples/recipes/impira_layoutlm-invoices/cpu/cpu/document-question-answering_fp16_config.json similarity index 100% rename from examples/recipes/impira_layoutlm-invoices/document-question-answering_fp16_config.json rename to examples/recipes/impira_layoutlm-invoices/cpu/cpu/document-question-answering_fp16_config.json From 5ff1df85344d256d0b778623ce673c8b94d404e6 Mon Sep 17 00:00:00 2001 From: "Shiyi Zheng (from Dev Box)" Date: Thu, 16 Jul 2026 10:41:42 +0800 Subject: [PATCH 3/4] test(layoutlm): cover document QA export config --- examples/recipes/README.md | 3 +- .../unit/export/test_layoutlm_onnx_config.py | 78 +++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 tests/unit/export/test_layoutlm_onnx_config.py diff --git a/examples/recipes/README.md b/examples/recipes/README.md index 87286109b..1077e4f74 100644 --- a/examples/recipes/README.md +++ b/examples/recipes/README.md @@ -14,7 +14,7 @@ Each *(model, task)* includes: ## Models -Total: **76** (model, task) tuples that pass fp16 eval on all 10 (EP, device) buckets. +Total: **75** (model, task) tuples that pass fp16 eval on all 10 (EP, device) buckets. | Model | Task | |---|---| @@ -61,7 +61,6 @@ Total: **76** (model, task) tuples that pass fp16 eval on all 10 (EP, device) bu | google-bert/bert-large-uncased-whole-word-masking-finetuned-squad | question-answering | | google/vit-base-patch16-224 | image-classification | | google/vit-base-patch16-224-in21k | image-feature-extraction | -| impira/layoutlm-invoices | document-question-answering | | joeddav/xlm-roberta-large-xnli | zero-shot-classification | | laion/CLIP-ViT-B-32-laion2B-s34B-b79K | feature-extraction | | mattmdjaga/segformer_b2_clothes | image-segmentation | diff --git a/tests/unit/export/test_layoutlm_onnx_config.py b/tests/unit/export/test_layoutlm_onnx_config.py new file mode 100644 index 000000000..17adbcb1b --- /dev/null +++ b/tests/unit/export/test_layoutlm_onnx_config.py @@ -0,0 +1,78 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- +"""Tests for LayoutLM v1 document-question-answering export registration.""" + +from __future__ import annotations + +import pytest +from optimum.exporters.tasks import TasksManager + +from winml.modelkit.export import resolve_io_specs +from winml.modelkit.models.hf.layoutlm import ( + LayoutLMDocumentQAOnnxConfig, + _adjust_position_embeddings, +) + + +@pytest.fixture +def layoutlm_config(): + """Return a small LayoutLM config with RoBERTa-style position offset.""" + from transformers import LayoutLMConfig + + return LayoutLMConfig( + vocab_size=100, + hidden_size=32, + num_hidden_layers=1, + num_attention_heads=4, + intermediate_size=64, + max_position_embeddings=514, + pad_token_id=1, + type_vocab_size=1, + ) + + +def test_document_qa_config_registered() -> None: + """The document-QA task resolves to the WinML LayoutLM override.""" + constructor = TasksManager.get_exporter_config_constructor( + exporter="onnx", + model_type="layoutlm", + task="document-question-answering", + library_name="transformers", + ) + assert constructor.func is LayoutLMDocumentQAOnnxConfig + + +def test_document_qa_io_specs(layoutlm_config) -> None: + """LayoutLM document-QA keeps all four inputs and emits two logits.""" + specs = resolve_io_specs( + "layoutlm", + "document-question-answering", + layoutlm_config, + ) + assert specs["input_names"] == [ + "input_ids", + "bbox", + "attention_mask", + "token_type_ids", + ] + assert specs["output_names"] == ["start_logits", "end_logits"] + + +def test_position_offset_is_adjusted_once(layoutlm_config) -> None: + """RoBERTa-style 514 positions become a usable sequence length of 512.""" + _adjust_position_embeddings(layoutlm_config) + assert layoutlm_config.max_position_embeddings == 512 + + _adjust_position_embeddings(layoutlm_config) + assert layoutlm_config.max_position_embeddings == 512 + + +def test_position_offset_rejects_non_positive_length(layoutlm_config) -> None: + """Invalid position metadata fails explicitly instead of tracing bad indices.""" + layoutlm_config.max_position_embeddings = 2 + layoutlm_config.pad_token_id = 1 + + with pytest.raises(ValueError, match="non-positive"): + _adjust_position_embeddings(layoutlm_config) From c3bb5691d50341172abea710799df733037851bd Mon Sep 17 00:00:00 2001 From: "Shiyi Zheng (from Dev Box)" Date: Thu, 16 Jul 2026 10:43:58 +0800 Subject: [PATCH 4/4] recipe(layoutlm): add verified CPU fp32 tuple --- ...cument-question-answering_fp32_config.json | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 examples/recipes/impira_layoutlm-invoices/cpu/cpu/document-question-answering_fp32_config.json diff --git a/examples/recipes/impira_layoutlm-invoices/cpu/cpu/document-question-answering_fp32_config.json b/examples/recipes/impira_layoutlm-invoices/cpu/cpu/document-question-answering_fp32_config.json new file mode 100644 index 000000000..f85339356 --- /dev/null +++ b/examples/recipes/impira_layoutlm-invoices/cpu/cpu/document-question-answering_fp32_config.json @@ -0,0 +1,82 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "input_ids", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 50265 + ] + }, + { + "name": "bbox", + "dtype": "int32", + "shape": [ + 1, + 512, + 4 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "attention_mask", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 2 + ] + }, + { + "name": "token_type_ids", + "dtype": "int32", + "shape": [ + 1, + 512 + ], + "value_range": [ + 0, + 1 + ] + } + ], + "output_tensors": [ + { + "name": "start_logits" + }, + { + "name": "end_logits" + } + ] + }, + "optim": { + + }, + "quant": null, + "compile": null, + "loader": { + "task": "document-question-answering", + "model_class": "AutoModelForDocumentQuestionAnswering", + "model_type": "layoutlm" + } +}