From c75cc0cac682153dc530f0e381e19d0d64d99fff Mon Sep 17 00:00:00 2001 From: Dhruv Date: Tue, 7 Jul 2026 16:02:16 +0530 Subject: [PATCH 1/2] Quantizing error for Qwen3.6 35B A3B --- .../units/default_disabled_quantizers.yaml | 13 +++++++ ..._nvfp4-fp8_attn-kv_fp8_cast.quant_cfg.yaml | 29 +++++++++------ ...p4_mse-fp8_attn-kv_fp8_cast.quant_cfg.yaml | 29 +++++++++------ tests/unit/recipe/test_loader.py | 35 +++++++++++++++++++ tests/unit/recipe/test_presets.py | 19 ++++++++++ 5 files changed, 103 insertions(+), 22 deletions(-) diff --git a/modelopt_recipes/configs/ptq/units/default_disabled_quantizers.yaml b/modelopt_recipes/configs/ptq/units/default_disabled_quantizers.yaml index 5e48dc73b7e..e2bb7e88744 100644 --- a/modelopt_recipes/configs/ptq/units/default_disabled_quantizers.yaml +++ b/modelopt_recipes/configs/ptq/units/default_disabled_quantizers.yaml @@ -24,6 +24,19 @@ enable: false - quantizer_name: '*linear_attn.in_proj_b*' enable: false + # Hybrid linear-attention (GatedDeltaNet, Qwen3.5/3.6) packed projections. + # TRT-LLM's Qwen3.5 weight mapper repacks in_proj_qkv/in_proj_z into the fused + # in_proj_qkvz by splitting every checkpoint tensor along dim 0; the scalar + # per-tensor scales of quantized modules (input_scale, weight_scale_2) are + # 0-dim and crash the split with "IndexError: tuple index out of range" + # (GitHub issue #1933). The `in_proj_qkv` pattern also covers the fused + # `in_proj_qkvz` module name used by older transformers releases. A recipe + # that intentionally quantizes these (e.g. the curated Qwen3.5 FP8 recipes) + # must re-enable them after importing this unit. + - quantizer_name: '*linear_attn.in_proj_qkv*' + enable: false + - quantizer_name: '*linear_attn.in_proj_z*' + enable: false - quantizer_name: '*lm_head*' enable: false - quantizer_name: '*mixer.conv1d*' diff --git a/modelopt_recipes/huggingface/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.quant_cfg.yaml b/modelopt_recipes/huggingface/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.quant_cfg.yaml index 6f8749e6a0b..ed4c8dcfeac 100644 --- a/modelopt_recipes/huggingface/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.quant_cfg.yaml +++ b/modelopt_recipes/huggingface/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.quant_cfg.yaml @@ -49,17 +49,10 @@ imports: - quantizer_name: '*self_attn*input_quantizer' cfg: {$import: fp8} - # FP8 large linear-attention projections. in_proj_a and in_proj_b are - # re-disabled explicitly below; conv1d stays disabled via base_disable_all - # (no rule re-enables it). - - quantizer_name: '*linear_attn.in_proj_qkv*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*linear_attn.in_proj_qkv*input_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*linear_attn.in_proj_z*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*linear_attn.in_proj_z*input_quantizer' - cfg: {$import: fp8} + # FP8 linear-attention out_proj. The large in_proj_qkv/in_proj_z projections + # are re-enabled after default_disabled_quantizers below; in_proj_a and + # in_proj_b are re-disabled explicitly below; conv1d stays disabled via + # base_disable_all (no rule re-enables it). - quantizer_name: '*linear_attn.out_proj*weight_quantizer' cfg: {$import: fp8} - quantizer_name: '*linear_attn.out_proj*input_quantizer' @@ -87,6 +80,20 @@ imports: - quantizer_name: '*mtp*' enable: false + # Re-enable FP8 on the large linear-attention projections per the reference + # recipe. Must come after default_disabled_quantizers, which disables + # `*linear_attn.in_proj_qkv*` / `*linear_attn.in_proj_z*` by default because + # TRT-LLM's Qwen3.5 weight mapper cannot split their scalar per-tensor scales + # (GitHub issue #1933). + - quantizer_name: '*linear_attn.in_proj_qkv*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*linear_attn.in_proj_qkv*input_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*linear_attn.in_proj_z*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*linear_attn.in_proj_z*input_quantizer' + cfg: {$import: fp8} + # Re-enable NVFP4 on lm_head weights. Must come after # default_disabled_quantizers, which disables `*lm_head*`. - quantizer_name: '*lm_head*weight_quantizer' diff --git a/modelopt_recipes/huggingface/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.quant_cfg.yaml b/modelopt_recipes/huggingface/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.quant_cfg.yaml index b04ca9f3451..05c15405957 100644 --- a/modelopt_recipes/huggingface/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.quant_cfg.yaml +++ b/modelopt_recipes/huggingface/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.quant_cfg.yaml @@ -56,17 +56,10 @@ imports: - quantizer_name: '*self_attn*input_quantizer' cfg: {$import: fp8} - # FP8 large linear-attention projections. in_proj_a and in_proj_b are - # re-disabled explicitly below; conv1d stays disabled via base_disable_all - # (no rule re-enables it). - - quantizer_name: '*linear_attn.in_proj_qkv*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*linear_attn.in_proj_qkv*input_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*linear_attn.in_proj_z*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*linear_attn.in_proj_z*input_quantizer' - cfg: {$import: fp8} + # FP8 linear-attention out_proj. The large in_proj_qkv/in_proj_z projections + # are re-enabled after default_disabled_quantizers below; in_proj_a and + # in_proj_b are re-disabled explicitly below; conv1d stays disabled via + # base_disable_all (no rule re-enables it). - quantizer_name: '*linear_attn.out_proj*weight_quantizer' cfg: {$import: fp8} - quantizer_name: '*linear_attn.out_proj*input_quantizer' @@ -94,6 +87,20 @@ imports: - quantizer_name: '*mtp*' enable: false + # Re-enable FP8 on the large linear-attention projections per the reference + # recipe. Must come after default_disabled_quantizers, which disables + # `*linear_attn.in_proj_qkv*` / `*linear_attn.in_proj_z*` by default because + # TRT-LLM's Qwen3.5 weight mapper cannot split their scalar per-tensor scales + # (GitHub issue #1933). + - quantizer_name: '*linear_attn.in_proj_qkv*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*linear_attn.in_proj_qkv*input_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*linear_attn.in_proj_z*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*linear_attn.in_proj_z*input_quantizer' + cfg: {$import: fp8} + # Re-enable NVFP4 on lm_head weights (static MSE scales). Must come after # default_disabled_quantizers, which disables `*lm_head*`. - quantizer_name: '*lm_head*weight_quantizer' diff --git a/tests/unit/recipe/test_loader.py b/tests/unit/recipe/test_loader.py index 3aaacaa3e0e..5284f17597a 100644 --- a/tests/unit/recipe/test_loader.py +++ b/tests/unit/recipe/test_loader.py @@ -201,6 +201,41 @@ def test_nvfp4_weight_only_recipe_disables_vllm_marlin_incompatible_projections( } <= disabled_quantizers +def _effective_enable(quant_cfg: list[dict], quantizer_name: str) -> bool: + """Resolve the final enable state of ``quantizer_name`` (entries apply in order).""" + enabled = False + for entry in quant_cfg: + # parent_class entries constrain by module class (BatchNorm, Embedding, + # ...); they never target the Linear quantizers probed here. + if entry.get("parent_class"): + continue + if fnmatch(quantizer_name, entry["quantizer_name"]): + enabled = entry.get("enable", True) is not False + return enabled + + +@pytest.mark.parametrize( + "recipe_path", + [ + "huggingface/qwen3_5_moe/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast", + "huggingface/qwen3_5_moe/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast", + ], +) +def test_qwen3_5_recipes_keep_fp8_on_packed_linear_attn_projections(recipe_path): + # default_disabled_quantizers now disables linear_attn.in_proj_qkv/in_proj_z + # (TRT-LLM cannot split their scalar per-tensor scales, GitHub issue #1933); + # the curated Qwen3.5/3.6 recipes intentionally re-enable them with FP8 + # after that import. Guard the entry ordering. + quant_cfg = load_recipe(recipe_path).quantize.model_dump()["quant_cfg"] + + for module in ("in_proj_qkv", "in_proj_z"): + name = f"model.layers.0.linear_attn.{module}.weight_quantizer" + assert _effective_enable(quant_cfg, name), f"{name} should stay FP8-quantized" + for module in ("in_proj_a", "in_proj_b", "conv1d"): + name = f"model.layers.0.linear_attn.{module}.weight_quantizer" + assert not _effective_enable(quant_cfg, name), f"{name} should stay unquantized" + + def test_nvfp4_mlp_only_novit_recipe_disables_vision_quantizers(): recipe = load_recipe("general/ptq/nvfp4_mlp_only-novit-kv_fp8") disabled_quantizers = { diff --git a/tests/unit/recipe/test_presets.py b/tests/unit/recipe/test_presets.py index 64d011f8ef0..93a6c77cff6 100644 --- a/tests/unit/recipe/test_presets.py +++ b/tests/unit/recipe/test_presets.py @@ -83,6 +83,25 @@ def test_w4a16_nvfp4_preset_disables_vllm_marlin_incompatible_projections(): } <= disabled_quantizers +@pytest.mark.parametrize("preset", ["nvfp4", "fp8"]) +def test_generic_presets_disable_packed_linear_attn_projections(preset): + # TRT-LLM's Qwen3.5/3.6 weight mapper repacks linear_attn.in_proj_qkv / + # in_proj_z by splitting every checkpoint tensor along dim 0; scalar + # per-tensor scales (input_scale, weight_scale_2) crash it with + # "IndexError: tuple index out of range" (GitHub issue #1933). The generic + # presets must keep these packed projections unquantized. + disabled_quantizers = { + entry["quantizer_name"] + for entry in presets.QUANT_CFG_CHOICES[preset]["quant_cfg"] + if entry.get("enable") is False + } + + assert { + "*linear_attn.in_proj_qkv*", + "*linear_attn.in_proj_z*", + } <= disabled_quantizers + + def test_load_quant_cfg_choices_rejects_stale_alias(): with pytest.raises(ValueError, match="does-not-exist"): presets.load_quant_cfg_choices( From 81a4d897b8ea22609f1b1f4fe0ef170fcb656631 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Thu, 9 Jul 2026 15:16:29 +0530 Subject: [PATCH 2/2] Address review feedback --- .../hf_ptq/scripts/huggingface_example.sh | 3 +- .../torch/quantization/test_quantize_cpu.py | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/examples/hf_ptq/scripts/huggingface_example.sh b/examples/hf_ptq/scripts/huggingface_example.sh index 84057e468c9..5a656999bcd 100755 --- a/examples/hf_ptq/scripts/huggingface_example.sh +++ b/examples/hf_ptq/scripts/huggingface_example.sh @@ -185,7 +185,8 @@ if [[ $TASKS =~ "quant" ]] || [[ ! -d "$SAVE_PATH" ]] || [[ ! $(ls -A $SAVE_PATH $PTQ_ARGS \ $AWQ_ARGS else - echo "Quantized model config $MODEL_CONFIG exists, skipping the quantization stage" + echo "Quantized model config $MODEL_CONFIG exists, skipping the quantization stage." + echo "The previously exported checkpoint in $SAVE_PATH is reused as-is; delete that directory to re-quantize (required to pick up ModelOpt or quantization config changes)." fi # for enc-dec model, users need to refer TRT-LLM example for deployment diff --git a/tests/unit/torch/quantization/test_quantize_cpu.py b/tests/unit/torch/quantization/test_quantize_cpu.py index 3e4925e7b63..78b3b92ac48 100644 --- a/tests/unit/torch/quantization/test_quantize_cpu.py +++ b/tests/unit/torch/quantization/test_quantize_cpu.py @@ -28,6 +28,7 @@ save_restore_test, ) from pydantic import ValidationError +from torch import nn import modelopt.torch.opt as mto import modelopt.torch.quantization as mtq @@ -464,6 +465,60 @@ def test_atomicity_later_cfg_entry_does_not_inherit_earlier(): ) +class _HybridLinearAttnModel(nn.Module): + """Minimal Qwen3.5/3.6-style hybrid layer with GatedDeltaNet module names.""" + + def __init__(self, h=64): + super().__init__() + linear_attn = nn.Module() + linear_attn.in_proj_qkv = nn.Linear(h, 3 * h, bias=False) + linear_attn.in_proj_z = nn.Linear(h, h, bias=False) + linear_attn.in_proj_a = nn.Linear(h, 4, bias=False) + linear_attn.in_proj_b = nn.Linear(h, 4, bias=False) + linear_attn.out_proj = nn.Linear(h, h, bias=False) + self_attn = nn.Module() + self_attn.q_proj = nn.Linear(h, h, bias=False) + layer = nn.Module() + layer.linear_attn = linear_attn + layer.self_attn = self_attn + self.model = nn.Module() + self.model.layers = nn.ModuleList([layer]) + + def forward(self, x): + attn = self.model.layers[0].linear_attn + h = x.shape[-1] + out = attn.out_proj(attn.in_proj_qkv(x)[..., :h] * attn.in_proj_z(x)) + out = out + attn.in_proj_a(x).sum() + attn.in_proj_b(x).sum() + return out + self.model.layers[0].self_attn.q_proj(x) + + def get_input(self): + return torch.randn(2, 64) + + +@pytest.mark.parametrize("config", [mtq.NVFP4_DEFAULT_CFG, mtq.FP8_DEFAULT_CFG]) +def test_default_presets_disable_packed_linear_attn_projections(config): + """Generic presets must not quantize GatedDeltaNet packed projections (issue #1933). + + TRT-LLM's Qwen3.5/3.6 weight mapper repacks ``linear_attn.in_proj_qkv`` / + ``in_proj_z`` by splitting every checkpoint tensor along dim 0; the scalar + per-tensor scales of quantized modules (``input_scale``, ``weight_scale_2``) + are 0-dim and crash the split. This exercises the fnmatch patterns from + ``default_disabled_quantizers`` against real module names via ``mtq.quantize``. + """ + model = mtq.quantize(_HybridLinearAttnModel(), config, lambda m: m(m.get_input())) + + prefix = "model.layers.0.linear_attn" + for module_name in ("in_proj_qkv", "in_proj_z", "in_proj_a", "in_proj_b"): + quantized = model.get_submodule(f"{prefix}.{module_name}") + assert not quantized.weight_quantizer.is_enabled, f"{module_name} must stay unquantized" + assert not quantized.input_quantizer.is_enabled, f"{module_name} must stay unquantized" + # out_proj is not repacked by the deployment mapper and self_attn projections + # are per-module in the checkpoint; both stay quantized. + for module_name in ("linear_attn.out_proj", "self_attn.q_proj"): + quantized = model.get_submodule(f"model.layers.0.{module_name}") + assert quantized.weight_quantizer.is_enabled, f"{module_name} should be quantized" + + def test_legacy_dict_format_end_to_end(): """Old dict-format quant_cfg works end-to-end through mtq.quantize via normalization.""" model = SimpleLinear()