From 406339640568d8aab2c5d1002ce7016b4a5ffd17 Mon Sep 17 00:00:00 2001 From: mkulakow Date: Tue, 21 Jul 2026 09:55:58 +0200 Subject: [PATCH 1/2] Fix issue wiht QWEN3 emnbeddings on NPU --- src/embeddings/embeddings_servable.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/embeddings/embeddings_servable.cpp b/src/embeddings/embeddings_servable.cpp index 7adcba6a8e..efa10ed499 100644 --- a/src/embeddings/embeddings_servable.cpp +++ b/src/embeddings/embeddings_servable.cpp @@ -415,11 +415,19 @@ std::shared_ptr EmbeddingsServable::applyPrePostProcessing(ov::Core& postProcInferRequestsQueue = std::make_unique(postProcCompiledModel, numberOfParallelInferRequests); npuPostprocessingRequired = true; - // These are the settings for NPU model - if (getMaxModelLength().has_value()) { - config.max_length = getMaxModelLength().value(); - } - // Models other than Qwen requires reshaping to static shape to work on NPU. + // NPU model settings. + // + // For short-context models (max_position_embeddings < 1024) we reshape to a static + // shape and pin config.max_length to the model's max_position_embeddings, since the + // NPU compiler needs a static sequence length for those. + // + // For long-context / Qwen-style dynamic models we deliberately do NOT propagate + // max_position_embeddings into config.max_length. It would be forwarded as + // NPUW_LLM_MAX_PROMPT_LEN, and the NPU compiler cannot legalize the resulting + // shape (e.g. VPU.NCE.Reduce fails EnsureNCEOpsSizeRequirements at seq_len=32768 + // for Qwen3-Embedding). Instead the user can override MAX_PROMPT_LEN via + // plugin_config in graph.pbtxt; otherwise get_npu_text_embedding_config falls + // back to a default of 1024, matching ov::genai::TextEmbeddingPipeline behaviour. if (getMaxModelLength().has_value() && getMaxModelLength().value() < 1024) { modelIsStatic = true; config.padding_side = "right"; From 63d0a03f662cd0c6b85935cfd085e33d4351e3c5 Mon Sep 17 00:00:00 2001 From: mkulakow Date: Wed, 22 Jul 2026 09:53:01 +0200 Subject: [PATCH 2/2] Remove part of a comment --- src/embeddings/embeddings_servable.cpp | 7 +------ src/rerank/BUILD | 1 + src/rerank/rerank_servable.hpp | 2 ++ 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/embeddings/embeddings_servable.cpp b/src/embeddings/embeddings_servable.cpp index efa10ed499..714b767368 100644 --- a/src/embeddings/embeddings_servable.cpp +++ b/src/embeddings/embeddings_servable.cpp @@ -422,12 +422,7 @@ std::shared_ptr EmbeddingsServable::applyPrePostProcessing(ov::Core& // NPU compiler needs a static sequence length for those. // // For long-context / Qwen-style dynamic models we deliberately do NOT propagate - // max_position_embeddings into config.max_length. It would be forwarded as - // NPUW_LLM_MAX_PROMPT_LEN, and the NPU compiler cannot legalize the resulting - // shape (e.g. VPU.NCE.Reduce fails EnsureNCEOpsSizeRequirements at seq_len=32768 - // for Qwen3-Embedding). Instead the user can override MAX_PROMPT_LEN via - // plugin_config in graph.pbtxt; otherwise get_npu_text_embedding_config falls - // back to a default of 1024, matching ov::genai::TextEmbeddingPipeline behaviour. + // max_position_embeddings into config.max_length. if (getMaxModelLength().has_value() && getMaxModelLength().value() < 1024) { modelIsStatic = true; config.padding_side = "right"; diff --git a/src/rerank/BUILD b/src/rerank/BUILD index b703aa1117..4db374994a 100644 --- a/src/rerank/BUILD +++ b/src/rerank/BUILD @@ -20,6 +20,7 @@ load("//:common_settings.bzl", "ovms_cc_library") ovms_cc_library( name = "rerank_servable", hdrs = ["rerank_servable.hpp"], + srcs = ["rerank_servable.cpp"], deps = ["//src:sidepacket_servable", "//src/port:rapidjson_document", "//src/port:rapidjson_istreamwrapper", diff --git a/src/rerank/rerank_servable.hpp b/src/rerank/rerank_servable.hpp index f287d5b60f..26cd25fda2 100644 --- a/src/rerank/rerank_servable.hpp +++ b/src/rerank/rerank_servable.hpp @@ -50,6 +50,8 @@ struct RerankServable : SidepacketServable { addBosToken = false; } } + + std::shared_ptr applyPrePostProcessing(ov::Core& core, std::shared_ptr model, ov::AnyMap& properties) override; }; using RerankServableMap = std::unordered_map>;