diff --git a/src/embeddings/embeddings_servable.cpp b/src/embeddings/embeddings_servable.cpp index 7adcba6a8e..714b767368 100644 --- a/src/embeddings/embeddings_servable.cpp +++ b/src/embeddings/embeddings_servable.cpp @@ -415,11 +415,14 @@ std::shared_ptr EmbeddingsServable::applyPrePostProcessing(ov::Core& postProcInferRequestsQueue = std::make_unique(postProcCompiledModel, numberOfParallelInferRequests); npuPostprocessingRequired = true; - // These are the settings for NPU model - if (getMaxModelLength().has_value()) { - config.max_length = getMaxModelLength().value(); - } - // Models other than Qwen requires reshaping to static shape to work on NPU. + // NPU model settings. + // + // For short-context models (max_position_embeddings < 1024) we reshape to a static + // shape and pin config.max_length to the model's max_position_embeddings, since the + // NPU compiler needs a static sequence length for those. + // + // For long-context / Qwen-style dynamic models we deliberately do NOT propagate + // max_position_embeddings into config.max_length. if (getMaxModelLength().has_value() && getMaxModelLength().value() < 1024) { modelIsStatic = true; config.padding_side = "right"; diff --git a/src/rerank/BUILD b/src/rerank/BUILD index b703aa1117..4db374994a 100644 --- a/src/rerank/BUILD +++ b/src/rerank/BUILD @@ -20,6 +20,7 @@ load("//:common_settings.bzl", "ovms_cc_library") ovms_cc_library( name = "rerank_servable", hdrs = ["rerank_servable.hpp"], + srcs = ["rerank_servable.cpp"], deps = ["//src:sidepacket_servable", "//src/port:rapidjson_document", "//src/port:rapidjson_istreamwrapper", diff --git a/src/rerank/rerank_servable.hpp b/src/rerank/rerank_servable.hpp index f287d5b60f..26cd25fda2 100644 --- a/src/rerank/rerank_servable.hpp +++ b/src/rerank/rerank_servable.hpp @@ -50,6 +50,8 @@ struct RerankServable : SidepacketServable { addBosToken = false; } } + + std::shared_ptr applyPrePostProcessing(ov::Core& core, std::shared_ptr model, ov::AnyMap& properties) override; }; using RerankServableMap = std::unordered_map>;