Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions src/embeddings/embeddings_servable.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -415,11 +415,14 @@ std::shared_ptr<ov::Model> EmbeddingsServable::applyPrePostProcessing(ov::Core&
postProcInferRequestsQueue = std::make_unique<OVInferRequestsQueue>(postProcCompiledModel, numberOfParallelInferRequests);
npuPostprocessingRequired = true;

// These are the settings for NPU model
if (getMaxModelLength().has_value()) {
config.max_length = getMaxModelLength().value();
}
// Models other than Qwen requires reshaping to static shape to work on NPU.
// NPU model settings.
//
// For short-context models (max_position_embeddings < 1024) we reshape to a static
// shape and pin config.max_length to the model's max_position_embeddings, since the
// NPU compiler needs a static sequence length for those.
Comment on lines +420 to +422

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good comment.

//
// For long-context / Qwen-style dynamic models we deliberately do NOT propagate
// max_position_embeddings into config.max_length.
if (getMaxModelLength().has_value() && getMaxModelLength().value() < 1024) {
modelIsStatic = true;
config.padding_side = "right";
Expand Down
1 change: 1 addition & 0 deletions src/rerank/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ load("//:common_settings.bzl", "ovms_cc_library")
ovms_cc_library(
name = "rerank_servable",
hdrs = ["rerank_servable.hpp"],
srcs = ["rerank_servable.cpp"],
deps = ["//src:sidepacket_servable",
"//src/port:rapidjson_document",
"//src/port:rapidjson_istreamwrapper",
Expand Down
2 changes: 2 additions & 0 deletions src/rerank/rerank_servable.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ struct RerankServable : SidepacketServable {
addBosToken = false;
}
}

std::shared_ptr<ov::Model> applyPrePostProcessing(ov::Core& core, std::shared_ptr<ov::Model> model, ov::AnyMap& properties) override;
};

using RerankServableMap = std::unordered_map<std::string, std::shared_ptr<RerankServable>>;
Expand Down