Offload token embd to CPU backend (breaks NPU)

Author: wine99

ggml/src/ggml-openvino/ggml-decoder.cpp

@@ -171,7 +171,8 @@ void GgmlOvDecoder::set_input_output(ggml_tensor * node, bool naive) {
         if (!naive && !src->view_src) {
             ggml_backend_buffer * buffer = src->buffer;
 
-            if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY || src->flags & GGML_TENSOR_FLAG_INPUT) {
+            if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY || src->flags & GGML_TENSOR_FLAG_INPUT ||
+                src_name.find("OPENVINO#") == 0) {
                 ov::PartialShape stateful_kv_shape;
                 // GGML_BACKEND_BUFFER_USAGE_ANY are kv caches
                 if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY) {
@@ -402,6 +403,11 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, co
         int len = m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1;
         input_shape = ov::PartialShape{1, 1, 1, len};
 
+    } else if (is_inp_emb(input, op)) {
+        // embeddings
+        input_shape = ov::PartialShape{get_shape(input)};
+        input_shape[2] = -1;
+
     } else if (is_output_idx(input, op)) {
         // output index
         input_shape = ov::PartialShape{1, 1, 1, m_is_static ? m_compute_params.output_len : -1};

ggml/src/ggml-openvino/ggml-decoder.h

@@ -226,7 +226,7 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
     }
 
     inline static bool is_inp_emb(const ggml_tensor * tensor, const ggml_tensor * op) {
-        return tensor->op == GGML_OP_GET_ROWS && op->op == GGML_OP_RMS_NORM;
+        return op->op == GGML_OP_RMS_NORM && (tensor->op == GGML_OP_GET_ROWS || tensor->op == GGML_OP_NONE);
     }
 
     inline static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) {

ggml/src/ggml-openvino/ggml-openvino.cpp

@@ -941,7 +941,7 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
 }
 
 static bool ggml_backend_openvino_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
-    return ggml_backend_buft_is_openvino(buft) || ggml_backend_buft_is_host(buft);
+    return ggml_backend_buft_is_openvino(buft);
     GGML_UNUSED(dev);
 }