Commit ebbb18522 for llama.cpp
commit ebbb185227c31f1652f1445e2623563d2f67fe5a
Author: Ravi Panchumarthy <ravi.panchumarthy@intel.com>
Date: Thu Sep 17 03:46:14 2026 -0700
openvino : Update OpenVINO to 2026.4;fix clangd,MSVC warnings; (#29009)
* Update to openvino-2026.4
* Update OV docs
* ggml-openvino : fix clangd and MSVC warnings
* fix int to ptr cast, more internal linkage enforcement, and avoiding duplicate switch case
---------
Co-authored-by: Mostafa Faheem <mostafaaafaheem@gmail.com>
diff --git a/.devops/openvino.Dockerfile b/.devops/openvino.Dockerfile
index 13301ba28..e301aa8f5 100644
--- a/.devops/openvino.Dockerfile
+++ b/.devops/openvino.Dockerfile
@@ -1,5 +1,5 @@
-ARG OPENVINO_VERSION_MAJOR=2026.3.1
-ARG OPENVINO_VERSION_FULL=2026.3.1.22476.56d9685302d
+ARG OPENVINO_VERSION_MAJOR=2026.4
+ARG OPENVINO_VERSION_FULL=2026.4.0.22959.99c81491cc3
ARG UBUNTU_VERSION=24.04
# Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
@@ -10,9 +10,9 @@ ARG COMPUTE_RUNTIME_VERSION_FULL=26.31.39395.13-0
ARG IGDGMM_VERSION=22.10.0
# Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
-ARG NPU_DRIVER_VERSION=v1.35.0
-ARG NPU_DRIVER_FULL=v1.35.0.20260722-29947505341
-ARG LIBZE1_VERSION=1.28.2-1~24.04~ppa1
+ARG NPU_DRIVER_VERSION=v1.38.0
+ARG NPU_DRIVER_FULL=v1.38.0.20260910-34487311128
+ARG LIBZE1_VERSION=1.32.0-1~24.04~ppa1
# Optional proxy build arguments
ARG http_proxy=
@@ -173,7 +173,7 @@ RUN --mount=type=cache,target=/var/cache/intel-npu,sharing=locked \
fi; \
DEB=/var/cache/intel-npu/libze1_${LIBZE1_VERSION}_amd64.deb; \
if [ ! -f "$DEB" ]; then \
- wget -q -O "$DEB" https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260606T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb; \
+ wget -q -O "$DEB" https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260830T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb; \
fi; \
mkdir /tmp/npu/ && cd /tmp/npu/ && tar -xf "$TGZ" && cp "$DEB" .; \
apt-get update; \
diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml
index 4a23ec2d4..27512a142 100644
--- a/.github/workflows/build-cache.yml
+++ b/.github/workflows/build-cache.yml
@@ -41,8 +41,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3.1"
- OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
+ OPENVINO_VERSION_MAJOR: "2026.4"
+ OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
steps:
- name: Clone
@@ -69,8 +69,8 @@ jobs:
env:
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3.1"
- OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
+ OPENVINO_VERSION_MAJOR: "2026.4"
+ OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
steps:
- name: Clone
diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml
index 86aba456c..daa08b1bf 100644
--- a/.github/workflows/build-openvino.yml
+++ b/.github/workflows/build-openvino.yml
@@ -41,8 +41,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3.1"
- OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
+ OPENVINO_VERSION_MAJOR: "2026.4"
+ OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
steps:
- name: Clone
@@ -96,8 +96,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3.1"
- OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
+ OPENVINO_VERSION_MAJOR: "2026.4"
+ OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
steps:
- name: Clone
diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml
index fd3722bcf..d54f71ac5 100644
--- a/.github/workflows/build-self-hosted.yml
+++ b/.github/workflows/build-self-hosted.yml
@@ -412,8 +412,8 @@ jobs:
env:
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3.1"
- OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
+ OPENVINO_VERSION_MAJOR: "2026.4"
+ OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
steps:
- name: Clone
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 8389f017b..cace91037 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -555,8 +555,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3.1"
- OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
+ OPENVINO_VERSION_MAJOR: "2026.4"
+ OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
steps:
- name: Set OpenVINO version output
@@ -669,8 +669,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3.1"
- OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
+ OPENVINO_VERSION_MAJOR: "2026.4"
+ OPENVINO_VERSION_FULL: "2026.4.0.22959.99c81491cc3"
steps:
- name: Set OpenVINO version output
diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md
index c1e39c5bf..3d7919775 100644
--- a/docs/backend/OPENVINO.md
+++ b/docs/backend/OPENVINO.md
@@ -12,6 +12,8 @@ The OpenVINO backend is implemented in `ggml/src/ggml-openvino` and provides a t
- Compiles and caches the model for the target device.
- Binds GGML tensor memory to OpenVINO inference tensors and runs inference.
+For guidance on contributing to the OpenVINO backend, see the [OpenVINO Backend Contributing Guide](https://github.com/ravi9/llamacpp-ov-dev-guide/blob/main/contributing-llamacpp-ov.md).
+
## Contents
- [Supported Devices](#supported-devices)
@@ -96,7 +98,7 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
- **SL** = Stateless (`GGML_OPENVINO_STATEFUL_EXECUTION=0`)
- **SF** = Stateful (`GGML_OPENVINO_STATEFUL_EXECUTION=1`)
- Note: The NPU operates in stateless mode only.
-- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel OpenCL GPU Driver 26.31.39395.13-0 | Intel NPU Driver 1.35.0.
+- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel Graphics Compiler 2.41.5 | Intel OpenCL GPU Driver 26.31.39395.13-0 | Intel NPU Driver 1.38.0.
- See [Known Limitations](#known-limitations) for context on observed failures.
| Model | CPU (SL / SF) | GPU (SL / SF) | NPU (SL) |
@@ -117,9 +119,9 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
| [lmstudio-community/Qwen3.5-9B-Q4_K_M](https://huggingface.co/lmstudio-community/Qwen3.5-9B-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
| | | | |
| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
-| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
-| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ |
-| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ |
+| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
+| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
| | | | |
| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
@@ -132,9 +134,9 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
| [bartowski/DeepSeek-R1-Distill-Llama-8B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
-| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ / ✓ | ✗ / ✗ | ✓ |
+| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [ibm-granite/granite-4.0-micro-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
-| [ibm-granite/granite-4.0-1b-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-1b-GGUF) | ✓ / ✓ | ✗ / ✗ | ✗ |
+| [ibm-granite/granite-4.0-1b-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-1b-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
| [ibm-research/granite-3.2-8b-instruct-Q4_K_M](https://huggingface.co/ibm-research/granite-3.2-8b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [HuggingFaceTB/smollm2-1.7b-instruct-q4_k_m](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
@@ -242,8 +244,8 @@ chmod +x build-llamacpp-ov.sh
# ============================================
set -euo pipefail
-OPENVINO_VERSION_MAJOR="2026.3.1"
-OPENVINO_VERSION_FULL="2026.3.1.22476.56d9685302d"
+OPENVINO_VERSION_MAJOR="2026.4"
+OPENVINO_VERSION_FULL="2026.4.0.22959.99c81491cc3"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
OPENVINO_INSTALL_DIR="/opt/intel/openvino_${OPENVINO_VERSION_MAJOR}"
@@ -340,7 +342,7 @@ echo " ./build/ReleaseOV/bin/llama-cli -m model.gguf"
```
> [!NOTE]
-> The script pins OpenVINO `2026.3.1` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release.
+> The script pins OpenVINO `2026.4` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release.
</details>
@@ -370,8 +372,8 @@ REM ============================================
REM llama.cpp OpenVINO Build Script (Ninja)
REM ============================================
-set "OPENVINO_VERSION_MAJOR=2026.3.1"
-set "OPENVINO_VERSION_FULL=2026.3.1.22476.56d9685302d"
+set "OPENVINO_VERSION_MAJOR=2026.4"
+set "OPENVINO_VERSION_FULL=2026.4.0.22959.99c81491cc3"
set "SCRIPT_DIR=%~dp0"
set "VCPKG_DIR=C:\vcpkg"
@@ -550,7 +552,7 @@ endlocal
```
> [!NOTE]
-> The script pins OpenVINO `2026.3.1` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**.
+> The script pins OpenVINO `2026.4` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**.
</details>
diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp
index 0b99834aa..cd06b22e8 100644
--- a/ggml/src/ggml-openvino/ggml-decoder.cpp
+++ b/ggml/src/ggml-openvino/ggml-decoder.cpp
@@ -245,7 +245,7 @@ void GgmlOvDecoder::set_input_output() {
if (src->op == GGML_OP_VIEW) {
// Traverse upward through nested VIEW operations
std::remove_reference_t<decltype(current_node_info.node_inputs_views[src_name])> view_chain;
- auto current = src;
+ auto * current = src;
while (current != nullptr) {
auto current_name = get_tensor_ov_name(m_cgraph, current);
@@ -612,9 +612,8 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
if (node->src[1]->view_src != nullptr) {
if (node->src[3] != nullptr) {
return 4; // decoder self-attention
- } else {
- return 5; // cross-attention or encoder self-attention
- };
+ }
+ return 5; // cross-attention or encoder self-attention
}
break;
default:
@@ -736,8 +735,7 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
bool rope_seen = false;
for (int i = 0; i < cgraph->n_nodes; i++) {
- auto * node = cgraph->nodes[i];
- std::string name = std::string(node->name);
+ ggml_tensor * node = cgraph->nodes[i];
const int attention_pattern_case = get_attention_pattern_case(node);
if (attention_pattern_case != -1) {
ggml_tensor * cache_k_permute = nullptr;
@@ -948,7 +946,6 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
if (m_naive) {
return input != nullptr ? ov::PartialShape{get_shape(input)} : ov::PartialShape{get_shape(op)};
}
- auto name = std::string(input->name);
ov::PartialShape input_shape;
if (is_inp_tok(input, op) || is_inp_pos(input, op)) {
@@ -1474,7 +1471,7 @@ std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor
void GgmlOvDecoder::dump_cgraph(const ggml_cgraph * cgraph, std::string & filename) {
std::ofstream file(filename);
if (!file.is_open()) {
- std::cerr << "Failed to open file" << std::endl;
+ std::cerr << "Failed to open file" << '\n';
return;
}
@@ -1580,11 +1577,11 @@ void print_tensor_address_map(const ggml_cgraph * cgraph) {
}
}
for (const auto & pair : address_map) {
- std::cout << "Address: " << pair.first << std::endl;
+ std::cout << "Address: " << pair.first << '\n';
for (const auto & name : pair.second) {
std::cout << name << " ; ";
}
- std::cout << std::endl << std::endl;
+ std::cout << "\n\n";
}
}
@@ -2226,7 +2223,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
std::cout << ", ";
}
}
- std::cout << "]" << std::endl;
+ std::cout << "]" << '\n';
// print the src name & shape with the dynamic dim for debugging
for (int j = 0; j < GGML_MAX_SRC; j++) {
ggml_tensor * src = node->src[j];
@@ -2245,9 +2242,9 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
std::cout << ", ";
}
}
- std::cout << "]" << std::endl;
+ std::cout << "]" << '\n';
}
- std::cout << std::endl;
+ std::cout << '\n';
}
}
}
diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h
index 7f9d45a48..056e39e87 100644
--- a/ggml/src/ggml-openvino/ggml-decoder.h
+++ b/ggml/src/ggml-openvino/ggml-decoder.h
@@ -354,41 +354,41 @@ public:
void update_io(ggml_cgraph * cgraph);
- inline static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) {
+ static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE;
}
- inline static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) {
+ static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_ROPE && tensor == op->src[1];
}
// IMROPE packs 4 stacked position planes (t/h/w/e) into inp_pos, each of length
// n_tokens; other modes carry a single position per token.
- inline static int get_inp_pos_n_planes(const ggml_tensor * op) {
+ static int get_inp_pos_n_planes(const ggml_tensor * op) {
return op->op_params[2] == GGML_ROPE_TYPE_IMROPE ? 4 : 1;
}
- inline static bool is_inp_emb(const ggml_tensor * tensor, const ggml_tensor * op) {
+ static bool is_inp_emb(const ggml_tensor * tensor, const ggml_tensor * op) {
return tensor->op == GGML_OP_GET_ROWS && op->op == GGML_OP_RMS_NORM;
}
- inline static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) {
+ static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]) ||
(op->op == GGML_OP_SOFT_MAX && tensor == op->src[1]);
}
- inline static bool is_inp_mean(const ggml_tensor * tensor, const ggml_tensor * op) {
+ static bool is_inp_mean(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_MUL_MAT && tensor == op->src[1] && tensor->op == GGML_OP_NONE &&
(tensor->flags & GGML_TENSOR_FLAG_INPUT) && tensor->type == GGML_TYPE_F32 &&
op->src[0] != nullptr && op->src[0]->op != GGML_OP_NONE;
}
- inline static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) {
+ static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_ROPE && tensor == op->src[2];
}
// also returns true for cache_s and cache_r in SSM/DeltaNet models
- inline static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) {
+ static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) {
if (tensor == nullptr) {
return false;
}
@@ -396,14 +396,14 @@ public:
(op != nullptr && op->op == GGML_OP_SET_ROWS && op->src[2] == tensor);
}
- inline static bool is_conv_state_writeback(const ggml_tensor * node) {
+ static bool is_conv_state_writeback(const ggml_tensor * node) {
return node->op == GGML_OP_CPY && node->view_src != nullptr && is_kvcache(node->view_src, nullptr) &&
node->src[0] != nullptr && node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr &&
node->src[0]->src[0]->op == GGML_OP_CONCAT && node->src[1] != nullptr &&
node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src;
}
- inline static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
+ static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_SET_ROWS && op->src[1] == tensor;
}
@@ -411,13 +411,13 @@ public:
return m_model_params.swa_mask != nullptr && tensor == m_model_params.swa_mask;
}
- inline static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
+ static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE &&
op->src[1]->op == GGML_OP_NONE;
}
// the state permutation index input used in SSM/DeltaNet models (inp->s_copy in llama-graph.cpp)
- inline static bool is_inp_s_copy(const ggml_tensor * tensor, const ggml_tensor * op) {
+ static bool is_inp_s_copy(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] &&
op->src[0]->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY;
}
@@ -481,5 +481,3 @@ private:
};
void print_tensor_address_map(const ggml_cgraph * cgraph);
-
-std::optional<int> extract_layer_from_name(const std::string & name);
diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp
index 52e1a297c..216e3b8a6 100644
--- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp
+++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp
@@ -472,10 +472,6 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
switch (tensor->type) {
case GGML_TYPE_MXFP4:
- layout.is_u4 = true;
- layout.is_symmetric = true;
- break;
-
case GGML_TYPE_Q4_0:
layout.is_u4 = true;
layout.is_symmetric = true;
diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp
index 044b4da1c..02c596223 100644
--- a/ggml/src/ggml-openvino/ggml-openvino.cpp
+++ b/ggml/src/ggml-openvino/ggml-openvino.cpp
@@ -28,12 +28,7 @@
#include <string>
#include <vector>
-#ifndef _WIN32
-# include <sys/mman.h>
-# include <unistd.h>
-#endif
-
-#if defined(_WIN32)
+#ifdef _WIN32
# define WIN32_LEAN_AND_MEAN
# ifndef NOMINMAX
# define NOMINMAX
@@ -61,6 +56,7 @@
// - CPU repack buffer: tensor->extra stores tensor_traits with repacked data
// =====================================================
+namespace {
// Buffer context that manages per-tensor allocations (no contiguous buffer for weights)
struct ggml_backend_openvino_buffer_context {
int device;
@@ -199,6 +195,7 @@ struct ggml_backend_openvino_buffer_type_context {
int device;
std::string name;
};
+} // namespace
// =====================================================
// Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS)
@@ -258,14 +255,16 @@ void ggml_openvino_release_weight_buffers() {
for (const auto & b : reg.buffers) {
// Align down/up to page boundaries so madvise only drops whole pages
// fully owned by this buffer.
- const long page = sysconf(_SC_PAGESIZE);
- uintptr_t start = reinterpret_cast<uintptr_t>(b.first);
- uintptr_t end = start + b.second;
- uintptr_t astart = (start + page - 1) & ~(uintptr_t) (page - 1);
- uintptr_t aend = end & ~(uintptr_t) (page - 1);
- if (aend > astart) {
- if (madvise(reinterpret_cast<void *>(astart), aend - astart, MADV_DONTNEED) == 0) {
- total += aend - astart;
+ const size_t page = (size_t) sysconf(_SC_PAGESIZE);
+ const uintptr_t ustart = reinterpret_cast<uintptr_t>(b.first);
+ const size_t offset_to_page = (page - (ustart & (page - 1))) & (page - 1);
+ if (b.second > offset_to_page) {
+ const size_t aligned_len = (b.second - offset_to_page) & ~(page - 1);
+ if (aligned_len > 0) {
+ char * astart = static_cast<char *>(b.first) + offset_to_page;
+ if (madvise(astart, aligned_len, MADV_DONTNEED) == 0) {
+ total += aligned_len;
+ }
}
}
}
@@ -876,11 +875,13 @@ GGML_BACKEND_API bool ggml_backend_is_openvino(ggml_backend_t backend) {
return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_openvino_guid());
}
+namespace {
struct ggml_backend_openvino_device_context {
int device;
std::string name;
std::string description;
};
+}
static const char * ggml_backend_openvino_device_get_name(ggml_backend_dev_t dev) {
ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context;
@@ -1588,9 +1589,11 @@ static const struct ggml_backend_device_i ggml_backend_openvino_device_interface
/* .event_synchronize = */ NULL,
};
+namespace {
struct ggml_backend_openvino_reg_context {
std::vector<ggml_backend_dev_t> devices;
};
+}
static const char * ggml_backend_openvino_reg_get_name(ggml_backend_reg_t reg) {
return GGML_OPENVINO_NAME;
diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp
index 93f9e8254..824d24478 100644
--- a/ggml/src/ggml-openvino/ggml-quants.cpp
+++ b/ggml/src/ggml-openvino/ggml-quants.cpp
@@ -34,6 +34,15 @@
#include <string>
#include <vector>
+// From <openvino>/src/common/transformations/include/transformations/utils/utils.hpp
+namespace ov::op::util {
+// From <openvino>/src/common/transformations/include/transformations/utils/utils.hpp
+bool get_single_value(const std::shared_ptr<ov::op::v0::Constant> & const_node,
+ float & value,
+ bool check_value_range = true);
+} // namespace ov::op::util
+
+namespace {
void unpack_32_4(const uint8_t * data, uint8_t * dst) {
std::fill_n(dst, 16, 0);
for (int j = 0; j < 16; ++j) {
@@ -48,11 +57,11 @@ void unpack_32_4(const uint8_t * data, uint8_t * dst) {
}
}
-static constexpr size_t MXFP4_BLOCK_SIZE = 32;
-static constexpr size_t MXFP4_BLOCK_QS_SIZE = MXFP4_BLOCK_SIZE / 2;
-static constexpr size_t MXFP4_BLOCK_BYTES = sizeof(uint8_t) + MXFP4_BLOCK_QS_SIZE;
+constexpr size_t MXFP4_BLOCK_SIZE = 32;
+constexpr size_t MXFP4_BLOCK_QS_SIZE = MXFP4_BLOCK_SIZE / 2;
+constexpr size_t MXFP4_BLOCK_BYTES = sizeof(uint8_t) + MXFP4_BLOCK_QS_SIZE;
-static void pack_32_mxfp4_for_openvino(const uint8_t * data, uint8_t * dst) {
+void pack_32_mxfp4_for_openvino(const uint8_t * data, uint8_t * dst) {
for (int j = 0; j < static_cast<int>(MXFP4_BLOCK_QS_SIZE); j += 2) {
const uint8_t v0 = data[j] & 0x0F;
const uint8_t v1 = (data[j + 1] & 0x0F) << 4;
@@ -419,7 +428,7 @@ void extract_q6_k_data(const ggml_tensor * tensor,
}
}
-static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) {
+inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) {
if (j < 4) {
*d = q[j] & 63;
*m = q[j + 4] & 63;
@@ -514,9 +523,9 @@ void extract_q5_k_data(const ggml_tensor * tensor,
ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight,
ov::Tensor & scales,
ov::Tensor & zp,
- size_t group_size,
- bool use_bias,
- bool for_gather_matmul) {
+ size_t group_size = GGML_QUANTIZATION_GROUP_SIZE,
+ bool use_bias = false,
+ bool for_gather_matmul = false) {
ov::Shape orig_shape = weight.get_shape();
bool is_signed = (weight.get_element_type() == ov::element::i8); // Symmetric: signed weights, no ZP
@@ -611,13 +620,24 @@ ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight,
return std::make_shared<ov::op::v0::Convert>(result, ov::element::f32);
}
+// If for_gather_matmul is true, the weight tensor may be N-D (e.g. 3D MoE expert weights
+// [n_expert, rows, cols]). The dequantization chain (Convert->[Subtract]->Multiply) is built as
+// usual but left in f16 (no final Convert to f32) -- ov::pass::MarkDequantization (registered in
+// translate_session.cpp) marks the chain so it survives model-build-time ConstantFolding -- see
+// make_int8_weights.cpp/make_int4_weights.cpp. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul
+// directly from the resulting f16 dequant chain.
+//
+// When use_bias is true (explicitly, or implicitly because for_gather_matmul is true), the zp
+// tensor is expected to hold an exact f16 bias value (rather than a rounded integer zero point);
+// it is converted in place into an exact zero_point = -bias/scale and consumed via Subtract, not
+// Add, so the chain still matches OpenVINO's Convert->Subtract->Multiply decompression pattern.
// See make_int8_weights for the meaning of for_gather_matmul.
ov::Output<ov::Node> make_int4_weights(ov::Tensor & weight,
ov::Tensor & scales,
ov::Tensor & zp,
- size_t group_size,
- bool use_bias,
- bool for_gather_matmul) {
+ size_t group_size = GGML_QUANTIZATION_GROUP_SIZE,
+ bool use_bias = false,
+ bool for_gather_matmul = false) {
ov::Shape orig_weight_shape = weight.get_shape();
bool is_signed = (weight.get_element_type() == ov::element::i4); // Symmetric: signed weights, no ZP
@@ -746,13 +766,262 @@ ov::Output<ov::Node> make_mxfp4_moe_packed_weights(ov::Tensor & weight) {
return weights_node;
}
+void quantize_q4_0(const float * x,
+ ov::Tensor & weights_arr,
+ ov::Tensor & scales_arr,
+ ov::Tensor & zp_arr,
+ int64_t k,
+ int64_t qk) {
+ assert(k % qk == 0);
+ const int nb = k / qk;
+
+ auto * weights = static_cast<uint8_t *>(weights_arr.data());
+ auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
+ bool is_symmetric = (weights_arr.get_element_type() == ov::element::i4); // Signed i4 path
+
+ if (!is_symmetric) {
+ auto * zp = static_cast<uint8_t *>(zp_arr.data());
+ for (int i = 0; i < nb; i++) {
+ float amax = 0.0f;
+ float max = 0.0f;
+ for (int j = 0; j < qk; j++) {
+ const float v = x[i * qk + j];
+ if (amax < fabsf(v)) {
+ amax = fabsf(v);
+ max = v;
+ }
+ }
+ const float d = max / -8;
+ if (d == 0) {
+ scales[i] = ov::float16(1.0f);
+ if (i % 2 == 0) {
+ zp[i / 2] = 8;
+ } else {
+ zp[i / 2] |= (8 << 4);
+ }
+ memset(weights + i * qk / 2, 8 | (8 << 4), qk / 2);
+ continue;
+ }
+ const float id = 1.0f / d;
+ scales[i] = ov::float16(d);
+ if (i % 2 == 0) {
+ zp[i / 2] = 8;
+ } else {
+ zp[i / 2] |= (8 << 4);
+ }
+ for (int j = 0; j < qk / 2; ++j) {
+ const float x0 = x[i * qk + 2 * j] * id;
+ const float x1 = x[i * qk + 2 * j + 1] * id;
+ const uint8_t xi0 = MIN(15, (int8_t) (x0 + 8.5f));
+ const uint8_t xi1 = MIN(15, (int8_t) (x1 + 8.5f));
+ weights[i * qk / 2 + j] = xi0 | (xi1 << 4);
+ }
+ }
+ } else {
+ // Symmetric: produce signed i4 values in [-8, 7]
+ for (int i = 0; i < nb; i++) {
+ float amax = 0.0f;
+ float max = 0.0f;
+ for (int j = 0; j < qk; j++) {
+ const float v = x[i * qk + j];
+ if (amax < fabsf(v)) {
+ amax = fabsf(v);
+ max = v;
+ }
+ }
+ const float d = max / -8;
+ if (d == 0) {
+ scales[i] = ov::float16(1.0f);
+ // i4 value 0 packed: 0x00
+ memset(weights + i * qk / 2, 0, qk / 2);
+ continue;
+ }
+ const float id = 1.0f / d;
+ scales[i] = ov::float16(d);
+ for (int j = 0; j < qk / 2; ++j) {
+ const float x0 = x[i * qk + 2 * j] * id;
+ const float x1 = x[i * qk + 2 * j + 1] * id;
+ // Signed i4: range [-8, 7]. Quantize as round(x*id), then pack as 4-bit two's complement.
+ int8_t si0 = (int8_t) std::max(-8, std::min(7, (int) roundf(x0)));
+ int8_t si1 = (int8_t) std::max(-8, std::min(7, (int) roundf(x1)));
+ weights[i * qk / 2 + j] = (si0 & 0x0F) | ((si1 & 0x0F) << 4);
+ }
+ }
+ }
+}
+
+// Asymmetric u4 quantization with a per-group scale and zero point.
+//
+// Unlike quantize_q4_0's unsigned branch, which pins the zero point to 8 and is therefore
+// symmetric, this keeps a real per-group zero point, so a group whose values are not centred on
+// zero does not waste half its range.
+void quantize_q4_1_asym(const float * x,
+ ov::Tensor & weights_arr,
+ ov::Tensor & scales_arr,
+ ov::Tensor & zp_arr,
+ int64_t k,
+ int64_t qk) {
+ assert(k % qk == 0);
+ const int nb = k / qk;
+
+ auto * weights = static_cast<uint8_t *>(weights_arr.data());
+ auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
+ auto * zp = static_cast<uint8_t *>(zp_arr.data());
+
+ // u4 zero points are packed two per byte, low nibble first, indexed by group -- the same
+ // convention as the unsigned branch of quantize_q4_0.
+ auto store_zp = [zp](int i, uint8_t v) {
+ if (i % 2 == 0) {
+ zp[i / 2] = v & 0x0F;
+ } else {
+ zp[i / 2] |= (uint8_t) ((v & 0x0F) << 4);
+ }
+ };
+
+ for (int i = 0; i < nb; i++) {
+ float vmin = x[i * qk];
+ float vmax = x[i * qk];
+ for (int j = 1; j < qk; j++) {
+ const float v = x[i * qk + j];
+ vmin = std::min(vmin, v);
+ vmax = std::max(vmax, v);
+ }
+ // Include 0 in the range so an all-positive or all-negative group still represents zero
+ // exactly -- these are weights, so an exact zero matters.
+ vmin = std::min(vmin, 0.0f);
+ vmax = std::max(vmax, 0.0f);
+
+ const float d = (vmax - vmin) / 15.0f;
+ if (d == 0.0f) {
+ scales[i] = ov::float16(1.0f);
+ store_zp(i, 0);
+ memset(weights + i * qk / 2, 0, qk / 2);
+ continue;
+ }
+ const float id = 1.0f / d;
+
+ // The zero point is itself a 4-bit integer, so round it and dequantize as (q - zq) * d.
+ const int zq = std::max(0, std::min(15, (int) lroundf(-vmin * id)));
+ scales[i] = ov::float16(d);
+ store_zp(i, (uint8_t) zq);
+
+ for (int j = 0; j < qk / 2; ++j) {
+ const float x0 = x[i * qk + 2 * j] * id;
+ const float x1 = x[i * qk + 2 * j + 1] * id;
+ const uint8_t q0 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x0) + zq));
+ const uint8_t q1 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x1) + zq));
+ weights[i * qk / 2 + j] = (uint8_t) (q0 | (q1 << 4));
+ }
+ }
+}
+
+void quantize_q8_0(const float * x,
+ ov::Tensor & weights_arr,
+ ov::Tensor & scales_arr,
+ ov::Tensor & zp_arr,
+ int64_t k,
+ int64_t qk,
+ int64_t block_offset = 0) {
+ assert(k % qk == 0);
+ const int nb = k / qk;
+
+ // block_offset lets a caller quantize a chunk of blocks into the right place in the
+ // output buffers (used for streaming requant). x points at this chunk's first block;
+ // outputs are advanced by block_offset blocks. Q8 has one scale/zp per block (no
+ // nibble packing), so any block boundary is safe.
+ auto * weights = static_cast<uint8_t *>(weights_arr.data()) + block_offset * qk;
+ auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>() + block_offset;
+ bool is_symmetric = (weights_arr.get_element_type() == ov::element::i8); // Signed i8 path
+
+ if (!is_symmetric) {
+ auto * zp = static_cast<uint8_t *>(zp_arr.data()) + block_offset;
+ for (int i = 0; i < nb; i++) {
+ float amax = 0.0f;
+ for (int j = 0; j < qk; j++) {
+ const float v = x[i * qk + j];
+ amax = std::max(amax, fabsf(v));
+ }
+ const float d = amax / 127.0f;
+ const float id = d ? 1.0f / d : 0.0f;
+ scales[i] = ov::float16(d);
+ zp[i] = 128;
+ for (int j = 0; j < qk; ++j) {
+ const float x0 = x[i * qk + j] * id;
+ const int8_t xi0 = roundf(x0);
+ weights[i * qk + j] = (uint8_t) (xi0 + 128);
+ }
+ }
+ } else {
+ // Symmetric: store signed int8 values directly
+ auto * signed_weights = reinterpret_cast<int8_t *>(weights);
+ for (int i = 0; i < nb; i++) {
+ float amax = 0.0f;
+ for (int j = 0; j < qk; j++) {
+ const float v = x[i * qk + j];
+ amax = std::max(amax, fabsf(v));
+ }
+ const float d = amax / 127.0f;
+ const float id = d ? 1.0f / d : 0.0f;
+ scales[i] = ov::float16(d);
+ for (int j = 0; j < qk; ++j) {
+ const float x0 = x[i * qk + j] * id;
+ signed_weights[i * qk + j] = (int8_t) roundf(x0);
+ }
+ }
+ }
+}
+
+void quantize_q8_1(const float * x,
+ ov::Tensor & weights_arr,
+ ov::Tensor & scales_arr,
+ ov::Tensor & zp_arr,
+ int64_t k,
+ int64_t qk,
+ int64_t block_offset = 0) {
+ assert(k % qk == 0);
+ const int nb = k / qk;
+
+ // See quantize_q8_0: block_offset places this chunk's output at the right block.
+ auto * weights = static_cast<uint8_t *>(weights_arr.data()) + block_offset * qk;
+ auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>() + block_offset;
+ auto * zp = static_cast<uint8_t *>(zp_arr.data()) + block_offset;
+ for (int i = 0; i < nb; i++) {
+ float min = std::numeric_limits<float>::max();
+ float max = std::numeric_limits<float>::lowest();
+
+ for (int j = 0; j < qk; j++) {
+ const float v = x[i * qk + j];
+ min = std::min(v, min);
+ max = std::max(v, max);
+ }
+
+ const float d = (max - min) / ((1 << 8) - 1);
+ const float id = d ? 1.0f / d : 0.0f;
+ scales[i] = ov::float16(d);
+ // zp = -min / scale (Q8_1 is asymmetric)
+ zp[i] = (d != 0.0f) ? (uint8_t) std::round(-min / d) : 0;
+
+ for (int j = 0; j < qk; ++j) {
+ const float x0 = (x[i * qk + j] - min) * id;
+ const uint8_t xi0 = roundf(x0);
+ weights[i * qk + j] = xi0;
+ }
+ }
+}
+
// Extract quantized weights from tensor and create weight subgraph
+// If weights/scales/zp are provided (non-empty), uses them as output buffers
+// Otherwise allocates new ov::Tensors internally
+// Returns the weight node (make_int4_weights or make_int8_weights result)
std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor,
- const void * data,
+ const void * data, // Source data pointer (may differ from tensor->data)
ov::Tensor & weights,
ov::Tensor & scales,
ov::Tensor & zp,
- bool use_bias) {
+ // Use an exact f16 zero point (vs. a rounded integer one); always
+ // used for for_gather_matmul (3D MoE expert) weights regardless of
+ // this flag, and also settable explicitly for test-backend-ops.
+ bool use_bias = false) {
// Create a temporary tensor for extraction functions that read from tensor->data
ggml_tensor temp_tensor = *tensor;
temp_tensor.data = const_cast<void *>(data);
@@ -837,9 +1106,11 @@ std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor,
return result;
}
-// Requantize weights to target format, writing to provided buffers
+// Requantize weights from tensor to target format, writing to provided buffers
+// For F16 target, only weights buffer is used (scales/zp ignored)
+// Returns the weight node
std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor,
- const void * data,
+ const void * data, // Source data pointer
ExtraQuantType requant_type,
int64_t block_size,
ov::Tensor & weights,
@@ -933,6 +1204,7 @@ std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor,
result->set_friendly_name(tensor->name);
return result;
}
+} // namespace
OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, void * output_base_ptr, bool use_bias) {
GGML_ASSERT(tensor != nullptr);
@@ -1030,7 +1302,9 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo
} else {
result.weights = ov::Tensor(ov::element::f16, node_shape);
}
- ov::Tensor dummy_scales, dummy_zp; // Not used for F16
+ // Not used for F16:
+ ov::Tensor dummy_scales;
+ ov::Tensor dummy_zp;
result.weight_node =
requantize_to_buffers(tensor, data, ExtraQuantType::F16, 0, result.weights, dummy_scales, dummy_zp);
return result;
@@ -1039,10 +1313,14 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo
// Quantized path (normal extraction or quantized requant)
// Create weight/scale/zp tensors - shared between both paths
// For symmetric quantization, use signed types (i4/i8) and no ZP tensor
- ov::element::Type weight_type = tensor->type == GGML_TYPE_MXFP4 ?
- ov::element::f4e2m1 :
- (layout.is_symmetric ? (layout.is_u4 ? ov::element::i4 : ov::element::i8) :
- (layout.is_u4 ? ov::element::u4 : ov::element::u8));
+ ov::element::Type weight_type;
+ if (tensor->type == GGML_TYPE_MXFP4) {
+ weight_type = ov::element::f4e2m1;
+ } else if (layout.is_symmetric) {
+ weight_type = layout.is_u4 ? ov::element::i4 : ov::element::i8;
+ } else {
+ weight_type = layout.is_u4 ? ov::element::u4 : ov::element::u8;
+ }
ov::Shape scale_shape = node_shape;
scale_shape.back() /= layout.weights_per_block;
@@ -1060,28 +1338,25 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo
scale_shape.back() /= layout.weights_per_block;
}
+ const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16;
+ ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8;
+ if (zp_is_f16) {
+ zp_type = ov::element::f16;
+ }
+
if (output_base_ptr) {
uint8_t * buf_base = static_cast<uint8_t *>(output_base_ptr);
result.weights = ov::Tensor(weight_type, node_shape, buf_base + layout.weights_offset);
- const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16;
result.scales = ov::Tensor(scale_type, scale_shape, buf_base + layout.scales_offset);
if (!layout.is_symmetric) {
- ov::element::Type zp_type =
- zp_is_f16 ? ov::element::f16 : (layout.is_u4 ? ov::element::u4 : ov::element::u8);
result.zp = ov::Tensor(zp_type, scale_shape, buf_base + layout.zp_offset);
}
// else: result.zp remains default-constructed (empty) for symmetric
} else {
result.weights = ov::Tensor(weight_type, node_shape);
- const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16;
result.scales = ov::Tensor(scale_type, scale_shape);
if (!layout.is_symmetric) {
- if (zp_is_f16) {
- result.zp = ov::Tensor(ov::element::f16, scale_shape);
- } else {
- ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8;
- result.zp = ov::Tensor(zp_type, scale_shape);
- }
+ result.zp = ov::Tensor(zp_type, scale_shape);
}
// else: result.zp remains default-constructed (empty) for symmetric
}
@@ -1096,246 +1371,3 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo
return result;
}
-
-void quantize_q4_0(const float * x,
- ov::Tensor & weights_arr,
- ov::Tensor & scales_arr,
- ov::Tensor & zp_arr,
- int64_t k,
- int64_t qk) {
- assert(k % qk == 0);
- const int nb = k / qk;
-
- auto * weights = static_cast<uint8_t *>(weights_arr.data());
- auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
- bool is_symmetric = (weights_arr.get_element_type() == ov::element::i4); // Signed i4 path
-
- if (!is_symmetric) {
- auto * zp = static_cast<uint8_t *>(zp_arr.data());
- for (int i = 0; i < nb; i++) {
- float amax = 0.0f;
- float max = 0.0f;
- for (int j = 0; j < qk; j++) {
- const float v = x[i * qk + j];
- if (amax < fabsf(v)) {
- amax = fabsf(v);
- max = v;
- }
- }
- const float d = max / -8;
- if (d == 0) {
- scales[i] = ov::float16(1.0f);
- if (i % 2 == 0) {
- zp[i / 2] = 8;
- } else {
- zp[i / 2] |= (8 << 4);
- }
- memset(weights + i * qk / 2, 8 | (8 << 4), qk / 2);
- continue;
- }
- const float id = 1.0f / d;
- scales[i] = ov::float16(d);
- if (i % 2 == 0) {
- zp[i / 2] = 8;
- } else {
- zp[i / 2] |= (8 << 4);
- }
- for (int j = 0; j < qk / 2; ++j) {
- const float x0 = x[i * qk + 2 * j] * id;
- const float x1 = x[i * qk + 2 * j + 1] * id;
- const uint8_t xi0 = MIN(15, (int8_t) (x0 + 8.5f));
- const uint8_t xi1 = MIN(15, (int8_t) (x1 + 8.5f));
- weights[i * qk / 2 + j] = xi0 | (xi1 << 4);
- }
- }
- } else {
- // Symmetric: produce signed i4 values in [-8, 7]
- for (int i = 0; i < nb; i++) {
- float amax = 0.0f;
- float max = 0.0f;
- for (int j = 0; j < qk; j++) {
- const float v = x[i * qk + j];
- if (amax < fabsf(v)) {
- amax = fabsf(v);
- max = v;
- }
- }
- const float d = max / -8;
- if (d == 0) {
- scales[i] = ov::float16(1.0f);
- // i4 value 0 packed: 0x00
- memset(weights + i * qk / 2, 0, qk / 2);
- continue;
- }
- const float id = 1.0f / d;
- scales[i] = ov::float16(d);
- for (int j = 0; j < qk / 2; ++j) {
- const float x0 = x[i * qk + 2 * j] * id;
- const float x1 = x[i * qk + 2 * j + 1] * id;
- // Signed i4: range [-8, 7]. Quantize as round(x*id), then pack as 4-bit two's complement.
- int8_t si0 = (int8_t) std::max(-8, std::min(7, (int) roundf(x0)));
- int8_t si1 = (int8_t) std::max(-8, std::min(7, (int) roundf(x1)));
- weights[i * qk / 2 + j] = (si0 & 0x0F) | ((si1 & 0x0F) << 4);
- }
- }
- }
-}
-
-// Asymmetric u4 quantization with a per-group scale and zero point.
-//
-// Unlike quantize_q4_0's unsigned branch, which pins the zero point to 8 and is therefore
-// symmetric, this keeps a real per-group zero point, so a group whose values are not centred on
-// zero does not waste half its range.
-void quantize_q4_1_asym(const float * x,
- ov::Tensor & weights_arr,
- ov::Tensor & scales_arr,
- ov::Tensor & zp_arr,
- int64_t k,
- int64_t qk) {
- assert(k % qk == 0);
- const int nb = k / qk;
-
- auto * weights = static_cast<uint8_t *>(weights_arr.data());
- auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
- auto * zp = static_cast<uint8_t *>(zp_arr.data());
-
- // u4 zero points are packed two per byte, low nibble first, indexed by group -- the same
- // convention as the unsigned branch of quantize_q4_0.
- auto store_zp = [zp](int i, uint8_t v) {
- if (i % 2 == 0) {
- zp[i / 2] = v & 0x0F;
- } else {
- zp[i / 2] |= (uint8_t) ((v & 0x0F) << 4);
- }
- };
-
- for (int i = 0; i < nb; i++) {
- float vmin = x[i * qk];
- float vmax = x[i * qk];
- for (int j = 1; j < qk; j++) {
- const float v = x[i * qk + j];
- vmin = std::min(vmin, v);
- vmax = std::max(vmax, v);
- }
- // Include 0 in the range so an all-positive or all-negative group still represents zero
- // exactly -- these are weights, so an exact zero matters.
- vmin = std::min(vmin, 0.0f);
- vmax = std::max(vmax, 0.0f);
-
- const float d = (vmax - vmin) / 15.0f;
- if (d == 0.0f) {
- scales[i] = ov::float16(1.0f);
- store_zp(i, 0);
- memset(weights + i * qk / 2, 0, qk / 2);
- continue;
- }
- const float id = 1.0f / d;
-
- // The zero point is itself a 4-bit integer, so round it and dequantize as (q - zq) * d.
- const int zq = std::max(0, std::min(15, (int) lroundf(-vmin * id)));
- scales[i] = ov::float16(d);
- store_zp(i, (uint8_t) zq);
-
- for (int j = 0; j < qk / 2; ++j) {
- const float x0 = x[i * qk + 2 * j] * id;
- const float x1 = x[i * qk + 2 * j + 1] * id;
- const uint8_t q0 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x0) + zq));
- const uint8_t q1 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x1) + zq));
- weights[i * qk / 2 + j] = (uint8_t) (q0 | (q1 << 4));
- }
- }
-}
-
-void quantize_q8_0(const float * x,
- ov::Tensor & weights_arr,
- ov::Tensor & scales_arr,
- ov::Tensor & zp_arr,
- int64_t k,
- int64_t qk,
- int64_t block_offset) {
- assert(k % qk == 0);
- const int nb = k / qk;
-
- // block_offset lets a caller quantize a chunk of blocks into the right place in the
- // output buffers (used for streaming requant). x points at this chunk's first block;
- // outputs are advanced by block_offset blocks. Q8 has one scale/zp per block (no
- // nibble packing), so any block boundary is safe.
- auto * weights = static_cast<uint8_t *>(weights_arr.data()) + block_offset * qk;
- auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>() + block_offset;
- bool is_symmetric = (weights_arr.get_element_type() == ov::element::i8); // Signed i8 path
-
- if (!is_symmetric) {
- auto * zp = static_cast<uint8_t *>(zp_arr.data()) + block_offset;
- for (int i = 0; i < nb; i++) {
- float amax = 0.0f;
- for (int j = 0; j < qk; j++) {
- const float v = x[i * qk + j];
- amax = std::max(amax, fabsf(v));
- }
- const float d = amax / 127.0f;
- const float id = d ? 1.0f / d : 0.0f;
- scales[i] = ov::float16(d);
- zp[i] = 128;
- for (int j = 0; j < qk; ++j) {
- const float x0 = x[i * qk + j] * id;
- const int8_t xi0 = roundf(x0);
- weights[i * qk + j] = (uint8_t) (xi0 + 128);
- }
- }
- } else {
- // Symmetric: store signed int8 values directly
- auto * signed_weights = reinterpret_cast<int8_t *>(weights);
- for (int i = 0; i < nb; i++) {
- float amax = 0.0f;
- for (int j = 0; j < qk; j++) {
- const float v = x[i * qk + j];
- amax = std::max(amax, fabsf(v));
- }
- const float d = amax / 127.0f;
- const float id = d ? 1.0f / d : 0.0f;
- scales[i] = ov::float16(d);
- for (int j = 0; j < qk; ++j) {
- const float x0 = x[i * qk + j] * id;
- signed_weights[i * qk + j] = (int8_t) roundf(x0);
- }
- }
- }
-}
-
-void quantize_q8_1(const float * x,
- ov::Tensor & weights_arr,
- ov::Tensor & scales_arr,
- ov::Tensor & zp_arr,
- int64_t k,
- int64_t qk,
- int64_t block_offset) {
- assert(k % qk == 0);
- const int nb = k / qk;
-
- // See quantize_q8_0: block_offset places this chunk's output at the right block.
- auto * weights = static_cast<uint8_t *>(weights_arr.data()) + block_offset * qk;
- auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>() + block_offset;
- auto * zp = static_cast<uint8_t *>(zp_arr.data()) + block_offset;
- for (int i = 0; i < nb; i++) {
- float min = std::numeric_limits<float>::max();
- float max = std::numeric_limits<float>::lowest();
-
- for (int j = 0; j < qk; j++) {
- const float v = x[i * qk + j];
- min = std::min(v, min);
- max = std::max(v, max);
- }
-
- const float d = (max - min) / ((1 << 8) - 1);
- const float id = d ? 1.0f / d : 0.0f;
- scales[i] = ov::float16(d);
- // zp = -min / scale (Q8_1 is asymmetric)
- zp[i] = (d != 0.0f) ? (uint8_t) std::round(-min / d) : 0;
-
- for (int j = 0; j < qk; ++j) {
- const float x0 = (x[i * qk + j] - min) * id;
- const uint8_t xi0 = roundf(x0);
- weights[i * qk + j] = xi0;
- }
- }
-}
diff --git a/ggml/src/ggml-openvino/ggml-quants.h b/ggml/src/ggml-openvino/ggml-quants.h
index d5273727e..04fe0218a 100644
--- a/ggml/src/ggml-openvino/ggml-quants.h
+++ b/ggml/src/ggml-openvino/ggml-quants.h
@@ -2,112 +2,12 @@
#include "ggml-openvino-extra.h" // For ExtraQuantType
#include "ggml.h"
-#include <cstdint>
-#include <openvino/op/constant.hpp>
#include <openvino/core/node_output.hpp>
+#include <openvino/op/constant.hpp>
#include <openvino/runtime/tensor.hpp>
-void unpack_32_4(const uint8_t * data, uint8_t * dst);
-
-void extract_q4_0_data(const ggml_tensor * tensor,
- ov::Tensor & weights_arr,
- ov::Tensor & scales_arr,
- ov::Tensor & zp_arr);
-
-void extract_q4_1_data(const ggml_tensor * tensor,
- ov::Tensor & weights_arr,
- ov::Tensor & scales_arr,
- ov::Tensor & zp_arr,
- bool use_bias = false);
-
-void extract_q5_1_data(const ggml_tensor * tensor,
- ov::Tensor & weights_arr,
- ov::Tensor & scales_arr,
- ov::Tensor & zp_arr,
- bool use_bias = false);
-
-void extract_q8_0_data(const ggml_tensor * tensor,
- ov::Tensor & weights_arr,
- ov::Tensor & scales_arr,
- ov::Tensor & zp_arr);
-
-void unpack_256_4(const uint8_t * data, uint8_t * dst);
-
-void extract_q4_k_data(const ggml_tensor * tensor,
- ov::Tensor & weights_arr,
- ov::Tensor & scales_arr,
- ov::Tensor & zp_arr,
- bool use_bias = false);
-
-void extract_q5_k_data(const ggml_tensor * tensor,
- ov::Tensor & weights_arr,
- ov::Tensor & scales_arr,
- ov::Tensor & zp_arr,
- bool use_bias = false);
-
-void extract_q6_k_data(const ggml_tensor * tensor,
- ov::Tensor & weights_arr,
- ov::Tensor & scales_arr,
- ov::Tensor & zp_arr);
-
-void extract_mxfp4_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr);
-
static constexpr size_t GGML_QUANTIZATION_GROUP_SIZE = 32;
-// If for_gather_matmul is true, the weight tensor may be N-D (e.g. 3D MoE expert weights
-// [n_expert, rows, cols]). The dequantization chain (Convert->[Subtract]->Multiply) is built as
-// usual but left in f16 (no final Convert to f32) -- ov::pass::MarkDequantization (registered in
-// translate_session.cpp) marks the chain so it survives model-build-time ConstantFolding -- see
-// make_int8_weights.cpp/make_int4_weights.cpp. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul
-// directly from the resulting f16 dequant chain.
-//
-// When use_bias is true (explicitly, or implicitly because for_gather_matmul is true), the zp
-// tensor is expected to hold an exact f16 bias value (rather than a rounded integer zero point);
-// it is converted in place into an exact zero_point = -bias/scale and consumed via Subtract, not
-// Add, so the chain still matches OpenVINO's Convert->Subtract->Multiply decompression pattern.
-ov::Output<ov::Node> make_int8_weights(ov::Tensor & weight,
- ov::Tensor & scales,
- ov::Tensor & zp,
- size_t group_size = GGML_QUANTIZATION_GROUP_SIZE,
- bool use_bias = false,
- bool for_gather_matmul = false);
-
-ov::Output<ov::Node> make_int4_weights(ov::Tensor & weight,
- ov::Tensor & scales,
- ov::Tensor & zp,
- size_t group_size = GGML_QUANTIZATION_GROUP_SIZE,
- bool use_bias = false,
- bool for_gather_matmul = false);
-
-ov::Output<ov::Node> make_mxfp4_weights(ov::Tensor & weight, ov::Tensor & scales);
-
-ov::Output<ov::Node> make_mxfp4_moe_packed_weights(ov::Tensor & weight);
-
-// Extract quantized weights from tensor and create weight subgraph
-// If weights/scales/zp are provided (non-empty), uses them as output buffers
-// Otherwise allocates new ov::Tensors internally
-// Returns the weight node (make_int4_weights or make_int8_weights result)
-std::shared_ptr<ov::Node> extract_quantized_weights(
- const ggml_tensor * tensor,
- const void * data, // Source data pointer (may differ from tensor->data)
- ov::Tensor & weights,
- ov::Tensor & scales,
- ov::Tensor & zp,
- bool use_bias = false); // Use an exact f16 zero point (vs. a rounded integer one); always
- // used for for_gather_matmul (3D MoE expert) weights regardless of
- // this flag, and also settable explicitly for test-backend-ops.
-
-// Requantize weights from tensor to target format, writing to provided buffers
-// For F16 target, only weights buffer is used (scales/zp ignored)
-// Returns the weight node
-std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor,
- const void * data, // Source data pointer
- ExtraQuantType requant_type,
- int64_t block_size,
- ov::Tensor & weights,
- ov::Tensor & scales,
- ov::Tensor & zp);
-
inline const char * extra_quant_type_name(ExtraQuantType t) {
switch (t) {
case ExtraQuantType::F16:
@@ -156,41 +56,3 @@ OvWeight process_weight_tensor(
// always used for for_gather_matmul (3D MoE expert) weights
// regardless of this flag, and also settable explicitly for
// test-backend-ops.
-
-void quantize_q4_0(const float * x,
- ov::Tensor & weights_arr,
- ov::Tensor & scales_arr,
- ov::Tensor & zp_arr,
- int64_t k,
- int64_t qk);
-void quantize_q8_1(const float * x,
- ov::Tensor & weights_arr,
- ov::Tensor & scales_arr,
- ov::Tensor & zp_arr,
- int64_t k,
- int64_t qk,
- int64_t block_offset = 0);
-void quantize_q4_1_asym(const float * x,
- ov::Tensor & weights_arr,
- ov::Tensor & scales_arr,
- ov::Tensor & zp_arr,
- int64_t k,
- int64_t qk);
-void quantize_q8_0(const float * x,
- ov::Tensor & weights_arr,
- ov::Tensor & scales_arr,
- ov::Tensor & zp_arr,
- int64_t k,
- int64_t qk,
- int64_t block_offset = 0);
-
-namespace ov {
-namespace op {
-namespace util {
-// From <openvino>/src/common/transformations/include/transformations/utils/utils.hpp
-bool get_single_value(const std::shared_ptr<ov::op::v0::Constant> & const_node,
- float & value,
- bool check_value_range = true);
-} // namespace util
-} // namespace op
-} // namespace ov
diff --git a/ggml/src/ggml-openvino/model-cache.cpp b/ggml/src/ggml-openvino/model-cache.cpp
index 3fc7028d8..3725fbd22 100644
--- a/ggml/src/ggml-openvino/model-cache.cpp
+++ b/ggml/src/ggml-openvino/model-cache.cpp
@@ -237,7 +237,8 @@ bool ggml_openvino_model_cache_verify_manifest(const std::string & path,
if (!f.is_open()) {
return false;
}
- std::string tag, val;
+ std::string tag;
+ std::string val;
// header: fingerprint
if (!(f >> tag >> val) || tag != "fingerprint" || val != hex64(fingerprint)) {
return false;
diff --git a/ggml/src/ggml-openvino/openvino/frontend.h b/ggml/src/ggml-openvino/openvino/frontend.h
index 72134a3e8..4e301d32e 100644
--- a/ggml/src/ggml-openvino/openvino/frontend.h
+++ b/ggml/src/ggml-openvino/openvino/frontend.h
@@ -12,7 +12,6 @@ namespace ggml {
class FrontEnd {
public:
- using Ptr = std::shared_ptr<FrontEnd>;
FrontEnd();
static std::shared_ptr<Model> convert(const InputModel::Ptr & model, bool naive = false);
diff --git a/ggml/src/ggml-openvino/openvino/op/add_id.cpp b/ggml/src/ggml-openvino/openvino/op/add_id.cpp
index e54d700d4..79bdbe877 100644
--- a/ggml/src/ggml-openvino/openvino/op/add_id.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/add_id.cpp
@@ -20,7 +20,7 @@ namespace op {
static ov::Output<ov::Node> reshape_add_id_input_to_2d(const ov::Output<ov::Node> & input,
const ov::PartialShape & input_shape,
const std::vector<int> & dims) {
- const auto actual_shape = input.get_partial_shape();
+ const auto & actual_shape = input.get_partial_shape();
if (actual_shape.rank().is_static() && actual_shape.rank().get_length() == 2) {
return input;
}
diff --git a/ggml/src/ggml-openvino/openvino/op/cont.cpp b/ggml/src/ggml-openvino/openvino/op/cont.cpp
index 1d6cc6721..9888f6b93 100644
--- a/ggml/src/ggml-openvino/openvino/op/cont.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/cont.cpp
@@ -3,12 +3,9 @@
#include "../op_table.h"
#include "../utils.h"
-#include <climits>
-#include <cstdint>
#include <memory>
#include <openvino/op/reshape.hpp>
#include <openvino/op/slice.hpp>
-#include <vector>
namespace ov {
namespace frontend {
diff --git a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp
index 06547f3d2..b06d01dca 100644
--- a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp
@@ -195,7 +195,9 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
auto tile_kv = [&](int64_t n_heads, int64_t n_heads_kv, int64_t hs, ov::Output<Node> kv) {
int64_t f = n_heads / n_heads_kv;
if (f > 1 && n_heads_kv > 1) {
- ov::Output<ov::Node> kv_broadcast_shape, kv_unsqueezed, new_kv_shape;
+ ov::Output<ov::Node> kv_broadcast_shape;
+ ov::Output<ov::Node> kv_unsqueezed;
+ ov::Output<ov::Node> new_kv_shape;
auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {2});
kv_unsqueezed = std::make_shared<ov::op::v0::Unsqueeze>(kv, unsqueeze_axes);
diff --git a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp
index 07eeb3c8f..8d07c90bf 100644
--- a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp
@@ -196,7 +196,7 @@ static OutputVector translate_gated_delta_net_ref(const NodeContext & context) {
}
// Merge batch and head dims: [B*H_v, T, S_v]
- auto merge_bh = [&](ov::Output<ov::Node> x, int64_t last_dim) {
+ auto merge_bh = [&](const ov::Output<ov::Node> & x, int64_t last_dim) {
auto shape = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector<int64_t>{B * H_v, T, last_dim});
return std::make_shared<ov::op::v1::Reshape>(x, shape, false);
};
diff --git a/ggml/src/ggml-openvino/openvino/op/im2col.cpp b/ggml/src/ggml-openvino/openvino/op/im2col.cpp
index 856e97f79..08b53f260 100644
--- a/ggml/src/ggml-openvino/openvino/op/im2col.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/im2col.cpp
@@ -1,7 +1,6 @@
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
-#include "ggml-impl.h"
#include <cstddef>
#include <memory>
diff --git a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp
index 0de6161be..a336924e1 100644
--- a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp
@@ -42,7 +42,7 @@ ov::Output<ov::Node> slice_axis(const ov::Output<ov::Node> & input, int64_t axis
ov::Output<ov::Node> static_shape_dims_or_shapeof(const ov::Output<ov::Node> & input,
const std::vector<int> & dims) {
- const auto partial_shape = input.get_partial_shape();
+ const auto & partial_shape = input.get_partial_shape();
if (partial_shape.is_static()) {
std::vector<int64_t> values;
values.reserve(dims.size());
diff --git a/ggml/src/ggml-openvino/openvino/op/pad.cpp b/ggml/src/ggml-openvino/openvino/op/pad.cpp
index d2b861142..ae3d7be18 100644
--- a/ggml/src/ggml-openvino/openvino/op/pad.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/pad.cpp
@@ -8,6 +8,7 @@
#include <openvino/op/pad.hpp>
#include <openvino/op/reshape.hpp>
#include <openvino/op/shape_of.hpp>
+#include <utility>
#include <vector>
namespace ov {
@@ -20,7 +21,7 @@ namespace {
ov::Output<ov::Node> translate_circular_pad(ov::Output<ov::Node> input,
const std::array<int32_t, 8> & pads,
const ov::Shape & input_shape) {
- ov::Output<ov::Node> result = input;
+ ov::Output<ov::Node> result = std::move(input);
const std::array<int32_t, 4> pads_begin = {pads[6], pads[4], pads[2], pads[0]};
const std::array<int32_t, 4> pads_end = {pads[7], pads[5], pads[3], pads[1]};
diff --git a/ggml/src/ggml-openvino/openvino/op/repeat.cpp b/ggml/src/ggml-openvino/openvino/op/repeat.cpp
index d58b59e4e..b7aeaa24f 100644
--- a/ggml/src/ggml-openvino/openvino/op/repeat.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/repeat.cpp
@@ -1,7 +1,6 @@
#include "../node_context.h"
#include "../op_table.h"
#include "../utils.h"
-#include "ggml.h"
#include <memory>
#include <openvino/op/broadcast.hpp>
diff --git a/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp b/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp
index 9cbce7db0..25c953545 100644
--- a/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp
@@ -25,9 +25,7 @@ OutputVector translate_rms_norm(const NodeContext & context) {
auto op_case = context.get_op_case();
ov::Output<ov::Node> input_node;
- if (op_case == 1) {
- input_node = process_view_input_new(context, 0);
- } else if (op_case == 2) {
+ if (op_case == 2) {
auto ssm_state_size = context.get_ssm_state_size();
// The GDN op packs [attn | new_state] along the row axis; the state occupies the last
// ssm_state_size * n_seqs rows. Slice it off (scaling by the active sequence count) to keep
diff --git a/ggml/src/ggml-openvino/openvino/op/view.cpp b/ggml/src/ggml-openvino/openvino/op/view.cpp
index 56f5ceec9..ca2d2dc08 100644
--- a/ggml/src/ggml-openvino/openvino/op/view.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/view.cpp
@@ -7,7 +7,6 @@
#include <openvino/op/reshape.hpp>
#include <openvino/op/shape_of.hpp>
#include <openvino/op/slice.hpp>
-#include <set>
namespace ov {
namespace frontend {
@@ -153,7 +152,8 @@ OutputVector translate_view(const NodeContext & context) {
return {input};
}
- int64_t src_elems = 1, dst_elems = 1;
+ int64_t src_elems = 1;
+ int64_t dst_elems = 1;
for (int64_t i = 0; i < src_shape.rank().get_length(); ++i) {
if (src_shape[i].is_dynamic()) {
return {input};
diff --git a/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp
index c9952b1d5..04de2d008 100644
--- a/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp
+++ b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp
@@ -84,7 +84,7 @@ bool KVStateSeqAxis::run_on_model(const std::shared_ptr<ov::Model> & model) {
// Readers still expect seq at dim 1. A reader that is itself the inverse
// Transpose wanted seq at dim 2 all along, so drop it; give anything else the
// inverse Transpose so its input is unchanged.
- for (auto & reader : readers) {
+ for (const auto & reader : readers) {
auto * node = reader.get_node();
if (ov::is_type<ov::op::v6::Assign>(node)) {
continue;
diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp
index 3170c2e4c..e56a4e41d 100644
--- a/ggml/src/ggml-openvino/openvino/translate_session.cpp
+++ b/ggml/src/ggml-openvino/openvino/translate_session.cpp
@@ -344,7 +344,7 @@ std::shared_ptr<Model> TranslateSession::translate_graph(const frontend::InputMo
}
};
- auto node_visitor = [&](std::shared_ptr<GgmlDecoder> decoder, int node_idx) {
+ auto node_visitor = [&](const std::shared_ptr<GgmlDecoder> & decoder, int node_idx) {
auto converted_outputs = translate_node(decoder, node_idx);
if (converted_outputs.empty()) {
return;
diff --git a/ggml/src/ggml-openvino/openvino/utils.cpp b/ggml/src/ggml-openvino/openvino/utils.cpp
index 8bb7678ee..98a85e632 100644
--- a/ggml/src/ggml-openvino/openvino/utils.cpp
+++ b/ggml/src/ggml-openvino/openvino/utils.cpp
@@ -1,7 +1,5 @@
#include "utils.h"
-#include "ggml-impl.h"
-
#include <cmath>
#include <cstddef>
#include <ctime>
@@ -28,13 +26,6 @@ namespace ov {
namespace frontend {
namespace ggml {
-std::string getCurrentTime() {
- std::time_t now = std::time(nullptr);
- char buf[100];
- std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", std::localtime(&now));
- return buf;
-}
-
void num_inputs_check(const NodeContext & context, size_t min_inputs, size_t max_inputs) {
auto input_size = context.get_input_size();
FRONT_END_OP_CONVERSION_CHECK(input_size >= min_inputs, "Got less inputs than expected");
@@ -82,7 +73,7 @@ namespace {
ov::Output<ov::Node> rope_yarn_ramp_mix(int n_dims, const float corr_dims[2], float ext_factor) {
int half_n_dims = n_dims / 2;
std::vector<float> dim_ids_vec(half_n_dims);
- std::iota(dim_ids_vec.begin(), dim_ids_vec.end(), 0);
+ std::iota(dim_ids_vec.begin(), dim_ids_vec.end(), 0.0f);
auto dim_ids = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, (size_t) half_n_dims}, dim_ids_vec);
auto corr_low = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {corr_dims[0]});
auto corr_high = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {corr_dims[1]});
@@ -551,6 +542,7 @@ ov::Output<ov::Node> process_view_input_new(const NodeContext & context, int inp
if (tail_begin >= 0 && tail_end <= tail_src_elems) {
std::vector<int64_t> flat_shape;
+ flat_shape.reserve(slice_dim);
for (int i = 0; i < slice_dim; ++i) {
flat_shape.push_back(static_cast<int64_t>(view_src_ggml_shape[i]));
}
diff --git a/ggml/src/ggml-openvino/openvino/utils.h b/ggml/src/ggml-openvino/openvino/utils.h
index 5d4c35386..d9858f923 100644
--- a/ggml/src/ggml-openvino/openvino/utils.h
+++ b/ggml/src/ggml-openvino/openvino/utils.h
@@ -14,8 +14,6 @@ namespace ggml {
std::string getCurrentTime();
-void dump_ov_model(std::shared_ptr<ov::Model> model);
-
void num_inputs_check(const NodeContext & context, size_t min_inputs, size_t max_inputs);
int non_cont_dim(std::vector<size_t> ne, std::vector<size_t> nb);
diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp
index 44a9b2c78..b1ee792fd 100644
--- a/ggml/src/ggml-openvino/utils.cpp
+++ b/ggml/src/ggml-openvino/utils.cpp
@@ -1,8 +1,8 @@
#include "utils.h"
#include "ggml-impl.h"
-#include "ggml-openvino.h"
#include "ggml-openvino-extra.h"
+#include "ggml-openvino.h"
#include "ggml-openvino/ggml-decoder.h"
#include "ggml.h"
#include "model-cache.h"
@@ -18,7 +18,6 @@
#include <cstring>
#include <fstream>
#include <functional>
-#include <future>
#include <iomanip>
#include <iostream>
#include <memory>
@@ -39,42 +38,7 @@
#include <unordered_map>
#include <vector>
-// Suppress deprecation warning for ov::Tensor::data()
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-
-// Both execution paths use two cache levels:
-// 1. Reuse this backend's decoder/request via graph_key and compatibility checks.
-// 2. On a local miss, look up compiled_graph_key in the shared compilation cache,
-// compile if needed, then create a private request from the compiled model.
-// The shared lock covers compilation and frontend cleanup, never inference.
-enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) {
- ggml_backend_openvino_context * ctx = (ggml_backend_openvino_context *) backend->context;
- try {
- if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_CGRAPH")) {
- std::string filename = "cgraph_ov.txt";
- GgmlOvDecoder::dump_cgraph(cgraph, filename);
- }
-
- const auto is_static = ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC");
-
- GGML_ASSERT(ctx->runtime_context != nullptr);
- std::shared_ptr<ov_runtime_context> r_ctx = std::static_pointer_cast<ov_runtime_context>(ctx->runtime_context);
- std::lock_guard<std::mutex> execution_lock(r_ctx->execution_mutex);
-
- return is_static ? ov_graph_compute_static(cgraph, r_ctx) : ov_graph_compute_dynamic(cgraph, r_ctx);
- } catch (const ov::Exception & e) {
- GGML_LOG_ERROR("GGML OpenVINO backend ov::Exception: %s\n", e.what());
- return GGML_STATUS_FAILED;
- } catch (const std::exception & e) {
- GGML_LOG_ERROR("GGML OpenVINO backend std::exception: %s\n", e.what());
- return GGML_STATUS_FAILED;
- } catch (...) {
- GGML_LOG_ERROR("GGML OpenVINO backend unknown exception\n");
- return GGML_STATUS_FAILED;
- }
-}
-
+namespace {
// For a KV cache input, return an ov::Tensor sized to n_kv (== attention_size
// for that layer) instead of the fully-allocated ctx_per_seq. Pre-conditions:
// * non-static (CPU/GPU) backend, single sequence, seq_active_start == 0
@@ -85,9 +49,9 @@ enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend)
// n_kv rows no longer contain the live prefix
// On any unmet pre-condition returns std::nullopt; the caller falls back to
// the full-size tensor.
-static std::optional<ov::Tensor> try_make_kv_sliced_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
- const std::string & name,
- const ggml_tensor * ggml_tensor) {
+std::optional<ov::Tensor> try_make_kv_sliced_tensor(const std::shared_ptr<GgmlOvDecoder> & ggml_decoder,
+ const std::string & name,
+ const ggml_tensor * ggml_tensor) {
static const bool kv_slice_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_SLICE");
if (kv_slice_disabled) {
return std::nullopt;
@@ -125,7 +89,7 @@ static std::optional<ov::Tensor> try_make_kv_sliced_tensor(std::shared_ptr<GgmlO
return std::nullopt;
}
- ov::Shape full_shape = ggml_decoder->get_shape(ggml_tensor);
+ ov::Shape full_shape = GgmlOvDecoder::get_shape(ggml_tensor);
if (full_shape.size() != 4 || full_shape[0] != 1 || full_shape[1] != 1 ||
static_cast<int>(full_shape[2]) != ctx_per_seq) {
return std::nullopt;
@@ -141,10 +105,10 @@ static std::optional<ov::Tensor> try_make_kv_sliced_tensor(std::shared_ptr<GgmlO
// return gpu_context.create_tensor(ggml_decoder->get_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data);
// }
- return ov::Tensor(ggml_decoder->get_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data);
+ return ov::Tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data);
}
-static uint64_t ggml_openvino_model_cache_extra_cfg(const std::string & device, bool stateful) {
+uint64_t ggml_openvino_model_cache_extra_cfg(const std::string & device, bool stateful) {
const char * manual_gqa_env = ggml_openvino_getenv_str("GGML_OPENVINO_MANUAL_GQA_ATTN");
const bool manual_gqa_enabled = manual_gqa_env != nullptr ?
ggml_openvino_getenv_int("GGML_OPENVINO_MANUAL_GQA_ATTN") > 0 :
@@ -158,7 +122,7 @@ static uint64_t ggml_openvino_model_cache_extra_cfg(const std::string & device,
return extra_cfg;
}
-static std::map<std::string, std::shared_ptr<ov::Node>> get_weight_names(ggml_cgraph * cgraph) {
+std::map<std::string, std::shared_ptr<ov::Node>> get_weight_names(ggml_cgraph * cgraph) {
std::map<std::string, std::shared_ptr<ov::Node>> names;
for (const auto & name : GgmlOvDecoder::collect_weight_names(cgraph)) {
names[name] = nullptr;
@@ -170,8 +134,10 @@ static std::map<std::string, std::shared_ptr<ov::Node>> get_weight_names(ggml_cg
// miss. Include topology, layouts, op parameters, constant extra inputs and weight
// allocation identities. Never use a sampled weight hash or a graph name alone:
// different models can have identical topology. OV buffer IDs survive address reuse.
-static std::string compiled_graph_key(const ggml_cgraph * graph, const GgmlOvDecoder & decoder,
- const std::string & device, int prefill_chunk_size = 0) {
+std::string compiled_graph_key(const ggml_cgraph * graph,
+ const GgmlOvDecoder & decoder,
+ const std::string & device,
+ int prefill_chunk_size = 0) {
std::string key;
auto append = [&key](const auto & value) {
key.append(reinterpret_cast<const char *>(&value), sizeof(value));
@@ -243,8 +209,8 @@ static std::string compiled_graph_key(const ggml_cgraph * graph, const GgmlOvDec
return has_weight_buffer_id ? key : std::string{};
}
-ov::Tensor create_ov_output_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
- std::shared_ptr<ov::InferRequest> infer_request,
+ov::Tensor create_ov_output_tensor(const std::shared_ptr<GgmlOvDecoder> & ggml_decoder,
+ const std::shared_ptr<ov::InferRequest> & infer_request,
int output_index,
const ggml_tensor * ggml_tensor) {
if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, std::string(ggml_tensor->name), ggml_tensor)) {
@@ -260,7 +226,7 @@ ov::Tensor create_ov_output_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
// }
// }
- auto output_type = ggml_decoder->get_ov_type(ggml_tensor);
+ auto output_type = GgmlOvDecoder::get_ov_type(ggml_tensor);
ov::Shape output_shape;
void * output_data = ggml_tensor->data;
if (ggml_decoder->is_static()) {
@@ -273,10 +239,10 @@ ov::Tensor create_ov_output_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
// pointer instead so the OV tensor matches the model output exactly.
if (ggml_tensor->op == GGML_OP_CPY && ggml_tensor->view_src != nullptr &&
ggml_nbytes(ggml_tensor) != ggml_nbytes(ggml_tensor->view_src)) {
- output_shape = ggml_decoder->get_shape(ggml_tensor->view_src);
+ output_shape = GgmlOvDecoder::get_shape(ggml_tensor->view_src);
output_data = ggml_tensor->view_src->data;
} else {
- output_shape = ggml_decoder->get_shape(ggml_tensor);
+ output_shape = GgmlOvDecoder::get_shape(ggml_tensor);
}
}
ov::Tensor output_tensor(output_type, output_shape, output_data);
@@ -286,7 +252,7 @@ ov::Tensor create_ov_output_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
// Rewrite ggml's KV rows into a relayout state that keeps the sequence on dim 2.
// ggml stores [seq][n_heads_kv * head_size]; the state wants [1, n_heads_kv, seq, head_size],
// a different element order, so the rows are copied instead of reinterpreted.
-static ov::Tensor kv_rows_to_seq_axis_2(const ov::Tensor & kv_tensor, size_t n_heads_kv) {
+ov::Tensor kv_rows_to_seq_axis_2(const ov::Tensor & kv_tensor, size_t n_heads_kv) {
const size_t rows = kv_tensor.get_shape()[2];
const size_t head_size = kv_tensor.get_shape()[3] / n_heads_kv;
const size_t elem = kv_tensor.get_element_type().size();
@@ -303,528 +269,392 @@ static ov::Tensor kv_rows_to_seq_axis_2(const ov::Tensor & kv_tensor, size_t n_h
return out;
}
-enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr<ov_runtime_context> r_ctx) {
- auto & core = ov_singleton_core();
- const auto & config = ggml_openvino_get_compile_config();
- const auto & device = r_ctx->device;
- const auto & stateful = r_ctx->stateful;
- static auto is_static = false;
+template <typename T> void set_zero_diagonal(std::vector<T> & matrix, size_t rows, size_t cols, T zero_value = T{}) {
+ for (size_t i = 0; i < rows; ++i) {
+ size_t diag_col = std::min(i, cols - 1);
+ matrix[i * cols + diag_col] = zero_value;
+ }
+}
- static const bool cache_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE");
+ov::Tensor make_contiguous_split_input_tensor(const struct ggml_tensor * ggml_tensor, const ov::Shape & input_shape) {
+ const size_t element_size = ggml_type_size(ggml_tensor->type);
+ const size_t block_size = ggml_blck_size(ggml_tensor->type);
- // is_model_splitted is O(n_nodes^2) plus a create_weight_nodes scan and takes ~20 ms
- // on a Llama-1B decode graph. It is called once per graph_compute invocation but the
- // graph shape is identical across all decode steps, so memoize by graph_key: compute
- // graph_key first (a few hundred us), and if the same key is already in decoder_cache
- // we know the graph is not splitted (only not-splitted graphs get inserted there).
- graph_key key(cgraph);
- bool key_seen = false;
- if (!cache_disabled) {
- std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
- key_seen = r_ctx->decoder_cache.find(key) != r_ctx->decoder_cache.end();
- }
+ GGML_ASSERT(block_size == 1 && "non-contiguous split inputs must be plain element types");
- bool model_is_splitted = key_seen ? false : is_model_splitted(cgraph);
+ const struct ggml_tensor * source_tensor = ggml_tensor->view_src != nullptr ? ggml_tensor->view_src : ggml_tensor;
+ const size_t source_offset = ggml_tensor->view_src != nullptr ? ggml_tensor->view_offs : 0;
- if (is_naive(cgraph)) {
- if (!model_is_splitted) {
- return naive_compute(cgraph, core, device, config, *r_ctx->compiled_cache);
- }
- }
+ std::vector<uint8_t> source_data(ggml_nbytes(source_tensor));
+ ggml_backend_tensor_get(source_tensor, source_data.data(), 0, source_data.size());
- auto start_time = ggml_time_us();
+ ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape);
+ auto * dst = static_cast<uint8_t *>(input_tensor.data());
+ size_t dst_offset = 0;
- std::shared_ptr<GgmlOvDecoder> ggml_decoder;
- std::shared_ptr<ov::InferRequest> infer_request;
- ModelParams m_params;
- ComputeParams c_params;
- std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static);
+ for (size_t i3 = 0; i3 < static_cast<size_t>(ggml_tensor->ne[3]); ++i3) {
+ for (size_t i2 = 0; i2 < static_cast<size_t>(ggml_tensor->ne[2]); ++i2) {
+ for (size_t i1 = 0; i1 < static_cast<size_t>(ggml_tensor->ne[1]); ++i1) {
+ for (size_t i0 = 0; i0 < static_cast<size_t>(ggml_tensor->ne[0]); ++i0) {
+ const size_t src_offset = source_offset + i3 * ggml_tensor->nb[3] + i2 * ggml_tensor->nb[2] +
+ i1 * ggml_tensor->nb[1] + i0 * ggml_tensor->nb[0];
+ std::memcpy(dst + dst_offset, source_data.data() + src_offset, element_size);
+ dst_offset += element_size;
+ }
+ }
+ }
+ }
- const bool cache_enabled = !model_is_splitted && !cache_disabled;
- bool cache_hit = false;
+ return input_tensor;
+}
- int64_t decoder_end_time;
- int64_t conversion_end_time;
- int64_t compile_end_time;
- int64_t infer_end_time;
- int64_t ov_raw_infer_start;
+ov::Tensor convert_ggml_input_to_ov(const std::shared_ptr<GgmlOvDecoder> & ggml_decoder, const std::string & name) {
+ const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(name);
- {
- std::shared_ptr<decoder_runtime_ctx> entry;
- ModelParams old_m_params;
+ if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, name, ggml_tensor)) {
+ return *sliced;
+ }
- if (cache_enabled) {
- std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
- auto it = r_ctx->decoder_cache.find(key);
- cache_hit = it != r_ctx->decoder_cache.end();
- if (cache_hit) {
- entry = it->second;
- } else {
- r_ctx->clear_caches_locked();
- auto mutex = std::make_shared<std::mutex>();
- entry = std::make_shared<decoder_runtime_ctx>(mutex);
- r_ctx->decoder_cache[key] = entry;
- }
- } else {
- auto mutex = std::make_shared<std::mutex>();
- entry = std::make_shared<decoder_runtime_ctx>(mutex);
- cache_hit = false;
+ if (ggml_tensor->extra != nullptr && !ggml_decoder->is_splited_model()) {
+ auto * extra_base = static_cast<ggml_openvino_extra_base *>(ggml_tensor->extra);
+ if (extra_base->type == ggml_openvino_extra_base::Type::TENSOR) {
+ // GGML_LOG_DEBUG("Using ggml_tensor->extra as ov::Tensor for input: %s\n", name.c_str());
+ auto * tensor_extra = static_cast<ggml_openvino_tensor_extra *>(extra_base);
+ return *tensor_extra->tensor;
}
+ }
- std::lock_guard<std::mutex> lock(*(entry->mutex));
- cache_hit = cache_hit && entry->ptr && r_ctx->infer_request_cache.count(key) != 0;
+ // GGML_LOG_DEBUG("Converting ggml tensor to ov::Tensor for input: %s\n", name.c_str());
+ auto * input_data = ggml_tensor->data;
+ ov::Shape input_shape;
+ if (ggml_tensor->op == GGML_OP_VIEW && !ggml_decoder->is_splited_model()) {
+ // This case is added to make test-backend-ops work
+ input_shape = GgmlOvDecoder::get_shape(ggml_tensor->view_src);
+ } else {
+ input_shape = GgmlOvDecoder::get_shape(ggml_tensor);
+ }
- if (cache_hit) {
- ggml_decoder = entry->ptr;
- old_m_params = ggml_decoder->get_model_params();
- if (!ggml_decoder->is_splited_model()) {
- cache_hit = old_m_params.can_reuse_dynamically(m_params);
- }
- }
+ if (ggml_decoder->is_splited_model() && !ggml_is_contiguous(ggml_tensor)) {
+ return make_contiguous_split_input_tensor(ggml_tensor, input_shape);
+ }
- std::vector<std::string> ov_input_names;
- std::vector<std::string> ov_output_names;
+ auto input_tensor = ov::Tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape, input_data);
+ return input_tensor;
+}
- if (cache_hit) {
- std::map<std::string, std::shared_ptr<ov::Node>> model_weights;
- ggml_decoder->set_compute_params(c_params);
- ggml_decoder->set_model_params(m_params);
- if (old_m_params.kv_buffer_changed(m_params)) {
- ggml_decoder->update_io(cgraph);
- }
- ggml_decoder->add_extra_inputs();
- {
- std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
- infer_request = r_ctx->infer_request_cache.at(key);
- ov_input_names = r_ctx->ov_input_names_cache.at(key);
- ov_output_names = r_ctx->ov_output_names_cache.at(key);
- }
+ov::Tensor get_ov_input_tensor(const std::shared_ptr<GgmlOvDecoder> & ggml_decoder, const std::string & param_name) {
+ ov::Tensor input_tensor;
+ auto extra_input = ggml_decoder->get_model_extra_inputs().find(param_name);
+ if (extra_input != ggml_decoder->get_model_extra_inputs().end()) {
+ input_tensor = ov::Tensor(extra_input->second.type, extra_input->second.shape);
+ *input_tensor.data<int64_t>() = extra_input->second.value;
+ } else {
+ input_tensor = convert_ggml_input_to_ov(ggml_decoder, param_name);
+ }
+ return input_tensor;
+}
- if (stateful) {
- const auto * inp_pos = get_inp_pos_tensor(cgraph);
- int32_t * pos_data = (int32_t *) inp_pos->data;
- auto pos_shape = ggml_decoder->get_shape(inp_pos);
- if (pos_data[0] == 0) {
- infer_request->reset_state();
- r_ctx->stateful_kv_size = pos_shape[3];
- } else if (r_ctx->stateful_kv_size == static_cast<size_t>(pos_data[0])) {
- r_ctx->stateful_kv_size += pos_shape[3];
- } else {
- const size_t pos_begin = static_cast<size_t>(pos_data[0]);
- const bool refill = pos_begin > r_ctx->stateful_kv_size;
+ov::Tensor get_ov_input_tensor_static_decode(const std::shared_ptr<GgmlOvDecoder> & ggml_decoder,
+ const std::string & param_name) {
+ // NPU decoding stage
+ if (ggml_decoder->get_model_extra_inputs().count(param_name)) {
+ return get_ov_input_tensor(ggml_decoder, param_name);
+ }
+ const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name);
+ const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor);
- // A refill seeds the state from ggml's KV cache, so it needs that cache to be a
- // plain prefix: cell i must hold position i. An SWA layer keeps only the last
- // n_swa positions, so once a position leaves the window ggml drops it and the
- // remaining cells shift - cell i stops holding position i. While every position
- // is still inside the window nothing has been dropped and the refill is sound.
- if (refill && !ggml_decoder->get_model_params().swa_layers.empty()) {
- const int n_swa = ggml_decoder->get_compute_params().swa_window;
- if (n_swa < 0 || static_cast<size_t>(n_swa) < pos_begin) {
- GGML_LOG_ERROR(
- "GGML OpenVINO backend stateful inference failed: cannot resume at position %zu from a "
- "state that holds %zu tokens, because the sliding-window layers keep only the last %d "
- "positions. Run without GGML_OPENVINO_STATEFUL_EXECUTION.\n",
- pos_begin, r_ctx->stateful_kv_size, n_swa);
- return GGML_STATUS_FAILED;
- }
- }
+ if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ||
+ GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) {
+ // IMROPE's inp_pos holds one value per t/h/w/e plane instead of a single position;
+ // with a single decode token the planes are still contiguous, so a flat copy works.
+ const int n_planes = GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ? GgmlOvDecoder::get_inp_pos_n_planes(op) : 1;
+ assert(ggml_tensor->ne[0] == n_planes);
+ ov::Shape input_shape = {1, 1, 1, (size_t) n_planes};
+ ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape);
+ std::memcpy(input_tensor.data(), ggml_tensor->data, n_planes * ggml_type_size(ggml_tensor->type));
+ return input_tensor;
+ }
- const bool relayout_enabled =
- !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT");
+ if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) {
+ ov::Shape input_shape = {1, 1, 1, 1};
+ ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape);
+ int32_t inp_out_id = *((int32_t *) ggml_tensor->data);
+ assert(ggml_tensor->ne[0] == 1);
+ assert(inp_out_id == 0);
+ *input_tensor.data<int32_t>() = inp_out_id;
+ return input_tensor;
+ }
- auto states = infer_request->query_state();
- for (auto state : states) {
- auto state_tensor = state.get_state();
- auto state_tensor_shape = state_tensor.get_shape();
+ if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) {
+ size_t context_size = ggml_decoder->get_ctx_size();
+ if (ggml_tensor->type == GGML_TYPE_F16) {
+ std::vector<ggml_fp16_t> padded_data =
+ pad_input<ggml_fp16_t>(ggml_tensor, 1, context_size, GGML_FP32_TO_FP16(-INFINITY));
+ ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, 1, context_size});
+ std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t));
+ return input_tensor;
+ }
- std::string state_name;
- if (auto it = r_ctx->kv_state_input_name_map.find(state.get_name());
- it != r_ctx->kv_state_input_name_map.end()) {
- state_name = it->second;
- }
+ std::vector<float> padded_data = pad_input<float>(ggml_tensor, 1, context_size, -INFINITY);
+ ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, 1, context_size});
+ auto * data_ptr = input_tensor.data<float>();
+ std::copy(padded_data.begin(), padded_data.begin() + context_size, data_ptr);
+ return input_tensor;
+ }
- // Which axis holds the sequence: pass::KVStateSeqAxis moves it from dim 1
- // to dim 2. The head count is still needed below, because only a 1-head
- // state stays byte-compatible with ggml's cache buffer. gemma-4 12B mixes
- // 1-head full layers with 8-head sliding layers, so it is per state.
- int n_heads_kv = ggml_decoder->get_model_params().n_heads_kv;
- if (auto layer = extract_layer_from_name(state_name); layer.has_value()) {
- n_heads_kv = ggml_decoder->get_n_heads_kv_for_layer(layer.value());
- }
- const bool relayout_this_state = relayout_enabled;
- const size_t seq_axis = relayout_this_state ? 2 : 1;
- const size_t head_axis = seq_axis == 2 ? 1 : 2;
-
- if (refill) {
- if (state_name.empty()) {
- GGML_LOG_ERROR(
- "GGML OpenVINO backend stateful inference failed: no input found for the state\n");
- return GGML_STATUS_FAILED;
- }
- auto kv_tensor = get_ov_input_tensor(ggml_decoder, state_name);
- if (relayout_this_state && n_heads_kv != 1) {
- // several heads with seq on dim 2: not the same bytes as ggml's
- // buffer, so the rows have to be copied into the new order
- state_tensor = kv_rows_to_seq_axis_2(kv_tensor, (size_t) n_heads_kv);
- } else {
- ov::Shape refill_shape(4);
- refill_shape[0] = state_tensor_shape[0];
- refill_shape[seq_axis] = kv_tensor.get_shape()[2];
- refill_shape[head_axis] = state_tensor_shape[head_axis];
- refill_shape[3] = state_tensor_shape[3];
- kv_tensor.set_shape(refill_shape);
- state_tensor = kv_tensor;
- }
- state_tensor_shape = state_tensor.get_shape();
- }
- // Only ever shrink to a prefix the source really has. Slicing past it used to
- // surface as a bare ov::Exception from the ROI constructor.
- if (state_tensor_shape[seq_axis] < pos_begin) {
- GGML_LOG_ERROR(
- "GGML OpenVINO backend stateful inference failed: state '%s' holds %zu tokens on axis "
- "%zu, cannot resume at position %zu\n",
- state.get_name().c_str(), state_tensor_shape[seq_axis], seq_axis, pos_begin);
- return GGML_STATUS_FAILED;
- }
- ov::Coordinate begin = {0, 0, 0, 0};
- ov::Coordinate end(state_tensor_shape.begin(), state_tensor_shape.end());
- end[seq_axis] = pos_begin;
- ov::Tensor new_state_tensor(state_tensor, begin, end);
- state.set_state(new_state_tensor);
- }
- r_ctx->stateful_kv_size = pos_begin + pos_shape[3];
- }
- }
+ return get_ov_input_tensor(ggml_decoder, param_name);
+}
- decoder_end_time = ggml_time_us();
- conversion_end_time = decoder_end_time;
- compile_end_time = decoder_end_time;
- } else {
- // Compilation can mutate shared weight nodes, so serialize cold paths.
- // The lock is released before binding tensors or running inference.
- auto shared_cache = r_ctx->compiled_cache;
- std::unique_lock<std::mutex> compile_lock(shared_cache->mutex);
- auto weight_names = get_weight_names(cgraph);
- ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, weight_names,
- is_static, stateful, model_is_splitted);
- const std::string shared_key = cache_enabled ? compiled_graph_key(cgraph, *ggml_decoder, device) : "";
- ov::CompiledModel shared_model;
- bool imported = false;
- auto shared_it = shared_cache->graphs.find(shared_key);
- if (!shared_key.empty() && shared_it != shared_cache->graphs.end()) {
- shared_model = shared_it->second.decode;
- infer_request = std::make_shared<ov::InferRequest>(shared_model.create_infer_request());
- ov_input_names = shared_it->second.input_names;
- ov_output_names = shared_it->second.output_names;
- imported = true;
- GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (dynamic)\n");
- }
- // Fail fast: a cache-miss recompile feeds weight data to compile_model, but
- // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU)
- // may have already dropped the host weight pages
- // (they would read as zeros). That mode requires stable graph shapes.
- if (!imported && ggml_openvino_weight_buffers_released()) {
- GGML_ABORT(
- "ggml-openvino: a new graph needs to be compiled but host weight buffers were already "
- "released via GGML_OPENVINO_RELEASE_WEIGHTS/GGML_OPENVINO_MEMORY_OPTIMIZE. This mode requires "
- "stable graph shapes; disable host weight release for dynamic workloads.");
- }
- if (cache_enabled) {
- std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
- r_ctx->infer_request_cache.erase(key);
- }
+ov::Tensor get_ov_input_tensor_static_prefill(const std::shared_ptr<GgmlOvDecoder> & ggml_decoder,
+ const std::string & param_name,
+ int chunk_index) {
+ // NPU prompt processing stage
+ const size_t input_len = ggml_decoder->get_input_len();
+ const size_t chunk_size = ggml_decoder->m_prefill_chunk_size;
+ const size_t chunk_valid_size = std::min(chunk_size, input_len - chunk_index * chunk_size);
+ const size_t chunk_pad_size = chunk_size - chunk_valid_size;
- // Frontend-level compiled-model cache (GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR): if this model
- // was compiled before, import the saved blob and skip requant + convert +
- // compile. Only the dynamic single-model path is cached (split models compile
- // two graphs and are left to the plugin-level ov::cache_dir). The decoder is
- // still needed for I/O mapping, but can be built without weight nodes since
- // the weights are baked into the imported CompiledModel.
- const std::string model_cache_dir = ggml_openvino_model_cache_dir();
- uint64_t model_fp = 0;
- std::string blob_path, manifest_path;
- // When the frontend model cache is active it supersedes the plugin-level
- // ov::cache_dir: a blob exported from a model compiled WITH cache_dir cannot
- // be re-imported (import returns an uninitialized model). Strip cache_dir /
- // cache_mode from the config used for the cached compile and the import.
- ov::AnyMap mc_config = config;
- if (!model_cache_dir.empty()) {
- mc_config.erase("CACHE_DIR");
- mc_config.erase("CACHE_MODE");
- }
- if (!imported && !model_cache_dir.empty() && !model_is_splitted) {
- const uint64_t extra_cfg = ggml_openvino_model_cache_extra_cfg(device, stateful);
- model_fp = ggml_openvino_model_fingerprint(cgraph, device, /*fa=*/true, m_params.rope_params,
- 16, extra_cfg);
- blob_path = ggml_openvino_model_cache_blob_path(model_cache_dir, model_fp);
- manifest_path = ggml_openvino_model_cache_manifest_path(model_cache_dir, model_fp);
+ if (param_name == "chunk_valid_len") {
+ ov::Tensor input_tensor(ov::element::i64, ov::Shape{1});
+ *input_tensor.data<int64_t>() = (int64_t) chunk_valid_size;
+ return input_tensor;
+ }
+ if (chunk_index > 0 && param_name == "cache_rs_reset_len") {
+ // The recurrent-state clear belongs to the start of the sequence. Re-applying it on every
+ // chunk would wipe the state accumulated by the preceding chunks, so disable it (a zero
+ // length makes scale.cpp's keep-mask select every slot) after the first chunk.
+ ov::Tensor input_tensor(ov::element::i64, ov::Shape{1});
+ *input_tensor.data<int64_t>() = 0;
+ return input_tensor;
+ }
+ if (ggml_decoder->get_model_extra_inputs().count(param_name)) {
+ return get_ov_input_tensor(ggml_decoder, param_name);
+ }
+ const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name);
+ const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor);
- std::ifstream blob_in(blob_path, std::ios::binary);
- bool blob_ok = blob_in.is_open();
- bool manifest_ok = blob_ok && ggml_openvino_model_cache_verify_manifest(manifest_path, cgraph, model_fp);
- if (blob_ok && manifest_ok) {
- int64_t import_start = ggml_time_us();
- try {
- ov::CompiledModel cm;
- auto remote_context = ggml_openvino_get_remote_context();
- if (remote_context.has_value()) {
- cm = core.import_model(blob_in, remote_context.value(), mc_config);
- } else {
- cm = core.import_model(blob_in, device, mc_config);
- }
- // Lightweight decoder: names-only weight map (membership is all the
- // decoder needs; weights live in the imported model).
- std::map<std::string, std::shared_ptr<ov::Node>> weight_names;
- for (const auto & n : GgmlOvDecoder::collect_weight_names(cgraph)) {
- weight_names[n] = nullptr;
- }
- ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, weight_names,
- is_static, stateful, model_is_splitted);
- infer_request = std::make_shared<ov::InferRequest>(cm.create_infer_request());
- shared_model = cm;
- entry->ptr = ggml_decoder;
- // Names must match the decoder's ggml-tensor keys. The non-cached
- // path keys off Parameter/Result *friendly names* (set by the
- // frontend); export_model preserves these, and each compiled-model
- // port's node is exactly that Parameter/Result. Use the port nodes
- // directly (NOT get_runtime_model(), whose graph differs and is
- // unsafe to deref this way).
- for (const auto & p : cm.inputs()) {
- ov_input_names.push_back(p.get_node()->get_friendly_name());
- }
- for (const auto & o : cm.outputs()) {
- ov_output_names.push_back(o.get_node()->get_friendly_name());
- }
- imported = true;
- if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) {
- GGML_LOG_INFO(" - Model cache import time: %.3f ms \n",
- (ggml_time_us() - import_start) / 1000.0);
- }
- GGML_LOG_INFO("ggml-openvino: model cache HIT %s\n", blob_path.c_str());
- } catch (const std::exception & e) {
- GGML_LOG_WARN("ggml-openvino: model cache import failed (%s), recompiling\n", e.what());
- imported = false;
- }
+ if (GgmlOvDecoder::is_inp_pos(ggml_tensor, op) && GgmlOvDecoder::get_inp_pos_n_planes(op) > 1) {
+ // IMROPE: inp_pos stacks n_planes (t/h/w/e) position planes, each of length
+ // input_len; pad every plane independently so they stay aligned to chunk_size.
+ const int n_planes = GgmlOvDecoder::get_inp_pos_n_planes(op);
+ const size_t element_size = ggml_type_size(ggml_tensor->type);
+ ov::Shape input_shape = {1, 1, 1, (size_t) n_planes * chunk_size};
+ ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape);
+ for (int p = 0; p < n_planes; p++) {
+ const char * src =
+ (const char *) ggml_tensor->data + (p * input_len + chunk_index * chunk_size) * element_size;
+ char * dst = (char *) input_tensor.data() + p * chunk_size * element_size;
+ std::memcpy(dst, src, chunk_valid_size * element_size);
+ if (chunk_pad_size > 0) {
+ if (ggml_tensor->type == GGML_TYPE_I32) {
+ int32_t last_value = *((const int32_t *) src + chunk_valid_size - 1);
+ int32_t * out = (int32_t *) dst;
+ std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1);
+ } else if (ggml_tensor->type == GGML_TYPE_I64) {
+ int64_t last_value = *((const int64_t *) src + chunk_valid_size - 1);
+ int64_t * out = (int64_t *) dst;
+ std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1);
+ } else {
+ throw std::runtime_error("Unexpected tensor type for " + param_name);
}
}
+ }
+ return input_tensor;
+ }
- std::shared_ptr<ov::Model> model;
- if (imported) {
- decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us();
+ if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ||
+ GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) {
+ ov::Shape input_shape = {1, 1, 1, chunk_size};
+ ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape);
+ // copy the chunk_index-th chunk from ggml_tensor
+ size_t element_size = ggml_type_size(ggml_tensor->type);
+ void * input_data = (char *) ggml_tensor->data + chunk_index * chunk_size * element_size;
+ std::memcpy(input_tensor.data(), input_data, chunk_valid_size * element_size);
+ // pad the rest with last_value + 1, so that kv's of padded positions are inserted
+ // to the next row after the valids row in the kvcache
+ if (chunk_pad_size > 0) {
+ if (ggml_tensor->type == GGML_TYPE_I32) {
+ int32_t last_value =
+ *((int32_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1));
+ int32_t * output_data = input_tensor.data<int32_t>();
+ std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1);
+ } else if (ggml_tensor->type == GGML_TYPE_I64) {
+ int64_t last_value =
+ *((int64_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1));
+ int64_t * output_data = input_tensor.data<int64_t>();
+ std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1);
} else {
- auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph);
-
- ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights, is_static,
- stateful, model_is_splitted);
- decoder_end_time = ggml_time_us();
-
- auto input_model = std::make_shared<ov::frontend::ggml::InputModel>(ggml_decoder);
- model = ov::frontend::ggml::FrontEnd::convert(input_model);
- ggml_decoder->clear_model_weights();
- conversion_end_time = ggml_time_us();
-
- if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) {
- char timestamped_filename[64];
- auto timestamp = (long long) ggml_time_us();
- snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%lld.xml", timestamp);
- ov::serialize(model, timestamped_filename);
- }
-
- // Use the cache-stripped config when the frontend model cache is active, so
- // the resulting CompiledModel can be exported and later re-imported.
- const ov::AnyMap & compile_config = model_cache_dir.empty() ? config : mc_config;
- ov::CompiledModel compiled_model;
- auto remote_context = ggml_openvino_get_remote_context();
- if (remote_context.has_value()) {
- compiled_model = core.compile_model(model, remote_context.value(), compile_config);
- } else {
- compiled_model = core.compile_model(model, device, compile_config);
- }
- compile_end_time = ggml_time_us();
-
- // Export to the frontend model cache for next time. Publish the blob first,
- // then the manifest, so a cache hit only sees fully written artifacts.
- if (!model_cache_dir.empty() && !model_is_splitted && model_fp != 0) {
- try {
- const std::string blob_tmp = blob_path + ".tmp";
- const std::string manifest_tmp = manifest_path + ".tmp";
- if (ggml_openvino_model_cache_write_manifest(manifest_tmp, cgraph, model_fp)) {
- std::ofstream blob_out(blob_tmp, std::ios::binary | std::ios::trunc);
- if (blob_out.is_open()) {
- compiled_model.export_model(blob_out);
- blob_out.close();
- if (blob_out.good()) {
- if (std::rename(blob_tmp.c_str(), blob_path.c_str()) == 0 &&
- std::rename(manifest_tmp.c_str(), manifest_path.c_str()) == 0) {
- GGML_LOG_INFO("ggml-openvino: model cache WROTE %s\n", blob_path.c_str());
- } else {
- std::remove(blob_tmp.c_str());
- std::remove(manifest_tmp.c_str());
- }
- } else {
- std::remove(blob_tmp.c_str());
- std::remove(manifest_tmp.c_str());
- }
- } else {
- std::remove(manifest_tmp.c_str());
- }
- }
- } catch (const std::exception & e) {
- GGML_LOG_WARN("ggml-openvino: model cache export failed: %s\n", e.what());
- }
- }
-
- infer_request = std::make_shared<ov::InferRequest>(compiled_model.create_infer_request());
- shared_model = compiled_model;
- entry->ptr = ggml_decoder;
-
- for (const auto & ov_param : model->get_parameters()) {
- ov_input_names.push_back(ov_param->get_friendly_name());
- }
- for (const auto & ov_output : model->get_results()) {
- ov_output_names.push_back(ov_output->get_friendly_name());
- }
- } // end non-imported (compile) path
-
- entry->ptr = ggml_decoder;
- if (!shared_key.empty() && shared_it == shared_cache->graphs.end()) {
- shared_cache->graphs.emplace(shared_key, ov_compiled_graph{shared_model, {}, ov_input_names,
- ov_output_names});
- }
- if (cache_enabled) {
- std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
- r_ctx->infer_request_cache[key] = infer_request;
- r_ctx->ov_input_names_cache[key] = ov_input_names;
- r_ctx->ov_output_names_cache[key] = ov_output_names;
+ throw std::runtime_error("Unexpected tensor type for " + param_name);
}
+ }
+ return input_tensor;
+ }
- if (stateful && cache_enabled) {
- const auto * inp_pos = get_inp_pos_tensor(cgraph);
- auto pos_shape = ggml_decoder->get_shape(inp_pos);
- // A freshly compiled model starts with an empty state, so it can only serve a
- // sequence from its beginning. A non-zero start position means the KV history was
- // built elsewhere (a restored ggml cache), which the state cannot adopt.
- const int32_t pos_begin = ((int32_t *) inp_pos->data)[0];
- if (pos_begin != 0) {
- GGML_LOG_ERROR(
- "GGML OpenVINO backend stateful inference failed: a new model was compiled for a sequence that "
- "starts at position %d, but its state is empty. Run without "
- "GGML_OPENVINO_STATEFUL_EXECUTION.\n",
- pos_begin);
- return GGML_STATUS_FAILED;
- }
- r_ctx->stateful_kv_size = pos_shape[3];
- const auto kv_param_res_names = ggml_decoder->get_kv_param_res_names();
- for (const auto & pair : kv_param_res_names) {
- r_ctx->kv_state_input_name_map[pair.first + pair.second] = pair.first;
- }
+ if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) {
+ size_t output_len = ggml_decoder->get_compute_params().output_len;
+ ov::Shape input_shape = {1, 1, 1, output_len};
+ ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape);
+ if (ggml_tensor->ne[0] == 0) {
+ *input_tensor.data<int32_t>() = 0;
+ } else {
+ auto * data_addr = input_tensor.data<int32_t>();
+ for (size_t i = 0; i < output_len; i++) {
+ data_addr[i] = ((int32_t *) ggml_tensor->data)[i] % chunk_size;
}
}
+ return input_tensor;
+ }
- for (size_t i = 0; i < ov_input_names.size(); i++) {
- auto param_name = ov_input_names[i];
- auto input_tensor = get_ov_input_tensor(ggml_decoder, param_name);
- infer_request->set_input_tensor(i, input_tensor);
-
- if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) {
- print_input_tensor_info(param_name, input_tensor);
- }
+ if (GgmlOvDecoder::is_inp_mean(ggml_tensor, op)) {
+ const size_t n_seqs = ggml_tensor->ne[1];
+ const size_t src_stride = ggml_tensor->ne[0];
+ const size_t copy_len = std::min<size_t>(chunk_valid_size, src_stride - chunk_index * chunk_size);
+ ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, n_seqs, chunk_size});
+ auto * dst = input_tensor.data<float>();
+ std::fill(dst, dst + n_seqs * chunk_size, 0.0f);
+ const auto * src = static_cast<const float *>(ggml_tensor->data) + chunk_index * chunk_size;
+ for (size_t s = 0; s < n_seqs; s++) {
+ std::memcpy(dst + s * chunk_size, src + s * src_stride, copy_len * sizeof(float));
}
+ return input_tensor;
+ }
- for (size_t i = 0; i < ov_output_names.size(); i++) {
- // Debug-only outputs added via GGML_OPENVINO_DEBUG_NODE (see
- // translate_session.cpp) have no corresponding ggml tensor; leave
- // them unbound so OpenVINO allocates its own tensor for them,
- // rather than aliasing a ggml buffer that may be overwritten by a
- // later in-place op before we get to read it.
- const auto & model_outputs = ggml_decoder->get_model_outputs();
- auto model_output_it = model_outputs.find(ov_output_names[i]);
- if (model_output_it == model_outputs.end()) {
- continue;
- }
- auto * ggml_tensor = model_output_it->second;
- if (ggml_nbytes(ggml_tensor) == 0) {
- continue;
- }
- auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor);
- infer_request->set_output_tensor(i, output_tensor);
+ if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) {
+ size_t cols = ggml_tensor->ne[0];
+ size_t rows = ggml_tensor->ne[1];
+ size_t chunk_valid_rows = std::min(chunk_size, rows - chunk_index * chunk_size);
+ size_t context_size = ggml_decoder->get_ctx_size();
+ if (ggml_tensor->type == GGML_TYPE_F16) {
+ const auto * ggml_data =
+ static_cast<const ggml_fp16_t *>(ggml_tensor->data) + chunk_index * chunk_size * cols;
+ std::vector<ggml_fp16_t> padded_data = pad_input<ggml_fp16_t>(ggml_data, chunk_valid_rows, cols, chunk_size,
+ context_size, GGML_FP32_TO_FP16(-INFINITY));
+ set_zero_diagonal(padded_data, chunk_size, context_size, GGML_FP32_TO_FP16(0.0f));
+ ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, chunk_size, context_size});
+ std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t));
+ return input_tensor;
}
- ov_raw_infer_start = ggml_time_us();
- infer_request->infer();
- infer_end_time = ggml_time_us();
+ const auto * ggml_data = static_cast<const float *>(ggml_tensor->data) + chunk_index * chunk_size * cols;
+ std::vector<float> padded_data =
+ pad_input<float>(ggml_data, chunk_valid_rows, cols, chunk_size, context_size, -INFINITY);
+ set_zero_diagonal(padded_data, chunk_size, context_size);
+ ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, chunk_size, context_size});
+ auto * data_ptr = input_tensor.data<float>();
+ std::copy(padded_data.begin(), padded_data.begin() + chunk_size * context_size, data_ptr);
+ return input_tensor;
+ }
- if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") ||
- ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) {
- for (size_t i = 0; i < ov_output_names.size(); i++) {
- const auto output_tensor = infer_request->get_output_tensor(i);
- print_output_tensor_info(ov_output_names[i], output_tensor, output_tensor.data());
- }
- }
+ return get_ov_input_tensor(ggml_decoder, param_name);
+}
- if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) {
- GGML_LOG_INFO("\nGGML OpenVINO Backend: \n");
- GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0);
- if (!cache_hit) {
- GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n",
- (conversion_end_time - decoder_end_time) / 1000.0);
- GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0);
- }
- GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0);
- GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", (infer_end_time - ov_raw_infer_start) / 1000.0);
- }
+enum ggml_status naive_compute(ggml_cgraph * cgraph,
+ ov::Core & core,
+ const std::string & device,
+ const ov::AnyMap & config,
+ ov_compiled_model_cache & cache) {
+ if (cgraph->n_nodes == 1 && (cgraph->nodes[0]->op == GGML_OP_NONE || cgraph->nodes[0]->op == GGML_OP_VIEW)) {
+ return GGML_STATUS_SUCCESS;
}
- // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU): the plugin holds its own device copy of
- // every weight after compile, so the host weight buffers can be dropped to reclaim
- // RSS. Release only while holding the compilation mutex so another context cannot
- // be reading host weights during conversion/compilation. Pin the shared compiled
- // models across backend teardown; a later context can create its own request without
- // reading the dropped pages. A new, uncached graph still fails fast above.
- if (cache_hit && ggml_openvino_release_weights_enabled(device)) {
- std::lock_guard<std::mutex> compile_lock(r_ctx->compiled_cache->mutex);
- if (!ggml_openvino_weight_buffers_released()) {
- ggml_openvino_release_weight_buffers();
- }
+ std::unique_lock<std::mutex> compile_lock(cache.mutex);
+ bool naive = true;
+ auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, naive);
+ auto decoder = std::make_shared<GgmlOvDecoder>(cgraph, model_weights);
+ auto input_model = std::make_shared<ov::frontend::ggml::InputModel>(decoder);
+ auto model = ov::frontend::ggml::FrontEnd::convert(input_model, naive);
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) {
+ ov::serialize(model, "IR_naive.xml");
}
- return GGML_STATUS_SUCCESS;
-}
+ std::shared_ptr<ov::InferRequest> infer_request;
+ auto remote_context = ggml_openvino_get_remote_context();
+ ov::AnyMap compile_config = config;
+ if (cgraph->nodes[0]->op == GGML_OP_MUL_MAT) {
+ // TODO ACCURACY hint triggers a bug in GPU plugin/driver on Lunar Lake. Remove once CVS-182166 is resolved
+ compile_config[ov::hint::execution_mode.name()] = ov::hint::ExecutionMode::PERFORMANCE;
+ } else {
+ compile_config[ov::hint::execution_mode.name()] = ov::hint::ExecutionMode::ACCURACY;
+ }
+ if (remote_context.has_value()) {
+ infer_request = std::make_shared<ov::InferRequest>(
+ core.compile_model(model, remote_context.value(), compile_config).create_infer_request());
+ } else {
+ infer_request = std::make_shared<ov::InferRequest>(
+ core.compile_model(model, device, compile_config).create_infer_request());
+ }
+ std::vector<std::string> input_names;
+ std::vector<std::string> output_names;
+ for (const auto & param : model->get_parameters()) {
+ input_names.push_back(param->get_friendly_name());
+ }
+ for (const auto & result : model->get_results()) {
+ output_names.push_back(result->get_friendly_name());
+ }
+ // Destroy the frontend graph under the compilation lock as well: it can
+ // still own edges into the shared weight nodes.
+ model.reset();
+ input_model.reset();
+ decoder->clear_model_weights();
+ model_weights.clear();
+ compile_lock.unlock();
-static ov::AnyMap without_npuw(const ov::AnyMap & config) {
- ov::AnyMap out;
- for (const auto & kv : config) {
- if (kv.first.rfind("NPUW", 0) == 0 || kv.first == "NPU_USE_NPUW") {
+ for (size_t i = 0; i < input_names.size(); i++) {
+ const auto & param_name = input_names[i];
+ auto input_tensor = get_ov_input_tensor(decoder, param_name);
+ infer_request->set_input_tensor(i, input_tensor);
+ }
+
+ // Use get_output_tensor + memcpy instead of set_output_tensor to avoid memory overwritten
+ // when i/o buffer overlaps, e.g. the cgraph is a single PERMUTE
+
+ infer_request->infer();
+
+ for (size_t i = 0; i < output_names.size(); i++) {
+ auto output_tensor = infer_request->get_output_tensor(i);
+ const auto & model_outputs = decoder->get_model_outputs();
+ auto model_output_it = model_outputs.find(output_names[i]);
+ if (model_output_it == model_outputs.end()) {
+ // Debug-only output added via GGML_OPENVINO_DEBUG_NODE; nothing to copy into.
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") ||
+ ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) {
+ print_output_tensor_info(output_names[i], output_tensor, output_tensor.data());
+ }
continue;
}
- out.insert(kv);
+ auto * ggml_tensor = model_output_it->second;
+ std::memcpy(ggml_tensor->data, output_tensor.data(), output_tensor.get_byte_size());
}
- return out;
+ return GGML_STATUS_SUCCESS;
}
-enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<ov_runtime_context> r_ctx) {
+enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::shared_ptr<ov_runtime_context> & r_ctx) {
auto & core = ov_singleton_core();
+ const auto & config = ggml_openvino_get_compile_config();
+ const auto & device = r_ctx->device;
+ const auto & stateful = r_ctx->stateful;
+ static auto is_static = false;
- auto get_prefill_chunk_size = [] {
- static const int chunk_size = []() {
- int env_prefill_chunk_size = ggml_openvino_getenv_int("GGML_OPENVINO_PREFILL_CHUNK_SIZE");
- return env_prefill_chunk_size > 0 ? env_prefill_chunk_size : 256;
- }();
- return chunk_size;
- };
+ static const bool cache_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE");
- // Normally NPU, but honors GGML_OPENVINO_DEVICE so GGML_OPENVINO_FORCE_STATIC can run the
- // static-shape path on CPU/GPU to isolate translation bugs from NPUW/NPU-driver issues.
- static std::string device = ggml_openvino_get_device_name();
- static auto is_static = true;
- static auto stateful = false;
+ // is_model_splitted is O(n_nodes^2) plus a create_weight_nodes scan and takes ~20 ms
+ // on a Llama-1B decode graph. It is called once per graph_compute invocation but the
+ // graph shape is identical across all decode steps, so memoize by graph_key: compute
+ // graph_key first (a few hundred us), and if the same key is already in decoder_cache
+ // we know the graph is not splitted (only not-splitted graphs get inserted there).
+ graph_key key(cgraph);
+ bool key_seen = false;
+ if (!cache_disabled) {
+ std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
+ key_seen = r_ctx->decoder_cache.find(key) != r_ctx->decoder_cache.end();
+ }
- auto prefill_chunk_size = get_prefill_chunk_size();
- const auto & config = ggml_openvino_get_compile_config();
+ bool model_is_splitted = key_seen ? false : is_model_splitted(cgraph);
if (is_naive(cgraph)) {
- return naive_compute(cgraph, core, device, config, *r_ctx->compiled_cache);
+ if (!model_is_splitted) {
+ return naive_compute(cgraph, core, device, config, *r_ctx->compiled_cache);
+ }
}
auto start_time = ggml_time_us();
@@ -835,749 +665,913 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
ComputeParams c_params;
std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static);
- const auto * inp_pos = get_inp_pos_tensor(cgraph);
- const bool no_kv_cache = m_params.is_cacheless_attn;
- const auto is_prefill = no_kv_cache ? true : get_is_prefill(cgraph, inp_pos);
- const ov::AnyMap compile_config = no_kv_cache ? without_npuw(config) : config;
- if (m_params.n_heads_kv == -1) {
- prefill_chunk_size = inp_pos->ne[0];
- }
- graph_key key(cgraph);
- static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE");
+ const bool cache_enabled = !model_is_splitted && !cache_disabled;
bool cache_hit = false;
- int64_t decoder_end_time;
- int64_t conversion_end_time;
- int64_t compile_end_time;
- int64_t infer_end_time;
- int64_t ov_raw_infer_start;
- int64_t ov_raw_infer_total = 0;
+ int64_t decoder_end_time;
+ int64_t conversion_end_time;
+ int64_t compile_end_time;
+ int64_t infer_end_time;
+ int64_t ov_raw_infer_start;
+
+ {
+ std::shared_ptr<decoder_runtime_ctx> entry;
+ ModelParams old_m_params;
+
+ if (cache_enabled) {
+ std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
+ auto it = r_ctx->decoder_cache.find(key);
+ cache_hit = it != r_ctx->decoder_cache.end();
+ if (cache_hit) {
+ entry = it->second;
+ } else {
+ r_ctx->clear_caches_locked();
+ auto mutex = std::make_shared<std::mutex>();
+ entry = std::make_shared<decoder_runtime_ctx>(mutex);
+ r_ctx->decoder_cache[key] = entry;
+ }
+ } else {
+ auto mutex = std::make_shared<std::mutex>();
+ entry = std::make_shared<decoder_runtime_ctx>(mutex);
+ cache_hit = false;
+ }
+
+ std::lock_guard<std::mutex> lock(*(entry->mutex));
+ cache_hit = cache_hit && entry->ptr && r_ctx->infer_request_cache.count(key) != 0;
+
+ if (cache_hit) {
+ ggml_decoder = entry->ptr;
+ old_m_params = ggml_decoder->get_model_params();
+ if (!ggml_decoder->is_splited_model()) {
+ cache_hit = old_m_params.can_reuse_dynamically(m_params);
+ }
+ }
+
+ std::vector<std::string> ov_input_names;
+ std::vector<std::string> ov_output_names;
+
+ if (cache_hit) {
+ std::map<std::string, std::shared_ptr<ov::Node>> model_weights;
+ ggml_decoder->set_compute_params(c_params);
+ ggml_decoder->set_model_params(m_params);
+ if (old_m_params.kv_buffer_changed(m_params)) {
+ ggml_decoder->update_io(cgraph);
+ }
+ ggml_decoder->add_extra_inputs();
+ {
+ std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
+ infer_request = r_ctx->infer_request_cache.at(key);
+ ov_input_names = r_ctx->ov_input_names_cache.at(key);
+ ov_output_names = r_ctx->ov_output_names_cache.at(key);
+ }
+
+ if (stateful) {
+ const auto * inp_pos = get_inp_pos_tensor(cgraph);
+ int32_t * pos_data = (int32_t *) inp_pos->data;
+ auto pos_shape = GgmlOvDecoder::get_shape(inp_pos);
+ if (pos_data[0] == 0) {
+ infer_request->reset_state();
+ r_ctx->stateful_kv_size = pos_shape[3];
+ } else if (r_ctx->stateful_kv_size == static_cast<size_t>(pos_data[0])) {
+ r_ctx->stateful_kv_size += pos_shape[3];
+ } else {
+ const size_t pos_begin = static_cast<size_t>(pos_data[0]);
+ const bool refill = pos_begin > r_ctx->stateful_kv_size;
+
+ // A refill seeds the state from ggml's KV cache, so it needs that cache to be a
+ // plain prefix: cell i must hold position i. An SWA layer keeps only the last
+ // n_swa positions, so once a position leaves the window ggml drops it and the
+ // remaining cells shift - cell i stops holding position i. While every position
+ // is still inside the window nothing has been dropped and the refill is sound.
+ if (refill && !ggml_decoder->get_model_params().swa_layers.empty()) {
+ const int n_swa = ggml_decoder->get_compute_params().swa_window;
+ if (n_swa < 0 || static_cast<size_t>(n_swa) < pos_begin) {
+ GGML_LOG_ERROR(
+ "GGML OpenVINO backend stateful inference failed: cannot resume at position %zu from a "
+ "state that holds %zu tokens, because the sliding-window layers keep only the last %d "
+ "positions. Run without GGML_OPENVINO_STATEFUL_EXECUTION.\n",
+ pos_begin, r_ctx->stateful_kv_size, n_swa);
+ return GGML_STATUS_FAILED;
+ }
+ }
+
+ const bool relayout_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT");
- std::shared_ptr<decoder_runtime_ctx> entry;
- ModelParams old_m_params;
+ auto states = infer_request->query_state();
+ for (auto state : states) {
+ auto state_tensor = state.get_state();
+ auto state_tensor_shape = state_tensor.get_shape();
- if (cache_enabled) {
- std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
- auto it = r_ctx->decoder_cache.find(key);
- cache_hit = it != r_ctx->decoder_cache.end();
- if (cache_hit) {
- entry = it->second;
- } else {
- r_ctx->clear_caches_locked();
- auto mutex = std::make_shared<std::mutex>();
- entry = std::make_shared<decoder_runtime_ctx>(mutex);
- r_ctx->decoder_cache[key] = entry;
- }
- } else {
- auto mutex = std::make_shared<std::mutex>();
- entry = std::make_shared<decoder_runtime_ctx>(mutex);
- cache_hit = false;
- }
+ std::string state_name;
+ if (auto it = r_ctx->kv_state_input_name_map.find(state.get_name());
+ it != r_ctx->kv_state_input_name_map.end()) {
+ state_name = it->second;
+ }
- std::lock_guard<std::mutex> lock(*(entry->mutex));
- cache_hit = cache_hit && entry->ptr && r_ctx->infer_request_cache.count(key) != 0 &&
- r_ctx->infer_request_cache_prefill.count(key) != 0;
+ // Which axis holds the sequence: pass::KVStateSeqAxis moves it from dim 1
+ // to dim 2. The head count is still needed below, because only a 1-head
+ // state stays byte-compatible with ggml's cache buffer. gemma-4 12B mixes
+ // 1-head full layers with 8-head sliding layers, so it is per state.
+ int n_heads_kv = ggml_decoder->get_model_params().n_heads_kv;
+ if (auto layer = extract_layer_from_name(state_name); layer.has_value()) {
+ n_heads_kv = ggml_decoder->get_n_heads_kv_for_layer(layer.value());
+ }
+ const bool relayout_this_state = relayout_enabled;
+ const size_t seq_axis = relayout_this_state ? 2 : 1;
+ const size_t head_axis = seq_axis == 2 ? 1 : 2;
- if (cache_hit) {
- ggml_decoder = entry->ptr;
- old_m_params = ggml_decoder->get_model_params();
- cache_hit = old_m_params.can_reuse_statically(m_params);
- }
+ if (refill) {
+ if (state_name.empty()) {
+ GGML_LOG_ERROR(
+ "GGML OpenVINO backend stateful inference failed: no input found for the state\n");
+ return GGML_STATUS_FAILED;
+ }
+ auto kv_tensor = get_ov_input_tensor(ggml_decoder, state_name);
+ if (relayout_this_state && n_heads_kv != 1) {
+ // several heads with seq on dim 2: not the same bytes as ggml's
+ // buffer, so the rows have to be copied into the new order
+ state_tensor = kv_rows_to_seq_axis_2(kv_tensor, (size_t) n_heads_kv);
+ } else {
+ ov::Shape refill_shape(4);
+ refill_shape[0] = state_tensor_shape[0];
+ refill_shape[seq_axis] = kv_tensor.get_shape()[2];
+ refill_shape[head_axis] = state_tensor_shape[head_axis];
+ refill_shape[3] = state_tensor_shape[3];
+ kv_tensor.set_shape(refill_shape);
+ state_tensor = kv_tensor;
+ }
+ state_tensor_shape = state_tensor.get_shape();
+ }
+ // Only ever shrink to a prefix the source really has. Slicing past it used to
+ // surface as a bare ov::Exception from the ROI constructor.
+ if (state_tensor_shape[seq_axis] < pos_begin) {
+ GGML_LOG_ERROR(
+ "GGML OpenVINO backend stateful inference failed: state '%s' holds %zu tokens on axis "
+ "%zu, cannot resume at position %zu\n",
+ state.get_name().c_str(), state_tensor_shape[seq_axis], seq_axis, pos_begin);
+ return GGML_STATUS_FAILED;
+ }
+ ov::Coordinate begin = {0, 0, 0, 0};
+ ov::Coordinate end(state_tensor_shape.begin(), state_tensor_shape.end());
+ end[seq_axis] = pos_begin;
+ ov::Tensor new_state_tensor(state_tensor, begin, end);
+ state.set_state(new_state_tensor);
+ }
+ r_ctx->stateful_kv_size = pos_begin + pos_shape[3];
+ }
+ }
- std::vector<std::string> ov_input_names_local;
- std::vector<std::string> ov_output_names_local;
+ decoder_end_time = ggml_time_us();
+ conversion_end_time = decoder_end_time;
+ compile_end_time = decoder_end_time;
+ } else {
+ // Compilation can mutate shared weight nodes, so serialize cold paths.
+ // The lock is released before binding tensors or running inference.
+ auto shared_cache = r_ctx->compiled_cache;
+ std::unique_lock<std::mutex> compile_lock(shared_cache->mutex);
+ auto weight_names = get_weight_names(cgraph);
+ ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, weight_names, is_static,
+ stateful, model_is_splitted);
+ const std::string shared_key = cache_enabled ? compiled_graph_key(cgraph, *ggml_decoder, device) : "";
+ ov::CompiledModel shared_model;
+ bool imported = false;
+ auto shared_it = shared_cache->graphs.find(shared_key);
+ if (!shared_key.empty() && shared_it != shared_cache->graphs.end()) {
+ shared_model = shared_it->second.decode;
+ infer_request = std::make_shared<ov::InferRequest>(shared_model.create_infer_request());
+ ov_input_names = shared_it->second.input_names;
+ ov_output_names = shared_it->second.output_names;
+ imported = true;
+ GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (dynamic)\n");
+ }
+ // Fail fast: a cache-miss recompile feeds weight data to compile_model, but
+ // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU)
+ // may have already dropped the host weight pages
+ // (they would read as zeros). That mode requires stable graph shapes.
+ if (!imported && ggml_openvino_weight_buffers_released()) {
+ GGML_ABORT(
+ "ggml-openvino: a new graph needs to be compiled but host weight buffers were already "
+ "released via GGML_OPENVINO_RELEASE_WEIGHTS/GGML_OPENVINO_MEMORY_OPTIMIZE. This mode requires "
+ "stable graph shapes; disable host weight release for dynamic workloads.");
+ }
+ if (cache_enabled) {
+ std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
+ r_ctx->infer_request_cache.erase(key);
+ }
- if (cache_hit) {
- std::map<std::string, std::shared_ptr<ov::Node>> model_weights;
- ggml_decoder->m_is_prefill = is_prefill;
- ggml_decoder->set_model_params(m_params);
- ggml_decoder->set_compute_params(c_params);
- if (old_m_params.kv_buffer_changed(m_params)) {
- ggml_decoder->update_io(cgraph);
- }
- ggml_decoder->add_extra_inputs();
- {
- std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
- infer_request =
- is_prefill ? r_ctx->infer_request_cache_prefill.at(key) : r_ctx->infer_request_cache.at(key);
- ov_input_names_local = r_ctx->ov_input_names_cache.at(key);
- ov_output_names_local = r_ctx->ov_output_names_cache.at(key);
- }
+ // Frontend-level compiled-model cache (GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR): if this model
+ // was compiled before, import the saved blob and skip requant + convert +
+ // compile. Only the dynamic single-model path is cached (split models compile
+ // two graphs and are left to the plugin-level ov::cache_dir). The decoder is
+ // still needed for I/O mapping, but can be built without weight nodes since
+ // the weights are baked into the imported CompiledModel.
+ const std::string model_cache_dir = ggml_openvino_model_cache_dir();
+ uint64_t model_fp = 0;
+ std::string blob_path;
+ std::string manifest_path;
+ // When the frontend model cache is active it supersedes the plugin-level
+ // ov::cache_dir: a blob exported from a model compiled WITH cache_dir cannot
+ // be re-imported (import returns an uninitialized model). Strip cache_dir /
+ // cache_mode from the config used for the cached compile and the import.
+ ov::AnyMap mc_config = config;
+ if (!model_cache_dir.empty()) {
+ mc_config.erase("CACHE_DIR");
+ mc_config.erase("CACHE_MODE");
+ }
+ if (!imported && !model_cache_dir.empty() && !model_is_splitted) {
+ const uint64_t extra_cfg = ggml_openvino_model_cache_extra_cfg(device, stateful);
+ model_fp =
+ ggml_openvino_model_fingerprint(cgraph, device, /*fa=*/true, m_params.rope_params, 16, extra_cfg);
+ blob_path = ggml_openvino_model_cache_blob_path(model_cache_dir, model_fp);
+ manifest_path = ggml_openvino_model_cache_manifest_path(model_cache_dir, model_fp);
- decoder_end_time = ggml_time_us();
- conversion_end_time = decoder_end_time;
- compile_end_time = decoder_end_time;
- } else {
- if (cache_enabled) {
- std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
- r_ctx->infer_request_cache.erase(key);
- r_ctx->infer_request_cache_prefill.erase(key);
- }
+ std::ifstream blob_in(blob_path, std::ios::binary);
+ bool blob_ok = blob_in.is_open();
+ bool manifest_ok =
+ blob_ok && ggml_openvino_model_cache_verify_manifest(manifest_path, cgraph, model_fp);
+ if (blob_ok && manifest_ok) {
+ int64_t import_start = ggml_time_us();
+ try {
+ ov::CompiledModel cm;
+ auto remote_context = ggml_openvino_get_remote_context();
+ if (remote_context.has_value()) {
+ cm = core.import_model(blob_in, remote_context.value(), mc_config);
+ } else {
+ cm = core.import_model(blob_in, device, mc_config);
+ }
+ // Lightweight decoder: names-only weight map (membership is all the
+ // decoder needs; weights live in the imported model).
+ std::map<std::string, std::shared_ptr<ov::Node>> weight_names;
+ for (const auto & n : GgmlOvDecoder::collect_weight_names(cgraph)) {
+ weight_names[n] = nullptr;
+ }
+ ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, weight_names,
+ is_static, stateful, model_is_splitted);
+ infer_request = std::make_shared<ov::InferRequest>(cm.create_infer_request());
+ shared_model = cm;
+ entry->ptr = ggml_decoder;
+ // Names must match the decoder's ggml-tensor keys. The non-cached
+ // path keys off Parameter/Result *friendly names* (set by the
+ // frontend); export_model preserves these, and each compiled-model
+ // port's node is exactly that Parameter/Result. Use the port nodes
+ // directly (NOT get_runtime_model(), whose graph differs and is
+ // unsafe to deref this way).
+ for (const auto & p : cm.inputs()) {
+ ov_input_names.push_back(p.get_node()->get_friendly_name());
+ }
+ for (const auto & o : cm.outputs()) {
+ ov_output_names.push_back(o.get_node()->get_friendly_name());
+ }
+ imported = true;
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) {
+ GGML_LOG_INFO(" - Model cache import time: %.3f ms \n",
+ (ggml_time_us() - import_start) / 1000.0);
+ }
+ GGML_LOG_INFO("ggml-openvino: model cache HIT %s\n", blob_path.c_str());
+ } catch (const std::exception & e) {
+ GGML_LOG_WARN("ggml-openvino: model cache import failed (%s), recompiling\n", e.what());
+ imported = false;
+ }
+ }
+ }
- // Static execution shares a compiled prefill/decode pair. Each backend
- // creates and retains its own requests for both phases.
- auto shared_cache = r_ctx->compiled_cache;
- std::unique_lock<std::mutex> compile_lock(shared_cache->mutex);
- auto weight_names = get_weight_names(cgraph);
- auto local_decoder = std::make_shared<GgmlOvDecoder>(
- cgraph, m_params, c_params, weight_names, is_static, stateful, false, is_prefill, prefill_chunk_size);
- const std::string shared_key = cache_enabled ?
- compiled_graph_key(cgraph, *local_decoder, device, prefill_chunk_size) : "";
- auto shared_it = shared_cache->graphs.find(shared_key);
- if (!shared_key.empty() && shared_it != shared_cache->graphs.end()) {
- auto & compiled = shared_it->second;
- auto prefill_request = std::make_shared<ov::InferRequest>(compiled.prefill.create_infer_request());
- auto decode_request = no_kv_cache ? prefill_request :
- std::make_shared<ov::InferRequest>(compiled.decode.create_infer_request());
- ggml_decoder = local_decoder;
- entry->ptr = ggml_decoder;
- infer_request = is_prefill ? prefill_request : decode_request;
- ov_input_names_local = compiled.input_names;
- ov_output_names_local = compiled.output_names;
- r_ctx->infer_request_cache_prefill[key] = prefill_request;
- r_ctx->infer_request_cache[key] = decode_request;
- r_ctx->ov_input_names_cache[key] = ov_input_names_local;
- r_ctx->ov_output_names_cache[key] = ov_output_names_local;
- decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us();
- GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (static)\n");
- } else {
std::shared_ptr<ov::Model> model;
- auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph);
-
- auto ggml_decoder_prefill = std::make_shared<GgmlOvDecoder>(
- cgraph, m_params, c_params, model_weights, is_static, stateful, false, true, prefill_chunk_size);
- auto ggml_decoder_decode =
- no_kv_cache ? ggml_decoder_prefill :
- std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights, is_static,
- stateful, false, false, prefill_chunk_size);
- decoder_end_time = ggml_time_us();
+ if (imported) {
+ decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us();
+ } else {
+ auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph);
- const bool dump_ir = ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR");
- const auto dump_ir_timestamp = static_cast<long long>(ggml_time_us());
+ ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights, is_static,
+ stateful, model_is_splitted);
+ decoder_end_time = ggml_time_us();
- auto build_static_model = [&core, &compile_config, dump_ir, dump_ir_timestamp](
- std::shared_ptr<GgmlOvDecoder> decoder,
- const char * tag,
- std::shared_ptr<ov::Model> & model,
- ov::CompiledModel & compiled_model,
- std::shared_ptr<ov::InferRequest> & infer_request,
- int64_t & local_conversion_end_time,
- int64_t & local_compile_end_time) {
- auto input_model = std::make_shared<ov::frontend::ggml::InputModel>(decoder);
+ auto input_model = std::make_shared<ov::frontend::ggml::InputModel>(ggml_decoder);
model = ov::frontend::ggml::FrontEnd::convert(input_model);
- decoder->clear_model_weights();
- local_conversion_end_time = ggml_time_us();
+ ggml_decoder->clear_model_weights();
+ conversion_end_time = ggml_time_us();
- if (dump_ir) {
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) {
char timestamped_filename[64];
- snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%s_%lld.xml", tag,
- dump_ir_timestamp);
+ auto timestamp = (long long) ggml_time_us();
+ snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%lld.xml", timestamp);
ov::serialize(model, timestamped_filename);
}
- compiled_model = core.compile_model(model, device, compile_config);
- infer_request = std::make_shared<ov::InferRequest>(compiled_model.create_infer_request());
- local_compile_end_time = ggml_time_us();
- };
- std::shared_ptr<ov::Model> model_prefill;
- std::shared_ptr<ov::Model> model_decode;
- ov::CompiledModel compiled_model_prefill;
- ov::CompiledModel compiled_model_decode;
- std::shared_ptr<ov::InferRequest> infer_request_prefill;
- std::shared_ptr<ov::InferRequest> infer_request_decode;
- int64_t prefill_conversion_end_time;
- int64_t decode_conversion_end_time;
- int64_t prefill_compile_end_time;
- int64_t decode_compile_end_time;
- build_static_model(ggml_decoder_prefill, "prefill", model_prefill, compiled_model_prefill,
- infer_request_prefill, prefill_conversion_end_time, prefill_compile_end_time);
- if (no_kv_cache) {
- model_decode = model_prefill;
- compiled_model_decode = compiled_model_prefill;
- infer_request_decode = infer_request_prefill;
- decode_conversion_end_time = prefill_conversion_end_time;
- decode_compile_end_time = prefill_compile_end_time;
- } else {
- build_static_model(ggml_decoder_decode, "decode", model_decode, compiled_model_decode, infer_request_decode,
- decode_conversion_end_time, decode_compile_end_time);
- }
- conversion_end_time = std::max(prefill_conversion_end_time, decode_conversion_end_time);
- compile_end_time = std::max(prefill_compile_end_time, decode_compile_end_time);
-
- model = is_prefill ? model_prefill : model_decode;
- ggml_decoder = is_prefill ? ggml_decoder_prefill : ggml_decoder_decode;
- infer_request = is_prefill ? infer_request_prefill : infer_request_decode;
- entry->ptr = ggml_decoder;
-
- for (const auto & ov_param : model->get_parameters()) {
- ov_input_names_local.push_back(ov_param->get_friendly_name());
- }
- for (const auto & ov_output : model->get_results()) {
- ov_output_names_local.push_back(ov_output->get_friendly_name());
- }
-
- if (!shared_key.empty()) {
- shared_cache->graphs.emplace(shared_key, ov_compiled_graph{compiled_model_decode, compiled_model_prefill,
- ov_input_names_local, ov_output_names_local});
- }
-
- if (cache_enabled) {
- std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
- r_ctx->infer_request_cache_prefill[key] = infer_request_prefill;
- r_ctx->infer_request_cache[key] = infer_request_decode;
- r_ctx->ov_input_names_cache[key] = ov_input_names_local;
- r_ctx->ov_output_names_cache[key] = ov_output_names_local;
- }
- }
-
- }
-
- if (is_prefill) {
- auto inp_len = get_inp_pos_n_tokens(cgraph, inp_pos);
- for (int chunk_index = 0; chunk_index * prefill_chunk_size < inp_len; chunk_index++) {
- for (size_t i = 0; i < ov_input_names_local.size(); i++) {
- auto param_name = ov_input_names_local[i];
- auto input_tensor = get_ov_input_tensor_static_prefill(ggml_decoder, param_name, chunk_index);
- infer_request->set_input_tensor(i, input_tensor);
-
- if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) {
- const auto input_tensor = infer_request->get_input_tensor(i);
- print_input_tensor_info(param_name, input_tensor);
+ // Use the cache-stripped config when the frontend model cache is active, so
+ // the resulting CompiledModel can be exported and later re-imported.
+ const ov::AnyMap & compile_config = model_cache_dir.empty() ? config : mc_config;
+ ov::CompiledModel compiled_model;
+ auto remote_context = ggml_openvino_get_remote_context();
+ if (remote_context.has_value()) {
+ compiled_model = core.compile_model(model, remote_context.value(), compile_config);
+ } else {
+ compiled_model = core.compile_model(model, device, compile_config);
}
- }
+ compile_end_time = ggml_time_us();
- for (size_t i = 0; i < ov_output_names_local.size(); i++) {
- const auto & model_outputs = ggml_decoder->get_model_outputs();
- auto model_output_it = model_outputs.find(ov_output_names_local[i]);
- if (model_output_it == model_outputs.end()) {
- continue;
- }
- auto * ggml_tensor = model_output_it->second;
- if (ggml_nbytes(ggml_tensor) == 0) {
- // Zero-row in-place writeback (e.g. the empty s_copy defrag remainder). The OV
- // Result is the full cache, so binding it over this 0-byte buffer overflows it.
- continue;
+ // Export to the frontend model cache for next time. Publish the blob first,
+ // then the manifest, so a cache hit only sees fully written artifacts.
+ if (!model_cache_dir.empty() && !model_is_splitted && model_fp != 0) {
+ try {
+ const std::string blob_tmp = blob_path + ".tmp";
+ const std::string manifest_tmp = manifest_path + ".tmp";
+ if (ggml_openvino_model_cache_write_manifest(manifest_tmp, cgraph, model_fp)) {
+ std::ofstream blob_out(blob_tmp, std::ios::binary | std::ios::trunc);
+ if (blob_out.is_open()) {
+ compiled_model.export_model(blob_out);
+ blob_out.close();
+ if (blob_out.good()) {
+ if (std::rename(blob_tmp.c_str(), blob_path.c_str()) == 0 &&
+ std::rename(manifest_tmp.c_str(), manifest_path.c_str()) == 0) {
+ GGML_LOG_INFO("ggml-openvino: model cache WROTE %s\n", blob_path.c_str());
+ } else {
+ std::remove(blob_tmp.c_str());
+ std::remove(manifest_tmp.c_str());
+ }
+ } else {
+ std::remove(blob_tmp.c_str());
+ std::remove(manifest_tmp.c_str());
+ }
+ } else {
+ std::remove(manifest_tmp.c_str());
+ }
+ }
+ } catch (const std::exception & e) {
+ GGML_LOG_WARN("ggml-openvino: model cache export failed: %s\n", e.what());
+ }
}
- auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor);
- infer_request->set_output_tensor(i, output_tensor);
- }
- ov_raw_infer_start = ggml_time_us();
- infer_request->infer();
- ov_raw_infer_total += ggml_time_us() - ov_raw_infer_start;
+ infer_request = std::make_shared<ov::InferRequest>(compiled_model.create_infer_request());
+ shared_model = compiled_model;
+ entry->ptr = ggml_decoder;
- if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") ||
- ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) {
- for (size_t i = 0; i < ov_output_names_local.size(); i++) {
- const auto output_tensor = infer_request->get_output_tensor(i);
- print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data());
+ for (const auto & ov_param : model->get_parameters()) {
+ ov_input_names.push_back(ov_param->get_friendly_name());
}
- }
- }
- infer_end_time = ggml_time_us();
- } else {
- for (size_t i = 0; i < ov_input_names_local.size(); i++) {
- auto param_name = ov_input_names_local[i];
- auto input_tensor = get_ov_input_tensor_static_decode(ggml_decoder, param_name);
- infer_request->set_input_tensor(i, input_tensor);
-
- if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) {
- const auto input_tensor = infer_request->get_input_tensor(i);
- print_input_tensor_info(param_name, input_tensor);
- }
- }
+ for (const auto & ov_output : model->get_results()) {
+ ov_output_names.push_back(ov_output->get_friendly_name());
+ }
+ } // end non-imported (compile) path
- for (size_t i = 0; i < ov_output_names_local.size(); i++) {
- const auto & model_outputs = ggml_decoder->get_model_outputs();
- auto model_output_it = model_outputs.find(ov_output_names_local[i]);
- if (model_output_it == model_outputs.end()) {
- continue;
- }
- auto * ggml_tensor = model_output_it->second;
- if (ggml_nbytes(ggml_tensor) == 0) {
- continue;
+ entry->ptr = ggml_decoder;
+ if (!shared_key.empty() && shared_it == shared_cache->graphs.end()) {
+ shared_cache->graphs.emplace(shared_key,
+ ov_compiled_graph{shared_model, {}, ov_input_names, ov_output_names});
}
- auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor);
- infer_request->set_output_tensor(i, output_tensor);
- }
-
- ov_raw_infer_start = ggml_time_us();
- infer_request->infer();
- infer_end_time = ggml_time_us();
- ov_raw_infer_total = infer_end_time - ov_raw_infer_start;
-
- if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") ||
- ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) {
- for (size_t i = 0; i < ov_output_names_local.size(); i++) {
- const auto output_tensor = infer_request->get_output_tensor(i);
- print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data());
+ if (cache_enabled) {
+ std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
+ r_ctx->infer_request_cache[key] = infer_request;
+ r_ctx->ov_input_names_cache[key] = ov_input_names;
+ r_ctx->ov_output_names_cache[key] = ov_output_names;
}
- }
- }
-
- if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) {
- GGML_LOG_INFO("\nGGML OpenVINO Backend: \n");
- GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0);
- if (!cache_hit) {
- GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n", (conversion_end_time - decoder_end_time) / 1000.0);
- GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0);
- }
- GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0);
- GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", ov_raw_infer_total / 1000.0);
- }
-
- return GGML_STATUS_SUCCESS;
-}
-
-// Detect whether a cgraph is a split subgraph or not.
-// Step 1 compares each node's recorded use_count with actual fan-out references in node->src.
-// Step 2 verifies that node inputs come from model nodes/weights/leafs; external sources imply split.
-bool is_model_splitted(ggml_cgraph * cgraph) {
- static const bool fallback_enabled = ggml_openvino_getenv_int("GGML_OPENVINO_ENABLE_FALLBACK") != 0;
- if (!fallback_enabled) {
- return false;
- }
-
- // Backend op tests execute each node through ggml_graph_view(), which preserves the original
- // graph use_counts while exposing only one node. Treat those single-node views as regular
- // naive graphs so intermediate ops do not look like split-model fragments.
- if (cgraph->n_nodes <= 1 && cgraph->n_leafs == 0) {
- return false;
- }
- // check the nodes of the model are used by the following nodes, through compare the node's use count and the count of nodes that use it as input. If does not match, return true, else return false.
- for (int i = 0; i < cgraph->n_nodes; i++) {
- ggml_tensor * node = cgraph->nodes[i];
- int use_count = cgraph->use_counts[ggml_hash_find(&cgraph->visited_hash_set, node)];
- // TODO: this is a workround for the tests case from llama.cpp, fix should from the root cause in the future.
- if ((cgraph->n_nodes <= 1 && use_count == 0) ||
- (cgraph->n_nodes <= 1 && node->op == GGML_OP_VIEW && use_count == 1 && node->src[0] != nullptr &&
- node->src[0]->op == GGML_OP_NONE)) {
- return false;
+ if (stateful && cache_enabled) {
+ const auto * inp_pos = get_inp_pos_tensor(cgraph);
+ auto pos_shape = GgmlOvDecoder::get_shape(inp_pos);
+ // A freshly compiled model starts with an empty state, so it can only serve a
+ // sequence from its beginning. A non-zero start position means the KV history was
+ // built elsewhere (a restored ggml cache), which the state cannot adopt.
+ const int32_t pos_begin = ((int32_t *) inp_pos->data)[0];
+ if (pos_begin != 0) {
+ GGML_LOG_ERROR(
+ "GGML OpenVINO backend stateful inference failed: a new model was compiled for a sequence that "
+ "starts at position %d, but its state is empty. Run without "
+ "GGML_OPENVINO_STATEFUL_EXECUTION.\n",
+ pos_begin);
+ return GGML_STATUS_FAILED;
+ }
+ r_ctx->stateful_kv_size = pos_shape[3];
+ const auto kv_param_res_names = ggml_decoder->get_kv_param_res_names();
+ for (const auto & pair : kv_param_res_names) {
+ r_ctx->kv_state_input_name_map[pair.first + pair.second] = pair.first;
+ }
+ }
}
- if (cgraph->n_nodes == 1 &&
- (cgraph->nodes[0]->op == GGML_OP_TRANSPOSE || cgraph->nodes[0]->op == GGML_OP_PERMUTE)) {
- return false;
+
+ for (size_t i = 0; i < ov_input_names.size(); i++) {
+ const auto & param_name = ov_input_names[i];
+ auto input_tensor = get_ov_input_tensor(ggml_decoder, param_name);
+ infer_request->set_input_tensor(i, input_tensor);
+
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) {
+ print_input_tensor_info(param_name, input_tensor);
+ }
}
- int input_use_count = 0;
- for (int j = 0; j < cgraph->n_nodes; j++) {
- ggml_tensor * other_node = cgraph->nodes[j];
- for (int k = 0; k < GGML_MAX_SRC; k++) {
- if (other_node->src[k] == node) {
- input_use_count++;
- }
+
+ for (size_t i = 0; i < ov_output_names.size(); i++) {
+ // Debug-only outputs added via GGML_OPENVINO_DEBUG_NODE (see
+ // translate_session.cpp) have no corresponding ggml tensor; leave
+ // them unbound so OpenVINO allocates its own tensor for them,
+ // rather than aliasing a ggml buffer that may be overwritten by a
+ // later in-place op before we get to read it.
+ const auto & model_outputs = ggml_decoder->get_model_outputs();
+ auto model_output_it = model_outputs.find(ov_output_names[i]);
+ if (model_output_it == model_outputs.end()) {
+ continue;
+ }
+ auto * ggml_tensor = model_output_it->second;
+ if (ggml_nbytes(ggml_tensor) == 0) {
+ continue;
}
+ auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor);
+ infer_request->set_output_tensor(i, output_tensor);
}
- if (use_count != input_use_count && node->op != GGML_OP_NONE) {
- return true;
+
+ ov_raw_infer_start = ggml_time_us();
+ infer_request->infer();
+ infer_end_time = ggml_time_us();
+
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") ||
+ ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) {
+ for (size_t i = 0; i < ov_output_names.size(); i++) {
+ const auto output_tensor = infer_request->get_output_tensor(i);
+ print_output_tensor_info(ov_output_names[i], output_tensor, output_tensor.data());
+ }
}
- }
- // if all nodes's src node's src is not come from the nodes in the model, we think the model is splitted. This is a complementary check for the above check, because for some special case like the output node is not used by any node, the use count and input use count are both 0, we can not determine whether the model is splitted or not just based on the first check.
- // Only weight-name membership is needed below. With GGML_OPENVINO_REDUCE_COMPILE_MEM
- // use the name-only collector (no weight extraction); otherwise keep the original
- // behavior of building (naive) weight nodes and take their names.
- std::set<std::string> model_weights;
- if (ggml_openvino_reduce_compile_mem_enabled()) {
- model_weights = GgmlOvDecoder::collect_weight_names(cgraph);
- } else {
- for (const auto & kv : GgmlOvDecoder::create_weight_nodes(cgraph, true)) {
- model_weights.insert(kv.first);
+
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) {
+ GGML_LOG_INFO("\nGGML OpenVINO Backend: \n");
+ GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0);
+ if (!cache_hit) {
+ GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n",
+ (conversion_end_time - decoder_end_time) / 1000.0);
+ GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0);
+ }
+ GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0);
+ GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", (infer_end_time - ov_raw_infer_start) / 1000.0);
}
}
- std::set<ggml_tensor *> model_nodes(cgraph->nodes, cgraph->nodes + cgraph->n_nodes);
- // leaf nodes
- std::set<ggml_tensor *> model_leafs(cgraph->leafs, cgraph->leafs + cgraph->n_leafs);
- for (int i = 0; i < cgraph->n_nodes; i++) {
- ggml_tensor * node = cgraph->nodes[i];
- for (int j = 0; j < GGML_MAX_SRC; j++) {
- ggml_tensor * src = node->src[j];
- // the src is also not the model weights, we think the model is splitted.
- // the src is also not in model leafs, we think the model is splitted.
- if (src != nullptr && model_nodes.find(src) == model_nodes.end() &&
- model_weights.find(std::string(src->name)) == model_weights.end() && !model_leafs.empty() == false &&
- model_leafs.find(src) == model_leafs.end()) {
- if (GgmlOvDecoder::is_inp_tok(src, node)) {
- return false;
- }
- return true;
- }
+
+ // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU): the plugin holds its own device copy of
+ // every weight after compile, so the host weight buffers can be dropped to reclaim
+ // RSS. Release only while holding the compilation mutex so another context cannot
+ // be reading host weights during conversion/compilation. Pin the shared compiled
+ // models across backend teardown; a later context can create its own request without
+ // reading the dropped pages. A new, uncached graph still fails fast above.
+ if (cache_hit && ggml_openvino_release_weights_enabled(device)) {
+ std::lock_guard<std::mutex> compile_lock(r_ctx->compiled_cache->mutex);
+ if (!ggml_openvino_weight_buffers_released()) {
+ ggml_openvino_release_weight_buffers();
}
}
- return false;
+
+ return GGML_STATUS_SUCCESS;
}
-bool is_naive(ggml_cgraph * cgraph) {
- constexpr int naive_graph_size_threshold = 20;
- int count = 0;
- for (int i = 0; i < cgraph->n_nodes; i++) {
- if (cgraph->nodes[i]->op != GGML_OP_NONE) {
- count++;
+ov::AnyMap without_npuw(const ov::AnyMap & config) {
+ ov::AnyMap out;
+ for (const auto & kv : config) {
+ if (kv.first.rfind("NPUW", 0) == 0 || kv.first == "NPU_USE_NPUW") {
+ continue;
}
+ out.insert(kv);
}
- return count < naive_graph_size_threshold;
+ return out;
}
-enum ggml_status naive_compute(ggml_cgraph * cgraph,
- ov::Core & core,
- const std::string & device,
- const ov::AnyMap & config,
- ov_compiled_model_cache & cache) {
- if (cgraph->n_nodes == 1 && (cgraph->nodes[0]->op == GGML_OP_NONE || cgraph->nodes[0]->op == GGML_OP_VIEW)) {
- return GGML_STATUS_SUCCESS;
- }
+enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared_ptr<ov_runtime_context> & r_ctx) {
+ auto & core = ov_singleton_core();
- std::unique_lock<std::mutex> compile_lock(cache.mutex);
- bool naive = true;
- auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, naive);
- auto decoder = std::make_shared<GgmlOvDecoder>(cgraph, model_weights);
- auto input_model = std::make_shared<ov::frontend::ggml::InputModel>(decoder);
- auto model = ov::frontend::ggml::FrontEnd::convert(input_model, naive);
- if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) {
- ov::serialize(model, "IR_naive.xml");
+ auto get_prefill_chunk_size = [] {
+ static const int chunk_size = []() {
+ int env_prefill_chunk_size = ggml_openvino_getenv_int("GGML_OPENVINO_PREFILL_CHUNK_SIZE");
+ return env_prefill_chunk_size > 0 ? env_prefill_chunk_size : 256;
+ }();
+ return chunk_size;
+ };
+
+ // Normally NPU, but honors GGML_OPENVINO_DEVICE so GGML_OPENVINO_FORCE_STATIC can run the
+ // static-shape path on CPU/GPU to isolate translation bugs from NPUW/NPU-driver issues.
+ static std::string device = ggml_openvino_get_device_name();
+ static auto is_static = true;
+ static auto stateful = false;
+
+ auto prefill_chunk_size = get_prefill_chunk_size();
+ const auto & config = ggml_openvino_get_compile_config();
+
+ if (is_naive(cgraph)) {
+ return naive_compute(cgraph, core, device, config, *r_ctx->compiled_cache);
}
+ auto start_time = ggml_time_us();
+
+ std::shared_ptr<GgmlOvDecoder> ggml_decoder;
std::shared_ptr<ov::InferRequest> infer_request;
- auto remote_context = ggml_openvino_get_remote_context();
- ov::AnyMap compile_config = config;
- if (cgraph->nodes[0]->op == GGML_OP_MUL_MAT) {
- // TODO ACCURACY hint triggers a bug in GPU plugin/driver on Lunar Lake. Remove once CVS-182166 is resolved
- compile_config[ov::hint::execution_mode.name()] = ov::hint::ExecutionMode::PERFORMANCE;
- } else {
- compile_config[ov::hint::execution_mode.name()] = ov::hint::ExecutionMode::ACCURACY;
+ ModelParams m_params;
+ ComputeParams c_params;
+ std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static);
+
+ const auto * inp_pos = get_inp_pos_tensor(cgraph);
+ const bool no_kv_cache = m_params.is_cacheless_attn;
+ const auto is_prefill = no_kv_cache ? true : get_is_prefill(cgraph, inp_pos);
+ const ov::AnyMap compile_config = no_kv_cache ? without_npuw(config) : config;
+ if (m_params.n_heads_kv == -1) {
+ prefill_chunk_size = inp_pos->ne[0];
}
- if (remote_context.has_value()) {
- infer_request = std::make_shared<ov::InferRequest>(
- core.compile_model(model, remote_context.value(), compile_config).create_infer_request());
+ graph_key key(cgraph);
+ static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE");
+ bool cache_hit = false;
+
+ int64_t decoder_end_time;
+ int64_t conversion_end_time;
+ int64_t compile_end_time;
+ int64_t infer_end_time;
+ int64_t ov_raw_infer_start;
+ int64_t ov_raw_infer_total = 0;
+
+ std::shared_ptr<decoder_runtime_ctx> entry;
+ ModelParams old_m_params;
+
+ if (cache_enabled) {
+ std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
+ auto it = r_ctx->decoder_cache.find(key);
+ cache_hit = it != r_ctx->decoder_cache.end();
+ if (cache_hit) {
+ entry = it->second;
+ } else {
+ r_ctx->clear_caches_locked();
+ auto mutex = std::make_shared<std::mutex>();
+ entry = std::make_shared<decoder_runtime_ctx>(mutex);
+ r_ctx->decoder_cache[key] = entry;
+ }
} else {
- infer_request =
- std::make_shared<ov::InferRequest>(core.compile_model(model, device, compile_config).create_infer_request());
- }
- std::vector<std::string> input_names;
- std::vector<std::string> output_names;
- for (const auto & param : model->get_parameters()) {
- input_names.push_back(param->get_friendly_name());
- }
- for (const auto & result : model->get_results()) {
- output_names.push_back(result->get_friendly_name());
+ auto mutex = std::make_shared<std::mutex>();
+ entry = std::make_shared<decoder_runtime_ctx>(mutex);
+ cache_hit = false;
}
- // Destroy the frontend graph under the compilation lock as well: it can
- // still own edges into the shared weight nodes.
- model.reset();
- input_model.reset();
- decoder->clear_model_weights();
- model_weights.clear();
- compile_lock.unlock();
- for (size_t i = 0; i < input_names.size(); i++) {
- const auto & param_name = input_names[i];
- auto input_tensor = get_ov_input_tensor(decoder, param_name);
- infer_request->set_input_tensor(i, input_tensor);
- }
+ std::lock_guard<std::mutex> lock(*(entry->mutex));
+ cache_hit = cache_hit && entry->ptr && r_ctx->infer_request_cache.count(key) != 0 &&
+ r_ctx->infer_request_cache_prefill.count(key) != 0;
- // Use get_output_tensor + memcpy instead of set_output_tensor to avoid memory overwritten
- // when i/o buffer overlaps, e.g. the cgraph is a single PERMUTE
+ if (cache_hit) {
+ ggml_decoder = entry->ptr;
+ old_m_params = ggml_decoder->get_model_params();
+ cache_hit = old_m_params.can_reuse_statically(m_params);
+ }
- infer_request->infer();
+ std::vector<std::string> ov_input_names_local;
+ std::vector<std::string> ov_output_names_local;
- for (size_t i = 0; i < output_names.size(); i++) {
- auto output_tensor = infer_request->get_output_tensor(i);
- const auto & model_outputs = decoder->get_model_outputs();
- auto model_output_it = model_outputs.find(output_names[i]);
- if (model_output_it == model_outputs.end()) {
- // Debug-only output added via GGML_OPENVINO_DEBUG_NODE; nothing to copy into.
- if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") ||
- ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) {
- print_output_tensor_info(output_names[i], output_tensor, output_tensor.data());
- }
- continue;
+ if (cache_hit) {
+ std::map<std::string, std::shared_ptr<ov::Node>> model_weights;
+ ggml_decoder->m_is_prefill = is_prefill;
+ ggml_decoder->set_model_params(m_params);
+ ggml_decoder->set_compute_params(c_params);
+ if (old_m_params.kv_buffer_changed(m_params)) {
+ ggml_decoder->update_io(cgraph);
+ }
+ ggml_decoder->add_extra_inputs();
+ {
+ std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
+ infer_request =
+ is_prefill ? r_ctx->infer_request_cache_prefill.at(key) : r_ctx->infer_request_cache.at(key);
+ ov_input_names_local = r_ctx->ov_input_names_cache.at(key);
+ ov_output_names_local = r_ctx->ov_output_names_cache.at(key);
}
- auto * ggml_tensor = model_output_it->second;
- std::memcpy(ggml_tensor->data, output_tensor.data(), output_tensor.get_byte_size());
- }
- return GGML_STATUS_SUCCESS;
-}
-
-namespace {
-template <typename T> void set_zero_diagonal(std::vector<T> & matrix, size_t rows, size_t cols, T zero_value = T{}) {
- for (size_t i = 0; i < rows; ++i) {
- size_t diag_col = std::min(i, cols - 1);
- matrix[i * cols + diag_col] = zero_value;
- }
-}
-ov::Tensor make_contiguous_split_input_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
- const struct ggml_tensor * ggml_tensor,
- const ov::Shape & input_shape) {
- const size_t element_size = ggml_type_size(ggml_tensor->type);
- const size_t block_size = ggml_blck_size(ggml_tensor->type);
+ decoder_end_time = ggml_time_us();
+ conversion_end_time = decoder_end_time;
+ compile_end_time = decoder_end_time;
+ } else {
+ if (cache_enabled) {
+ std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
+ r_ctx->infer_request_cache.erase(key);
+ r_ctx->infer_request_cache_prefill.erase(key);
+ }
- GGML_ASSERT(block_size == 1 && "non-contiguous split inputs must be plain element types");
+ // Static execution shares a compiled prefill/decode pair. Each backend
+ // creates and retains its own requests for both phases.
+ auto shared_cache = r_ctx->compiled_cache;
+ std::unique_lock<std::mutex> compile_lock(shared_cache->mutex);
+ auto weight_names = get_weight_names(cgraph);
+ auto local_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, weight_names, is_static,
+ stateful, false, is_prefill, prefill_chunk_size);
+ const std::string shared_key =
+ cache_enabled ? compiled_graph_key(cgraph, *local_decoder, device, prefill_chunk_size) : "";
+ auto shared_it = shared_cache->graphs.find(shared_key);
+ if (!shared_key.empty() && shared_it != shared_cache->graphs.end()) {
+ auto & compiled = shared_it->second;
+ auto prefill_request = std::make_shared<ov::InferRequest>(compiled.prefill.create_infer_request());
+ auto decode_request = no_kv_cache ?
+ prefill_request :
+ std::make_shared<ov::InferRequest>(compiled.decode.create_infer_request());
+ ggml_decoder = local_decoder;
+ entry->ptr = ggml_decoder;
+ infer_request = is_prefill ? prefill_request : decode_request;
+ ov_input_names_local = compiled.input_names;
+ ov_output_names_local = compiled.output_names;
+ r_ctx->infer_request_cache_prefill[key] = prefill_request;
+ r_ctx->infer_request_cache[key] = decode_request;
+ r_ctx->ov_input_names_cache[key] = ov_input_names_local;
+ r_ctx->ov_output_names_cache[key] = ov_output_names_local;
+ decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us();
+ GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (static)\n");
+ } else {
+ std::shared_ptr<ov::Model> model;
+ auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph);
- const struct ggml_tensor * source_tensor = ggml_tensor->view_src != nullptr ? ggml_tensor->view_src : ggml_tensor;
- const size_t source_offset = ggml_tensor->view_src != nullptr ? ggml_tensor->view_offs : 0;
+ auto ggml_decoder_prefill = std::make_shared<GgmlOvDecoder>(
+ cgraph, m_params, c_params, model_weights, is_static, stateful, false, true, prefill_chunk_size);
+ auto ggml_decoder_decode =
+ no_kv_cache ? ggml_decoder_prefill :
+ std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights, is_static,
+ stateful, false, false, prefill_chunk_size);
+ decoder_end_time = ggml_time_us();
- std::vector<uint8_t> source_data(ggml_nbytes(source_tensor));
- ggml_backend_tensor_get(source_tensor, source_data.data(), 0, source_data.size());
+ const bool dump_ir = ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR");
+ const auto dump_ir_timestamp = static_cast<long long>(ggml_time_us());
- ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape);
- auto * dst = static_cast<uint8_t *>(input_tensor.data());
- size_t dst_offset = 0;
+ auto build_static_model = [&core, &compile_config, dump_ir, dump_ir_timestamp](
+ const std::shared_ptr<GgmlOvDecoder> & decoder, const char * tag,
+ std::shared_ptr<ov::Model> & model, ov::CompiledModel & compiled_model,
+ std::shared_ptr<ov::InferRequest> & infer_request,
+ int64_t & local_conversion_end_time, int64_t & local_compile_end_time) {
+ auto input_model = std::make_shared<ov::frontend::ggml::InputModel>(decoder);
+ model = ov::frontend::ggml::FrontEnd::convert(input_model);
+ decoder->clear_model_weights();
+ local_conversion_end_time = ggml_time_us();
- for (size_t i3 = 0; i3 < static_cast<size_t>(ggml_tensor->ne[3]); ++i3) {
- for (size_t i2 = 0; i2 < static_cast<size_t>(ggml_tensor->ne[2]); ++i2) {
- for (size_t i1 = 0; i1 < static_cast<size_t>(ggml_tensor->ne[1]); ++i1) {
- for (size_t i0 = 0; i0 < static_cast<size_t>(ggml_tensor->ne[0]); ++i0) {
- const size_t src_offset = source_offset + i3 * ggml_tensor->nb[3] + i2 * ggml_tensor->nb[2] +
- i1 * ggml_tensor->nb[1] + i0 * ggml_tensor->nb[0];
- std::memcpy(dst + dst_offset, source_data.data() + src_offset, element_size);
- dst_offset += element_size;
+ if (dump_ir) {
+ char timestamped_filename[64];
+ snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%s_%lld.xml", tag,
+ dump_ir_timestamp);
+ ov::serialize(model, timestamped_filename);
}
+
+ compiled_model = core.compile_model(model, device, compile_config);
+ infer_request = std::make_shared<ov::InferRequest>(compiled_model.create_infer_request());
+ local_compile_end_time = ggml_time_us();
+ };
+ std::shared_ptr<ov::Model> model_prefill;
+ std::shared_ptr<ov::Model> model_decode;
+ ov::CompiledModel compiled_model_prefill;
+ ov::CompiledModel compiled_model_decode;
+ std::shared_ptr<ov::InferRequest> infer_request_prefill;
+ std::shared_ptr<ov::InferRequest> infer_request_decode;
+ int64_t prefill_conversion_end_time;
+ int64_t decode_conversion_end_time;
+ int64_t prefill_compile_end_time;
+ int64_t decode_compile_end_time;
+ build_static_model(ggml_decoder_prefill, "prefill", model_prefill, compiled_model_prefill,
+ infer_request_prefill, prefill_conversion_end_time, prefill_compile_end_time);
+ if (no_kv_cache) {
+ model_decode = model_prefill;
+ compiled_model_decode = compiled_model_prefill;
+ infer_request_decode = infer_request_prefill;
+ decode_conversion_end_time = prefill_conversion_end_time;
+ decode_compile_end_time = prefill_compile_end_time;
+ } else {
+ build_static_model(ggml_decoder_decode, "decode", model_decode, compiled_model_decode,
+ infer_request_decode, decode_conversion_end_time, decode_compile_end_time);
}
- }
- }
+ conversion_end_time = std::max(prefill_conversion_end_time, decode_conversion_end_time);
+ compile_end_time = std::max(prefill_compile_end_time, decode_compile_end_time);
- return input_tensor;
-}
+ model = is_prefill ? model_prefill : model_decode;
+ ggml_decoder = is_prefill ? ggml_decoder_prefill : ggml_decoder_decode;
+ infer_request = is_prefill ? infer_request_prefill : infer_request_decode;
+ entry->ptr = ggml_decoder;
-ov::Tensor convert_ggml_input_to_ov(std::shared_ptr<GgmlOvDecoder> ggml_decoder, const std::string & name) {
- const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(name);
+ for (const auto & ov_param : model->get_parameters()) {
+ ov_input_names_local.push_back(ov_param->get_friendly_name());
+ }
+ for (const auto & ov_output : model->get_results()) {
+ ov_output_names_local.push_back(ov_output->get_friendly_name());
+ }
- if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, name, ggml_tensor)) {
- return *sliced;
- }
+ if (!shared_key.empty()) {
+ shared_cache->graphs.emplace(
+ shared_key, ov_compiled_graph{compiled_model_decode, compiled_model_prefill, ov_input_names_local,
+ ov_output_names_local});
+ }
- if (ggml_tensor->extra != nullptr && !ggml_decoder->is_splited_model()) {
- auto * extra_base = static_cast<ggml_openvino_extra_base *>(ggml_tensor->extra);
- if (extra_base->type == ggml_openvino_extra_base::Type::TENSOR) {
- // GGML_LOG_DEBUG("Using ggml_tensor->extra as ov::Tensor for input: %s\n", name.c_str());
- auto * tensor_extra = static_cast<ggml_openvino_tensor_extra *>(extra_base);
- return *tensor_extra->tensor;
+ if (cache_enabled) {
+ std::lock_guard<std::mutex> map_lock(r_ctx->ctx_mutex);
+ r_ctx->infer_request_cache_prefill[key] = infer_request_prefill;
+ r_ctx->infer_request_cache[key] = infer_request_decode;
+ r_ctx->ov_input_names_cache[key] = ov_input_names_local;
+ r_ctx->ov_output_names_cache[key] = ov_output_names_local;
+ }
}
}
- // GGML_LOG_DEBUG("Converting ggml tensor to ov::Tensor for input: %s\n", name.c_str());
- auto * input_data = ggml_tensor->data;
- ov::Shape input_shape;
- if (ggml_tensor->op == GGML_OP_VIEW && !ggml_decoder->is_splited_model()) {
- // This case is added to make test-backend-ops work
- input_shape = ggml_decoder->get_shape(ggml_tensor->view_src);
- } else {
- input_shape = ggml_decoder->get_shape(ggml_tensor);
- }
+ if (is_prefill) {
+ auto inp_len = get_inp_pos_n_tokens(cgraph, inp_pos);
+ for (int chunk_index = 0; chunk_index * prefill_chunk_size < inp_len; chunk_index++) {
+ for (size_t i = 0; i < ov_input_names_local.size(); i++) {
+ const auto & param_name = ov_input_names_local[i];
+ auto input_tensor = get_ov_input_tensor_static_prefill(ggml_decoder, param_name, chunk_index);
+ infer_request->set_input_tensor(i, input_tensor);
- if (ggml_decoder->is_splited_model() && !ggml_is_contiguous(ggml_tensor)) {
- return make_contiguous_split_input_tensor(ggml_decoder, ggml_tensor, input_shape);
- }
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) {
+ const auto input_tensor = infer_request->get_input_tensor(i);
+ print_input_tensor_info(param_name, input_tensor);
+ }
+ }
- auto input_tensor = ov::Tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape, input_data);
- return input_tensor;
-}
-} // namespace
+ for (size_t i = 0; i < ov_output_names_local.size(); i++) {
+ const auto & model_outputs = ggml_decoder->get_model_outputs();
+ auto model_output_it = model_outputs.find(ov_output_names_local[i]);
+ if (model_output_it == model_outputs.end()) {
+ continue;
+ }
+ auto * ggml_tensor = model_output_it->second;
+ if (ggml_nbytes(ggml_tensor) == 0) {
+ // Zero-row in-place writeback (e.g. the empty s_copy defrag remainder). The OV
+ // Result is the full cache, so binding it over this 0-byte buffer overflows it.
+ continue;
+ }
+ auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor);
+ infer_request->set_output_tensor(i, output_tensor);
+ }
-ov::Tensor get_ov_input_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, const std::string & param_name) {
- ov::Tensor input_tensor;
- auto extra_input = ggml_decoder->get_model_extra_inputs().find(param_name);
- if (extra_input != ggml_decoder->get_model_extra_inputs().end()) {
- input_tensor = ov::Tensor(extra_input->second.type, extra_input->second.shape);
- *input_tensor.data<int64_t>() = extra_input->second.value;
+ ov_raw_infer_start = ggml_time_us();
+ infer_request->infer();
+ ov_raw_infer_total += ggml_time_us() - ov_raw_infer_start;
+
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") ||
+ ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) {
+ for (size_t i = 0; i < ov_output_names_local.size(); i++) {
+ const auto output_tensor = infer_request->get_output_tensor(i);
+ print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data());
+ }
+ }
+ }
+ infer_end_time = ggml_time_us();
} else {
- input_tensor = convert_ggml_input_to_ov(ggml_decoder, param_name);
- }
- return input_tensor;
-}
+ for (size_t i = 0; i < ov_input_names_local.size(); i++) {
+ const auto & param_name = ov_input_names_local[i];
+ auto input_tensor = get_ov_input_tensor_static_decode(ggml_decoder, param_name);
+ infer_request->set_input_tensor(i, input_tensor);
-ov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
- const std::string & param_name) {
- // NPU decoding stage
- if (ggml_decoder->get_model_extra_inputs().count(param_name)) {
- return get_ov_input_tensor(ggml_decoder, param_name);
- }
- const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name);
- const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor);
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) {
+ const auto input_tensor = infer_request->get_input_tensor(i);
+ print_input_tensor_info(param_name, input_tensor);
+ }
+ }
- if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ||
- GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) {
- // IMROPE's inp_pos holds one value per t/h/w/e plane instead of a single position;
- // with a single decode token the planes are still contiguous, so a flat copy works.
- const int n_planes = GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ? GgmlOvDecoder::get_inp_pos_n_planes(op) : 1;
- assert(ggml_tensor->ne[0] == n_planes);
- ov::Shape input_shape = {1, 1, 1, (size_t) n_planes};
- ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape);
- std::memcpy(input_tensor.data(), ggml_tensor->data, n_planes * ggml_type_size(ggml_tensor->type));
- return input_tensor;
- }
+ for (size_t i = 0; i < ov_output_names_local.size(); i++) {
+ const auto & model_outputs = ggml_decoder->get_model_outputs();
+ auto model_output_it = model_outputs.find(ov_output_names_local[i]);
+ if (model_output_it == model_outputs.end()) {
+ continue;
+ }
+ auto * ggml_tensor = model_output_it->second;
+ if (ggml_nbytes(ggml_tensor) == 0) {
+ continue;
+ }
+ auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor);
+ infer_request->set_output_tensor(i, output_tensor);
+ }
- if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) {
- ov::Shape input_shape = {1, 1, 1, 1};
- ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape);
- int32_t inp_out_id = *((int32_t *) ggml_tensor->data);
- assert(ggml_tensor->ne[0] == 1);
- assert(inp_out_id == 0);
- *input_tensor.data<int32_t>() = inp_out_id;
- return input_tensor;
- }
+ ov_raw_infer_start = ggml_time_us();
+ infer_request->infer();
+ infer_end_time = ggml_time_us();
+ ov_raw_infer_total = infer_end_time - ov_raw_infer_start;
- if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) {
- size_t context_size = ggml_decoder->get_ctx_size();
- if (ggml_tensor->type == GGML_TYPE_F16) {
- std::vector<ggml_fp16_t> padded_data =
- pad_input<ggml_fp16_t>(ggml_tensor, 1, context_size, GGML_FP32_TO_FP16(-INFINITY));
- ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, 1, context_size});
- std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t));
- return input_tensor;
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") ||
+ ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) {
+ for (size_t i = 0; i < ov_output_names_local.size(); i++) {
+ const auto output_tensor = infer_request->get_output_tensor(i);
+ print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data());
+ }
}
+ }
- std::vector<float> padded_data = pad_input<float>(ggml_tensor, 1, context_size, -INFINITY);
- ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, 1, context_size});
- auto * data_ptr = input_tensor.data<float>();
- std::copy(padded_data.begin(), padded_data.begin() + context_size, data_ptr);
- return input_tensor;
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) {
+ GGML_LOG_INFO("\nGGML OpenVINO Backend: \n");
+ GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0);
+ if (!cache_hit) {
+ GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n", (conversion_end_time - decoder_end_time) / 1000.0);
+ GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0);
+ }
+ GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0);
+ GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", ov_raw_infer_total / 1000.0);
}
- return get_ov_input_tensor(ggml_decoder, param_name);
+ return GGML_STATUS_SUCCESS;
}
+} // namespace
-ov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
- const std::string & param_name,
- int chunk_index) {
- // NPU prompt processing stage
- const size_t input_len = ggml_decoder->get_input_len();
- const size_t chunk_size = ggml_decoder->m_prefill_chunk_size;
- const size_t chunk_valid_size = std::min(chunk_size, input_len - chunk_index * chunk_size);
- const size_t chunk_pad_size = chunk_size - chunk_valid_size;
+// Both execution paths use two cache levels:
+// 1. Reuse this backend's decoder/request via graph_key and compatibility checks.
+// 2. On a local miss, look up compiled_graph_key in the shared compilation cache,
+// compile if needed, then create a private request from the compiled model.
+// The shared lock covers compilation and frontend cleanup, never inference.
+enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) {
+ ggml_backend_openvino_context * ctx = (ggml_backend_openvino_context *) backend->context;
+ try {
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_CGRAPH")) {
+ std::string filename = "cgraph_ov.txt";
+ GgmlOvDecoder::dump_cgraph(cgraph, filename);
+ }
- if (param_name == "chunk_valid_len") {
- ov::Tensor input_tensor(ov::element::i64, ov::Shape{1});
- *input_tensor.data<int64_t>() = (int64_t) chunk_valid_size;
- return input_tensor;
+ const auto is_static = ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC");
+
+ GGML_ASSERT(ctx->runtime_context != nullptr);
+ std::shared_ptr<ov_runtime_context> r_ctx = std::static_pointer_cast<ov_runtime_context>(ctx->runtime_context);
+ std::lock_guard<std::mutex> execution_lock(r_ctx->execution_mutex);
+
+ return is_static ? ov_graph_compute_static(cgraph, r_ctx) : ov_graph_compute_dynamic(cgraph, r_ctx);
+ } catch (const ov::Exception & e) {
+ GGML_LOG_ERROR("GGML OpenVINO backend ov::Exception: %s\n", e.what());
+ return GGML_STATUS_FAILED;
+ } catch (const std::exception & e) {
+ GGML_LOG_ERROR("GGML OpenVINO backend std::exception: %s\n", e.what());
+ return GGML_STATUS_FAILED;
+ } catch (...) {
+ GGML_LOG_ERROR("GGML OpenVINO backend unknown exception\n");
+ return GGML_STATUS_FAILED;
}
- if (chunk_index > 0 && param_name == "cache_rs_reset_len") {
- // The recurrent-state clear belongs to the start of the sequence. Re-applying it on every
- // chunk would wipe the state accumulated by the preceding chunks, so disable it (a zero
- // length makes scale.cpp's keep-mask select every slot) after the first chunk.
- ov::Tensor input_tensor(ov::element::i64, ov::Shape{1});
- *input_tensor.data<int64_t>() = 0;
- return input_tensor;
+}
+
+// Detect whether a cgraph is a split subgraph or not.
+// Step 1 compares each node's recorded use_count with actual fan-out references in node->src.
+// Step 2 verifies that node inputs come from model nodes/weights/leafs; external sources imply split.
+bool is_model_splitted(ggml_cgraph * cgraph) {
+ static const bool fallback_enabled = ggml_openvino_getenv_int("GGML_OPENVINO_ENABLE_FALLBACK") != 0;
+ if (!fallback_enabled) {
+ return false;
}
- if (ggml_decoder->get_model_extra_inputs().count(param_name)) {
- return get_ov_input_tensor(ggml_decoder, param_name);
+
+ // Backend op tests execute each node through ggml_graph_view(), which preserves the original
+ // graph use_counts while exposing only one node. Treat those single-node views as regular
+ // naive graphs so intermediate ops do not look like split-model fragments.
+ if (cgraph->n_nodes <= 1 && cgraph->n_leafs == 0) {
+ return false;
}
- const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name);
- const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor);
- if (GgmlOvDecoder::is_inp_pos(ggml_tensor, op) && GgmlOvDecoder::get_inp_pos_n_planes(op) > 1) {
- // IMROPE: inp_pos stacks n_planes (t/h/w/e) position planes, each of length
- // input_len; pad every plane independently so they stay aligned to chunk_size.
- const int n_planes = GgmlOvDecoder::get_inp_pos_n_planes(op);
- const size_t element_size = ggml_type_size(ggml_tensor->type);
- ov::Shape input_shape = {1, 1, 1, (size_t) n_planes * chunk_size};
- ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape);
- for (int p = 0; p < n_planes; p++) {
- const char * src =
- (const char *) ggml_tensor->data + (p * input_len + chunk_index * chunk_size) * element_size;
- char * dst = (char *) input_tensor.data() + p * chunk_size * element_size;
- std::memcpy(dst, src, chunk_valid_size * element_size);
- if (chunk_pad_size > 0) {
- if (ggml_tensor->type == GGML_TYPE_I32) {
- int32_t last_value = *((const int32_t *) src + chunk_valid_size - 1);
- int32_t * out = (int32_t *) dst;
- std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1);
- } else if (ggml_tensor->type == GGML_TYPE_I64) {
- int64_t last_value = *((const int64_t *) src + chunk_valid_size - 1);
- int64_t * out = (int64_t *) dst;
- std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1);
- } else {
- throw std::runtime_error("Unexpected tensor type for " + param_name);
+ // check the nodes of the model are used by the following nodes, through compare the node's use count and the count of nodes that use it as input. If does not match, return true, else return false.
+ for (int i = 0; i < cgraph->n_nodes; i++) {
+ ggml_tensor * node = cgraph->nodes[i];
+ int use_count = cgraph->use_counts[ggml_hash_find(&cgraph->visited_hash_set, node)];
+ // TODO: this is a workround for the tests case from llama.cpp, fix should from the root cause in the future.
+ if ((cgraph->n_nodes <= 1 && use_count == 0) ||
+ (cgraph->n_nodes <= 1 && node->op == GGML_OP_VIEW && use_count == 1 && node->src[0] != nullptr &&
+ node->src[0]->op == GGML_OP_NONE)) {
+ return false;
+ }
+ if (cgraph->n_nodes == 1 &&
+ (cgraph->nodes[0]->op == GGML_OP_TRANSPOSE || cgraph->nodes[0]->op == GGML_OP_PERMUTE)) {
+ return false;
+ }
+ int input_use_count = 0;
+ for (int j = 0; j < cgraph->n_nodes; j++) {
+ ggml_tensor * other_node = cgraph->nodes[j];
+ for (int k = 0; k < GGML_MAX_SRC; k++) {
+ if (other_node->src[k] == node) {
+ input_use_count++;
}
}
}
- return input_tensor;
- }
-
- if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ||
- GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) {
- ov::Shape input_shape = {1, 1, 1, chunk_size};
- ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape);
- // copy the chunk_index-th chunk from ggml_tensor
- size_t element_size = ggml_type_size(ggml_tensor->type);
- void * input_data = (char *) ggml_tensor->data + chunk_index * chunk_size * element_size;
- std::memcpy(input_tensor.data(), input_data, chunk_valid_size * element_size);
- // pad the rest with last_value + 1, so that kv's of padded positions are inserted
- // to the next row after the valids row in the kvcache
- if (chunk_pad_size > 0) {
- if (ggml_tensor->type == GGML_TYPE_I32) {
- int32_t last_value =
- *((int32_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1));
- int32_t * output_data = input_tensor.data<int32_t>();
- std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1);
- } else if (ggml_tensor->type == GGML_TYPE_I64) {
- int64_t last_value =
- *((int64_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1));
- int64_t * output_data = input_tensor.data<int64_t>();
- std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1);
- } else {
- throw std::runtime_error("Unexpected tensor type for " + param_name);
- }
+ if (use_count != input_use_count && node->op != GGML_OP_NONE) {
+ return true;
}
- return input_tensor;
}
-
- if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) {
- size_t output_len = ggml_decoder->get_compute_params().output_len;
- ov::Shape input_shape = {1, 1, 1, output_len};
- ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape);
- if (ggml_tensor->ne[0] == 0) {
- *input_tensor.data<int32_t>() = 0;
- } else {
- auto * data_addr = input_tensor.data<int32_t>();
- for (size_t i = 0; i < output_len; i++) {
- data_addr[i] = ((int32_t *) ggml_tensor->data)[i] % chunk_size;
- }
+ // if all nodes's src node's src is not come from the nodes in the model, we think the model is splitted. This is a complementary check for the above check, because for some special case like the output node is not used by any node, the use count and input use count are both 0, we can not determine whether the model is splitted or not just based on the first check.
+ // Only weight-name membership is needed below. With GGML_OPENVINO_REDUCE_COMPILE_MEM
+ // use the name-only collector (no weight extraction); otherwise keep the original
+ // behavior of building (naive) weight nodes and take their names.
+ std::set<std::string> model_weights;
+ if (ggml_openvino_reduce_compile_mem_enabled()) {
+ model_weights = GgmlOvDecoder::collect_weight_names(cgraph);
+ } else {
+ for (const auto & kv : GgmlOvDecoder::create_weight_nodes(cgraph, true)) {
+ model_weights.insert(kv.first);
}
- return input_tensor;
}
-
- if (GgmlOvDecoder::is_inp_mean(ggml_tensor, op)) {
- const size_t n_seqs = ggml_tensor->ne[1];
- const size_t src_stride = ggml_tensor->ne[0];
- const size_t copy_len = std::min<size_t>(chunk_valid_size, src_stride - chunk_index * chunk_size);
- ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, n_seqs, chunk_size});
- auto * dst = input_tensor.data<float>();
- std::fill(dst, dst + n_seqs * chunk_size, 0.0f);
- const auto * src = static_cast<const float *>(ggml_tensor->data) + chunk_index * chunk_size;
- for (size_t s = 0; s < n_seqs; s++) {
- std::memcpy(dst + s * chunk_size, src + s * src_stride, copy_len * sizeof(float));
+ std::set<ggml_tensor *> model_nodes(cgraph->nodes, cgraph->nodes + cgraph->n_nodes);
+ // leaf nodes
+ std::set<ggml_tensor *> model_leafs(cgraph->leafs, cgraph->leafs + cgraph->n_leafs);
+ for (int i = 0; i < cgraph->n_nodes; i++) {
+ ggml_tensor * node = cgraph->nodes[i];
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
+ ggml_tensor * src = node->src[j];
+ // the src is also not the model weights, we think the model is splitted.
+ // the src is also not in model leafs, we think the model is splitted.
+ if (src != nullptr && model_nodes.find(src) == model_nodes.end() &&
+ model_weights.find(std::string(src->name)) == model_weights.end() && !model_leafs.empty() == false &&
+ model_leafs.find(src) == model_leafs.end()) {
+ if (GgmlOvDecoder::is_inp_tok(src, node)) {
+ return false;
+ }
+ return true;
+ }
}
- return input_tensor;
}
+ return false;
+}
- if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) {
- size_t cols = ggml_tensor->ne[0];
- size_t rows = ggml_tensor->ne[1];
- size_t chunk_valid_rows = std::min(chunk_size, rows - chunk_index * chunk_size);
- size_t context_size = ggml_decoder->get_ctx_size();
- if (ggml_tensor->type == GGML_TYPE_F16) {
- const auto * ggml_data =
- static_cast<const ggml_fp16_t *>(ggml_tensor->data) + chunk_index * chunk_size * cols;
- std::vector<ggml_fp16_t> padded_data = pad_input<ggml_fp16_t>(ggml_data, chunk_valid_rows, cols, chunk_size,
- context_size, GGML_FP32_TO_FP16(-INFINITY));
- set_zero_diagonal(padded_data, chunk_size, context_size, GGML_FP32_TO_FP16(0.0f));
- ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, chunk_size, context_size});
- std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t));
- return input_tensor;
+bool is_naive(ggml_cgraph * cgraph) {
+ constexpr int naive_graph_size_threshold = 20;
+ int count = 0;
+ for (int i = 0; i < cgraph->n_nodes; i++) {
+ if (cgraph->nodes[i]->op != GGML_OP_NONE) {
+ count++;
}
-
- const auto * ggml_data = static_cast<const float *>(ggml_tensor->data) + chunk_index * chunk_size * cols;
- std::vector<float> padded_data =
- pad_input<float>(ggml_data, chunk_valid_rows, cols, chunk_size, context_size, -INFINITY);
- set_zero_diagonal(padded_data, chunk_size, context_size);
- ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, chunk_size, context_size});
- auto * data_ptr = input_tensor.data<float>();
- std::copy(padded_data.begin(), padded_data.begin() + chunk_size * context_size, data_ptr);
- return input_tensor;
}
-
- return get_ov_input_tensor(ggml_decoder, param_name);
+ return count < naive_graph_size_threshold;
}
size_t checksum(const void * data, size_t size) {
@@ -1651,15 +1645,15 @@ bool save_ggml_tensor_data_to_txt(const ggml_tensor * tensor, const std::string
void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor) {
std::cout << "Input name: " << name << ", Input shape: " << tensor.get_shape() << ", Address: " << tensor.data()
- << std::endl;
+ << '\n';
switch (tensor.get_element_type()) {
case ov::element::f32: {
if (name.find("self_kq_mask") == std::string::npos && name.find("KQ_mask") == std::string::npos) {
- std::cout << *(tensor.data<float>()) << std::endl;
+ std::cout << *(tensor.data<float>()) << '\n';
} else {
size_t rows = tensor.get_shape()[2];
size_t cols = tensor.get_shape()[3];
- auto * data = tensor.data<float>();
+ const float * data = tensor.data<float>();
for (size_t i = 0; i < rows; ++i) {
for (size_t j = 0; j < cols; ++j) {
float val = data[i * cols + j];
@@ -1669,26 +1663,26 @@ void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor
std::cout << std::setw(5) << val;
}
}
- std::cout << std::endl;
+ std::cout << '\n';
}
}
break;
}
case ov::element::f16:
- std::cout << *(tensor.data<ov::float16>()) << std::endl;
+ std::cout << *(tensor.data<ov::float16>()) << '\n';
break;
case ov::element::i32:
for (size_t i = 0; i < tensor.get_size(); ++i) {
- std::cout << tensor.data<int32_t>()[i] << " ";
+ std::cout << tensor.data<int32_t>()[i] << ' ';
}
- std::cout << std::endl;
+ std::cout << '\n';
break;
case ov::element::i64:
for (size_t i = 0; i < tensor.get_size(); ++i) {
- std::cout << tensor.data<int64_t>()[i] << " ";
+ std::cout << tensor.data<int64_t>()[i] << ' ';
}
- std::cout << std::endl;
+ std::cout << '\n';
break;
default:
break;
@@ -1697,7 +1691,7 @@ void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor
void print_output_tensor_info(const std::string & name, const ov::Tensor & tensor, const void * output_dst) {
std::cout << "Output name: " << name << ", Output shape: " << tensor.get_shape() << ", Address: " << output_dst
- << std::endl;
+ << '\n';
auto print_float_stats = [](const std::string & type_name, size_t size, auto get_value) {
if (size == 0) {
@@ -1711,20 +1705,16 @@ void print_output_tensor_info(const std::string & name, const ov::Tensor & tenso
for (size_t i = 1; i < size; ++i) {
float v = get_value(i);
- if (v < min) {
- min = v;
- }
- if (v > max) {
- max = v;
- }
+ min = std::min(v, min);
+ max = std::max(v, max);
sum += v;
}
double mean = sum / size;
std::cout << std::right << std::setw(6) << type_name << std::right << std::setw(12) << "First" << std::setw(12)
- << "Min" << std::setw(12) << "Max" << std::setw(12) << "Mean" << std::endl;
+ << "Min" << std::setw(12) << "Max" << std::setw(12) << "Mean" << '\n';
std::cout << std::right << std::setw(6) << "" << std::right << std::setw(12) << first << std::setw(12) << min
- << std::setw(12) << max << std::setw(12) << mean << std::endl;
+ << std::setw(12) << max << std::setw(12) << mean << '\n';
};
switch (tensor.get_element_type()) {
@@ -1781,5 +1771,3 @@ int64_t get_inp_pos_n_tokens(ggml_cgraph * cgraph, const ggml_tensor * inp_pos)
bool get_is_prefill(ggml_cgraph * cgraph, const ggml_tensor * inp_pos) {
return get_inp_pos_n_tokens(cgraph, inp_pos) > 1;
}
-
-#pragma GCC diagnostic pop
diff --git a/ggml/src/ggml-openvino/utils.h b/ggml/src/ggml-openvino/utils.h
index 235b15d7e..74c25f0ac 100644
--- a/ggml/src/ggml-openvino/utils.h
+++ b/ggml/src/ggml-openvino/utils.h
@@ -142,9 +142,6 @@ struct ov_runtime_context {
enum ggml_status ov_graph_compute(struct ggml_cgraph * cgraph, ggml_backend_t backend);
-enum ggml_status ov_graph_compute_dynamic(struct ggml_cgraph * cgraph, std::shared_ptr<ov_runtime_context> r_ctx);
-enum ggml_status ov_graph_compute_static(struct ggml_cgraph * cgraph, std::shared_ptr<ov_runtime_context> r_ctx);
-
size_t checksum(const void * data, size_t size);
bool save_ggml_tensor_data_to_txt(const ggml_tensor * tensor, const std::string & file_path);
@@ -185,18 +182,6 @@ int64_t get_inp_pos_n_tokens(struct ggml_cgraph * cgraph, const ggml_tensor * in
bool get_is_prefill(struct ggml_cgraph * cgraph, const ggml_tensor * inp_pos);
-ov::Tensor get_ov_input_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder, const std::string & param_name);
-ov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
- const std::string & param_name);
-ov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
- const std::string & param_name,
- int chunk_index);
-
-ov::Tensor create_ov_output_tensor(std::shared_ptr<GgmlOvDecoder> ggml_decoder,
- std::shared_ptr<ov::InferRequest> infer_request,
- int output_index,
- const ggml_tensor * ggml_tensor);
-
bool is_naive(struct ggml_cgraph * cgraph);
/**
@@ -205,9 +190,3 @@ bool is_naive(struct ggml_cgraph * cgraph);
* @return true if the graph is identified as split; otherwise false.
*/
bool is_model_splitted(struct ggml_cgraph * cgraph);
-
-enum ggml_status naive_compute(struct ggml_cgraph * cgraph,
- ov::Core & core,
- const std::string & device,
- const ov::AnyMap & config,
- ov_compiled_model_cache & cache);