Commit cf24317be for llama.cpp
commit cf24317bea431b807dc93319d74c61bed6e20aff
Author: Pascal <admin@serveurperso.com>
Date: Wed Oct 28 23:15:01 2026 +0100
common, mtmd: bound control vector layer indices and M-RoPE position loops (#31336)
* common: bound control vector layer indices by the model layer count
A direction tensor layer index is parsed from the tensor name with no
upper bound, and the buffer size and write offset are computed in int.
A large index overflows both, so the accumulation writes before the
start of the buffer.
Pass the model layer count to the loader, reject indices above it like
the existing zero index check, and compute the sizes in size_t.
* mtmd: fill M-RoPE positions by whole merge tiles
The position loops of the Qwen-VL, GLM, PaddleOCR and Qwen2.5-VL
encoders hardcode a 2x2 merge inside a stride of n_merge, so a patch
grid that is not a multiple of the merge writes past the positions
buffer. Iterate over whole merge tiles with n_merge inner steps, which
bounds the writes by the patch count and gives the same positions on
aligned grids.
PaddleOCR also requires a merge aligned image at graph build, as the
Qwen-VL family already does.
diff --git a/common/common.cpp b/common/common.cpp
index 221f0897c..edd829d56 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1473,7 +1473,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1;
if (params.control_vector_layer_end <= 0) params.control_vector_layer_end = llama_model_n_layer(model);
- const auto cvec = common_control_vector_load(params.control_vectors);
+ const auto cvec = common_control_vector_load(params.control_vectors, llama_model_n_layer(model));
if (cvec.n_embd == -1) {
return res;
}
@@ -1995,7 +1995,7 @@ float common_embd_similarity_cos(const float * embd1, const float * embd2, int n
// Control vector utils
//
-static common_control_vector_data common_control_vector_load_one(const common_control_vector_load_info & load_info) {
+static common_control_vector_data common_control_vector_load_one(const common_control_vector_load_info & load_info, int32_t n_layer) {
common_control_vector_data result = { -1, {} };
ggml_context * ctx = nullptr;
@@ -2036,6 +2036,10 @@ static common_control_vector_data common_control_vector_load_one(const common_co
COM_ERR("invalid (zero) direction tensor layer index in %s\n", load_info.fname.c_str());
result.n_embd = -1;
break;
+ } else if (layer_idx > n_layer) {
+ COM_ERR("direction tensor layer index %d exceeds the model layer count %d in %s\n", layer_idx, n_layer, load_info.fname.c_str());
+ result.n_embd = -1;
+ break;
}
struct ggml_tensor * tensor = ggml_get_tensor(ctx, name.c_str());
@@ -2059,10 +2063,10 @@ static common_control_vector_data common_control_vector_load_one(const common_co
}
// extend if necessary - do not store data for layer 0 (it's not used)
- result.data.resize(std::max(result.data.size(), static_cast<size_t>(result.n_embd * layer_idx)), 0.0f);
+ result.data.resize(std::max(result.data.size(), (size_t) result.n_embd * layer_idx), 0.0f);
const float * src = (const float *) tensor->data;
- float * dst = result.data.data() + result.n_embd * (layer_idx - 1); // layer 1 at [0]
+ float * dst = result.data.data() + (size_t) result.n_embd * (layer_idx - 1); // layer 1 at [0]
for (int j = 0; j < result.n_embd; j++) {
dst[j] += src[j] * load_info.strength; // allows multiple directions for same layer in same file
}
@@ -2080,11 +2084,11 @@ static common_control_vector_data common_control_vector_load_one(const common_co
return result;
}
-common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos) {
+common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos, int32_t n_layer) {
common_control_vector_data result = { -1, {} };
for (const auto & info : load_infos) {
- auto cur = common_control_vector_load_one(info);
+ auto cur = common_control_vector_load_one(info, n_layer);
if (cur.n_embd == -1) {
result.n_embd = -1;
diff --git a/common/common.h b/common/common.h
index cd89a27e9..ab08d1591 100644
--- a/common/common.h
+++ b/common/common.h
@@ -1237,8 +1237,9 @@ struct common_control_vector_load_info {
};
// Load control vectors, scale each by strength, and add them together.
+// Direction layer indices must be in [1, n_layer].
// On error, returns {-1, empty}
-common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos);
+common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos, int32_t n_layer);
//
// Split utils
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 1f6498efd..767ad9f79 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -5685,18 +5685,20 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
case PROJECTOR_TYPE_GLM5V:
{
const int merge_ratio = hparams.n_merge;
- const int pw = image_size_width / patch_size;
- const int ph = image_size_height / patch_size;
+ const int pw = image_size_width / patch_size / merge_ratio;
+ const int ph = image_size_height / patch_size / merge_ratio;
std::vector<int> positions(n_pos * 4);
int ptr = 0;
- for (int y = 0; y < ph; y += merge_ratio) {
- for (int x = 0; x < pw; x += merge_ratio) {
- for (int dy = 0; dy < 2; dy++) {
- for (int dx = 0; dx < 2; dx++) {
- positions[ ptr] = y + dy;
- positions[ num_patches + ptr] = x + dx;
- positions[2 * num_patches + ptr] = y + dy;
- positions[3 * num_patches + ptr] = x + dx;
+ for (int ty = 0; ty < ph; ty++) {
+ for (int tx = 0; tx < pw; tx++) {
+ for (int dy = 0; dy < merge_ratio; dy++) {
+ for (int dx = 0; dx < merge_ratio; dx++) {
+ const int y = ty * merge_ratio + dy;
+ const int x = tx * merge_ratio + dx;
+ positions[ ptr] = y;
+ positions[ num_patches + ptr] = x;
+ positions[2 * num_patches + ptr] = y;
+ positions[3 * num_patches + ptr] = x;
ptr++;
}
}
@@ -5720,19 +5722,21 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
case PROJECTOR_TYPE_PADDLEOCR:
{
const int merge_ratio = hparams.n_merge;
- const int pw = image_size_width / patch_size;
- const int ph = image_size_height / patch_size;
+ const int pw = image_size_width / patch_size / merge_ratio;
+ const int ph = image_size_height / patch_size / merge_ratio;
std::vector<int> positions(n_pos * 4);
int ptr = 0;
// NOTE: same as Qwen-VL, but x and y are swapped
- for (int y = 0; y < ph; y += merge_ratio) {
- for (int dy = 0; dy < 2; dy++) {
- for (int x = 0; x < pw; x += merge_ratio) {
- for (int dx = 0; dx < 2; dx++) {
- positions[ ptr] = y + dy;
- positions[ num_patches + ptr] = x + dx;
- positions[2 * num_patches + ptr] = y + dy;
- positions[3 * num_patches + ptr] = x + dx;
+ for (int ty = 0; ty < ph; ty++) {
+ for (int dy = 0; dy < merge_ratio; dy++) {
+ for (int tx = 0; tx < pw; tx++) {
+ for (int dx = 0; dx < merge_ratio; dx++) {
+ const int y = ty * merge_ratio + dy;
+ const int x = tx * merge_ratio + dx;
+ positions[ ptr] = y;
+ positions[ num_patches + ptr] = x;
+ positions[2 * num_patches + ptr] = y;
+ positions[3 * num_patches + ptr] = x;
ptr++;
}
}
@@ -5850,17 +5854,19 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
std::vector<int> positions(n_pos * 4);
int ptr = 0;
- for (int y = 0; y < iph; y += merge_ratio) {
- for (int x = 0; x < ipw; x += merge_ratio) {
- for (int dy = 0; dy < 2; dy++) {
- for (int dx = 0; dx < 2; dx++) {
+ for (int ty = 0; ty < ph; ty++) {
+ for (int tx = 0; tx < pw; tx++) {
+ for (int dy = 0; dy < merge_ratio; dy++) {
+ for (int dx = 0; dx < merge_ratio; dx++) {
auto remap = idx[ptr / mpow];
remap = (remap * mpow) + (ptr % mpow);
- positions[ remap] = y + dy;
- positions[ num_patches + remap] = x + dx;
- positions[2 * num_patches + remap] = y + dy;
- positions[3 * num_patches + remap] = x + dx;
+ const int y = ty * merge_ratio + dy;
+ const int x = tx * merge_ratio + dx;
+ positions[ remap] = y;
+ positions[ num_patches + remap] = x;
+ positions[2 * num_patches + remap] = y;
+ positions[3 * num_patches + remap] = x;
ptr++;
}
}
diff --git a/tools/mtmd/models/paddleocr.cpp b/tools/mtmd/models/paddleocr.cpp
index 5d3a13fb5..0e6b3f4b6 100644
--- a/tools/mtmd/models/paddleocr.cpp
+++ b/tools/mtmd/models/paddleocr.cpp
@@ -1,6 +1,9 @@
#include "models.h"
ggml_cgraph * clip_graph_paddleocr::build() {
+ GGML_ASSERT(img.nx() % (patch_size * model.hparams.n_merge) == 0);
+ GGML_ASSERT(img.ny() % (patch_size * model.hparams.n_merge) == 0);
+
const int n_pos = n_patches;
const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position