Commit f711c8f30 for llama.cpp
commit f711c8f3038020d71f69b8ced09162b014fd7335
Author: bosh <98094229+boshjerns@users.noreply.github.com>
Date: Tue Oct 20 06:14:40 2026 +0700
mtmd : match the transformers Qwen-VL resize (#30736)
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp
index dfa3e5e2c..2d0f24874 100644
--- a/tests/test-mtmd-impl.cpp
+++ b/tests/test-mtmd-impl.cpp
@@ -66,6 +66,31 @@ MAKE_TEST(test_image_preprocessor_longest_edge) {
}
}
+MAKE_TEST(test_image_preprocessor_dyn_size) {
+ // { patch_size, image size, expected size }
+ // a side that is an even number plus .5 merged patches rounds down, like Python round() in smart_resize
+ const std::vector<std::tuple<int, clip_image_size, clip_image_size>> cases = {
+ { 16, { 768, 400 }, { 768, 384 } },
+ { 16, { 800, 336 }, { 800, 320 } },
+ { 16, { 400, 400 }, { 384, 384 } },
+ { 16, { 640, 240 }, { 640, 256 } },
+ { 14, { 966, 560 }, { 952, 560 } },
+ };
+
+ for (const auto & [patch_size, input, expected] : cases) {
+ clip_hparams hparams;
+ hparams.patch_size = patch_size;
+ hparams.n_merge = 2;
+ hparams.image_round_half_even = true;
+ hparams.set_limit_image_tokens(8, 4096);
+
+ const clip_image_size actual = mtmd_image_preprocessor_dyn_size::get_target_size(hparams, input);
+ const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height);
+ t.assert_equal(name + " width", expected.width, actual.width);
+ t.assert_equal(name + " height", expected.height, actual.height);
+ }
+}
+
MAKE_TEST(test_image_preprocessor_lfm2) {
clip_hparams hparams;
hparams.patch_size = 16;
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h
index 92ffe5e54..fefbd4f46 100644
--- a/tools/mtmd/clip-model.h
+++ b/tools/mtmd/clip-model.h
@@ -65,6 +65,7 @@ struct clip_hparams {
int32_t image_max_pixels = -1;
resize_algo image_resize_algo = RESIZE_ALGO_BICUBIC;
pad_style image_resize_pad = PAD_CEIL; // padding style when resizing
+ bool image_round_half_even = false; // smart_resize rounds .5 to even, like Python round() in the HF processor
std::array<uint8_t, 3> image_pad_color = {0, 0, 0};
// (preprocessor) for llava-uhd style models
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 793b8cd1a..faeb37702 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -1634,6 +1634,8 @@ struct clip_model_loader {
case PROJECTOR_TYPE_DOTS_OCR:
{
hparams.rope_theta = 10000.0f;
+ hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding
+ hparams.image_round_half_even = true; // smart_resize uses Python round()
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
@@ -1750,6 +1752,8 @@ struct clip_model_loader {
{
hparams.n_merge = 2; // default value for Qwen 2 and 2.5
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
+ hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding
+ hparams.image_round_half_even = true; // smart_resize uses Python round()
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, model.proj_type == PROJECTOR_TYPE_QWEN25VL); // only 2.5 requires it
// ref: https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct/blob/main/preprocessor_config.json
@@ -1825,6 +1829,8 @@ struct clip_model_loader {
{
hparams.n_merge = 2; // spatial_merge_size
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
+ hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding
+ hparams.image_round_half_even = true; // smart_resize uses Python round()
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
// 1D banded sliding-window radius (visual_token_window_size); required
@@ -2092,6 +2098,8 @@ struct clip_model_loader {
case PROJECTOR_TYPE_EXAONE4_5:
{
hparams.n_merge = 2;
+ hparams.image_resize_pad = PAD_NONE; // the HF processor resizes to the smart_resize size, no padding
+ hparams.image_round_half_even = true; // smart_resize uses Python round()
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, false);
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp
index 5debfe6f0..e56f33e77 100644
--- a/tools/mtmd/mtmd-image.cpp
+++ b/tools/mtmd/mtmd-image.cpp
@@ -115,6 +115,7 @@ struct img_tool {
int max_pixels = 0; // 0 = disabled
// applied before min/max_pixels, so min_pixels can push an edge back above longest_edge
int longest_edge = 0; // 0 = disabled
+ bool round_half_even = false; // Python round() instead of std::round
};
// calculate the size of the **resized** image, while preserving the aspect ratio and
@@ -127,7 +128,10 @@ struct img_tool {
return {0, 0};
}
- auto round_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::round(x / static_cast<float>(f))) * f; };
+ auto round_by_factor = [f = opts.align_size, even = opts.round_half_even](float x) {
+ const float q = x / static_cast<float>(f);
+ return static_cast<int>(even ? std::nearbyint(q) : std::round(q)) * f;
+ };
auto ceil_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::ceil(x / static_cast<float>(f))) * f; };
auto floor_by_factor = [f = opts.align_size](float x) { return static_cast<int>(std::floor(x / static_cast<float>(f))) * f; };
@@ -767,20 +771,24 @@ mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip
// mtmd_image_preprocessor_dyn_size
//
-mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const {
+clip_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const clip_hparams & hparams, const clip_image_size & original_size) {
GGML_ASSERT(hparams.image_min_pixels > 0 && hparams.image_max_pixels > 0);
- clip_image_u8 resized_image;
- const clip_image_size original_size = img.get_size();
// the original pixtral model doesn't have n_merge
const int cur_merge = hparams.n_merge;
- const clip_image_size target_size = img_tool::calc_size_preserved_ratio(
+ return img_tool::calc_size_preserved_ratio(
original_size,
{
- /* align_size */ hparams.patch_size * cur_merge,
- /* min_pixels */ hparams.image_min_pixels,
- /* max_pixels */ hparams.image_max_pixels,
- /* longest_edge */ 0,
+ /* align_size */ hparams.patch_size * cur_merge,
+ /* min_pixels */ hparams.image_min_pixels,
+ /* max_pixels */ hparams.image_max_pixels,
+ /* longest_edge */ 0,
+ /* round_half_even */ hparams.image_round_half_even,
});
+}
+
+mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const {
+ clip_image_u8 resized_image;
+ const clip_image_size target_size = get_target_size(hparams, img.get_size());
img_tool::resize(img, resized_image, target_size,
hparams.image_resize_algo,
hparams.image_resize_pad,
diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h
index 1798ce51d..26193e4e4 100644
--- a/tools/mtmd/mtmd-image.h
+++ b/tools/mtmd/mtmd-image.h
@@ -131,6 +131,7 @@ struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor {
// this is used by models with native support for dynamic image size, for example: Qwen-VL, Pixtral, Kimi-VL, etc
struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor {
mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
+ static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size);
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
};