Commit 52ffa0274 for llama.cpp
commit 52ffa027455662ea59f78d0e4e5a8f38ee63a5fe
Author: Takamichi Ohara <tokinasin@tutanota.com>
Date: Wed Oct 21 05:48:45 2026 +0900
model : support Nemotron-3(.5)-Embed (#30877)
diff --git a/conversion/__init__.py b/conversion/__init__.py
index efa537b0f..bddfe3c1f 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -200,7 +200,9 @@ TEXT_MODEL_MAP: dict[str, str] = {
"MiniMaxM3SparseForCausalLM": "minimax",
"MiniMaxM3SparseForConditionalGeneration": "minimax",
"Ministral3ForCausalLM": "mistral3",
+ "Ministral3Model": "mistral3",
"Mistral3ForConditionalGeneration": "mistral3",
+ "Mistral3Model": "mistral3",
"VoxtralRealtimeForConditionalGeneration": "mistral3",
"MistralForCausalLM": "llama",
"MixtralForCausalLM": "llama",
@@ -358,6 +360,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"MiniCPMV4_6ForConditionalGeneration": "minicpm",
"MiniCPMV4_7ForConditionalGeneration": "minicpm",
"Mistral3ForConditionalGeneration": "llava",
+ "Mistral3Model": "llava",
"NemotronH_Omni_Reasoning_V3": "nemotron",
"NemotronH_Nano_VL_V2": "nemotron",
"MuseGlimmerForConditionalGeneration": "muse_glimmer",
diff --git a/conversion/llava.py b/conversion/llava.py
index 98a004f98..e193f0b95 100644
--- a/conversion/llava.py
+++ b/conversion/llava.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import json
-from typing import Iterable, TYPE_CHECKING
+from typing import Callable, Iterable, TYPE_CHECKING
if TYPE_CHECKING:
from torch import Tensor
@@ -128,3 +128,24 @@ class LlavaVisionModel(MmprojModel):
yield from super().modify_tensors(img_break_embd, name, bid)
return # skip other tensors
+
+
+@ModelBase.register("Mistral3Model")
+@ModelBase.example("nvidia/Nemotron-3.5-Embed-1B-BF16")
+class Mistral3EmbeddingVisionModel(LlavaVisionModel):
+ def __init__(self, dir_model, *args, **kwargs):
+ hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
+ text_config = hparams.get("text_config", hparams)
+ if text_config.get("is_causal") is not False:
+ raise NotImplementedError("Expected a Nemotron Embed model for Mistral3Model, but is_causal is not False")
+ super().__init__(dir_model, *args, **kwargs)
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ if item[0].endswith("embed_tokens.weight"):
+ return ModelBase.filter_tensors(item)
+ return super().filter_tensors(item)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_vision_preproc_image_size(self.preprocessor_config["size"]["longest_edge"])
diff --git a/conversion/mistral3.py b/conversion/mistral3.py
index 661475251..106c329e7 100644
--- a/conversion/mistral3.py
+++ b/conversion/mistral3.py
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING
if TYPE_CHECKING:
from torch import Tensor
-from .base import ModelBase, TextModel, gguf
+from .base import ModelBase, TextModel, get_model_architecture, gguf
from .deepseek import DeepseekV2Model
from .llama import LlamaModel
@@ -13,9 +13,11 @@ from .llama import LlamaModel
@ModelBase.register(
"Mistral3ForConditionalGeneration",
+ "Mistral3Model",
"Ministral3ForCausalLM",
+ "Ministral3Model",
)
-@ModelBase.example("mistralai/Mistral-Small-3.1-24B-Instruct-2503", "hf-tiny-v2/tiny-random-Ministral3ForCausalLM")
+@ModelBase.example("mistralai/Mistral-Small-3.1-24B-Instruct-2503", "hf-tiny-v2/tiny-random-Ministral3ForCausalLM", "nvidia/Nemotron-3-Embed-1B-BF16", "nvidia/Nemotron-3.5-Embed-1B-BF16")
class Mistral3Model(TextModel):
class Ministral3Model(LlamaModel):
model_arch = gguf.MODEL_ARCH.MISTRAL3
@@ -29,6 +31,18 @@ class Mistral3Model(TextModel):
self.gguf_writer.add_rope_scaling_yarn_log_mul(rope_params["mscale_all_dim"])
self.gguf_writer.add_attn_temperature_scale(rope_params["llama_4_scaling_beta"])
+ class Mistral3EmbedNemotronModel(Ministral3Model):
+ model_arch = gguf.MODEL_ARCH.MISTRAL3
+
+ def set_vocab(self):
+ super().set_vocab()
+ self.gguf_writer.add_add_bos_token(False)
+ self.gguf_writer.add_add_eos_token(False)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self._try_set_pooling_type()
+
class Mistral4Model(DeepseekV2Model):
model_arch = gguf.MODEL_ARCH.MISTRAL4
skip_mtp = False # model contains no MTP layers, so no need to skip
@@ -42,12 +56,20 @@ class Mistral3Model(TextModel):
model_arch = gguf.MODEL_ARCH.MISTRAL3 # unused
impl: TextModel
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
+ def __init__(self, dir_model, *args, **kwargs):
+ hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
+ hf_arch = get_model_architecture(hparams, self.model_type)
+ if hf_arch in ("Ministral3Model", "Mistral3Model"):
+ text_config = hparams.get("text_config", hparams)
+ if text_config.get("is_causal") is not False:
+ raise NotImplementedError(f"Expected a Nemotron Embed model for {hf_arch}, but is_causal is not False")
+ super().__init__(dir_model, *args, **kwargs)
if self.hparams.get("model_type") == "mistral4":
- self.impl = Mistral3Model.Mistral4Model(*args, **kwargs)
+ self.impl = Mistral3Model.Mistral4Model(dir_model, *args, **kwargs)
+ elif self.hf_arch in ("Ministral3Model", "Mistral3Model"):
+ self.impl = Mistral3Model.Mistral3EmbedNemotronModel(dir_model, *args, **kwargs)
else:
- self.impl = Mistral3Model.Ministral3Model(*args, **kwargs)
+ self.impl = Mistral3Model.Ministral3Model(dir_model, *args, **kwargs)
def set_vocab(self):
self.impl.set_vocab()
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h
index 64e006e8d..5caa93b42 100644
--- a/tools/mtmd/clip-model.h
+++ b/tools/mtmd/clip-model.h
@@ -66,6 +66,7 @@ struct clip_hparams {
resize_algo image_resize_algo = RESIZE_ALGO_BICUBIC;
pad_style image_resize_pad = PAD_CEIL; // padding style when resizing
bool image_round_half_even = false; // smart_resize rounds .5 to even, like Python round() in the HF processor
+ bool image_floor_before_align = false;
std::array<uint8_t, 3> image_pad_color = {0, 0, 0};
// (preprocessor) for llava-uhd style models
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 26941c9a8..df61f82f0 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -1625,7 +1625,23 @@ struct clip_model_loader {
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
hparams.rope_theta = 10000.0f;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
- hparams.set_limit_image_tokens(8, 1024);
+ get_u32(KEY_PREPROC_IMAGE_SIZE, hparams.image_longest_edge, false);
+ if (hparams.image_longest_edge > 0) {
+ const int64_t patch_size = int64_t(hparams.patch_size) * hparams.n_merge;
+ if (patch_size <= 0) {
+ throw std::runtime_error("Pixtral merged patch size must be positive");
+ }
+ const int64_t n_tokens_per_side =
+ (hparams.image_longest_edge + patch_size - 1) / patch_size;
+ if (n_tokens_per_side * patch_size > std::sqrt(std::numeric_limits<int>::max())) {
+ throw std::runtime_error("Pixtral preprocessing image size is too large");
+ }
+ hparams.image_resize_pad = PAD_NONE;
+ hparams.image_floor_before_align = true;
+ hparams.set_limit_image_tokens(0, n_tokens_per_side * n_tokens_per_side);
+ } else {
+ hparams.set_limit_image_tokens(8, 1024);
+ }
hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_LIGHTONOCR:
diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp
index e0cd0bf1d..ab6067b97 100644
--- a/tools/mtmd/mtmd-image.cpp
+++ b/tools/mtmd/mtmd-image.cpp
@@ -873,16 +873,28 @@ mtmd_image_preproc_out mtmd_image_preprocessor_glm5v::preprocess(const clip_imag
clip_image_size mtmd_image_preprocessor_longest_edge::get_target_size(const clip_hparams & hparams, const clip_image_size & original_size) {
GGML_ASSERT(hparams.image_longest_edge > 0);
+ clip_image_size size = original_size;
+ if (hparams.image_floor_before_align) {
+ const double ratio = double(std::max(size.width, size.height)) / hparams.image_longest_edge;
+ if (ratio > 1.0) {
+ size.width = static_cast<int>(std::floor(size.width / ratio));
+ size.height = static_cast<int>(std::floor(size.height / ratio));
+ if (size.width <= 0 || size.height <= 0) {
+ throw std::runtime_error("image dimensions must be positive after resizing");
+ }
+ }
+ }
+
// the original pixtral model doesn't have n_merge
const int cur_merge = hparams.n_merge == 0 ? 1 : hparams.n_merge;
return img_tool::calc_size_preserved_ratio(
- original_size,
+ size,
{
/* align_size */ hparams.patch_size * cur_merge,
/* min_pixels */ std::max(0, hparams.image_min_pixels),
/* max_pixels */ std::max(0, hparams.image_max_pixels),
/* longest_edge */ std::min(hparams.image_longest_edge,
- std::max(original_size.width, original_size.height)),
+ std::max(size.width, size.height)),
});
}
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 26d5ee410..b3ac1979e 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -804,7 +804,11 @@ struct mtmd_context {
{
// https://github.com/huggingface/transformers/blob/1cd110c6cb6a6237614130c470e9a902dbc1a4bd/docs/source/en/model_doc/pixtral.md
img_end = "[IMG_END]";
- image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ if (clip_get_hparams(ctx_v)->image_longest_edge > 0) {
+ image_preproc = std::make_unique<mtmd_image_preprocessor_longest_edge>(ctx_v);
+ } else {
+ image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
+ }
} break;
case PROJECTOR_TYPE_PHI4:
{