Commit 19cce6919 for llama.cpp
commit 19cce6919992fdffd92e0ac152caaf1e2b8f254e
Author: Tarek Dakhran <tarek@liquid.ai>
Date: Tue Oct 27 17:06:40 2026 +0100
model : add LiquidAI/LFM3-1.7B and LiquidAI/LFM3-VL-2B (#31252)
* model : add LiquidAI/LFM3-1.7B and LiquidAI/LFM3-VL-2B
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
Co-authored-by: Xuan-Son Nguyen <son@huggingface.co>
Assisted-by: Claude Opus 5.5
* tests : set the SWA rope base of the LFM3 test model
Assisted-by: Claude Opus 5.5
* llama : enable recurrent state rollback for LFM3
Assisted-by: Claude Opus 5.5
* Apply batched suggestions from code review
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
Co-authored-by: Tarek Dakhran <github@dakhran.com>
* gguf-py : align the lfm3vl mapping comments
Assisted-by: Claude Opus 5.5
* llama : fix CRLF line endings from the review suggestions
Assisted-by: Claude Opus 5.5
* tests : add the LFM3 Metal fusion baseline
Assisted-by: Claude Opus 5.5
* llama : keep the LFM3-VL depthwise conv kernels in F32 when quantizing
Assisted-by: Claude Opus 5.5
---------
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
Co-authored-by: Xuan-Son Nguyen <son@huggingface.co>
diff --git a/conversion/__init__.py b/conversion/__init__.py
index b40936edd..4e5c5f4c6 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -172,6 +172,8 @@ TEXT_MODEL_MAP: dict[str, str] = {
"D1OmniModel": "lfm2",
"Lfm2Model": "lfm2",
"Lfm2MoeForCausalLM": "lfm2",
+ "Lfm3ForCausalLM": "lfm3",
+ "Lfm3VlForConditionalGeneration": "lfm3",
"Llama4ForCausalLM": "llama",
"Llama4ForConditionalGeneration": "llama",
"LlamaBidirectionalModel": "llama",
@@ -358,6 +360,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"Lfm2AudioForConditionalGeneration": "lfm2",
"D1OmniModel": "lfm2",
"Lfm2VlForConditionalGeneration": "lfm2",
+ "Lfm3VlForConditionalGeneration": "lfm3",
"LightOnOCRForConditionalGeneration": "lighton_ocr",
"Llama4ForConditionalGeneration": "llama4",
"LlavaForConditionalGeneration": "llava",
diff --git a/conversion/lfm3.py b/conversion/lfm3.py
new file mode 100644
index 000000000..79ff6a196
--- /dev/null
+++ b/conversion/lfm3.py
@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+from typing import Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import MmprojModel, ModelBase, TextModel, gguf
+
+
+@ModelBase.register("Lfm3ForCausalLM", "Lfm3VlForConditionalGeneration")
+@ModelBase.example("LiquidAI/LFM3-1.7B", "LiquidAI/LFM3-1.7B-Base")
+class LFM3Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.LFM3
+
+ def set_gguf_parameters(self):
+ layer_types = self.hparams["layer_types"]
+ n_kv_heads = self.hparams["num_key_value_heads"]
+
+ # set num_key_value_heads only for attention layers
+ self.hparams["num_key_value_heads"] = [
+ n_kv_heads if lt != "conv" else 0
+ for lt in layer_types
+ ]
+
+ super().set_gguf_parameters()
+
+ self.gguf_writer.add_rope_dimension_count(self.hparams["head_dim"])
+
+ self.gguf_writer.add_sliding_window_pattern([lt == "sliding_attention" for lt in layer_types])
+ self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
+
+ self.gguf_writer.add_shortconv_l_cache(self.hparams["conv_L_cache"])
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ # Partial RoPE is represented by masking trailing frequency pairs.
+ # rope_parameters is keyed by layer type; only full_attention layers use partial RoPE.
+ rope_parameters = self.hparams["rope_parameters"]["full_attention"]
+ partial = rope_parameters.get("partial_rotary_factor", 1.0)
+ head_dim = self.hparams["head_dim"]
+ rope_angles = int(partial * head_dim) // 2
+ n_pairs = head_dim // 2
+ assert 0 < rope_angles <= n_pairs
+ values = [1.0] * rope_angles + [1e30] * (n_pairs - rope_angles)
+ rope_freqs = torch.tensor(values, dtype=torch.float32)
+ yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), rope_freqs)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # conv op requires 2d tensor
+ if "conv.conv" in name:
+ data_torch = data_torch.squeeze(1)
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
+@ModelBase.register("Lfm3VlForConditionalGeneration")
+@ModelBase.example("LiquidAI/LFM3-VL-2B")
+class LFM3VLMmprojModel(MmprojModel):
+ has_vision_encoder = True
+ has_audio_encoder = False
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ assert self.hparams_vision is not None
+ # dynamic resolution, image_size is only set for MmprojModel compatibility
+ self.hparams_vision["image_size"] = 512
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ assert self.hparams_vision is not None
+ # clip.cpp hardcodes the vision rope base, the RGB patchifier, the 3x3 spatial conv and the GELU projector
+ if (self.hparams_vision["rope_parameters"]["rope_theta"] != 1e6 or self.hparams_vision["in_channels"] != 3
+ or self.hparams_vision["conv_kernel_size"] != 3):
+ raise ValueError("unsupported vision rope_theta / in_channels / conv_kernel_size")
+ if (self.global_config["projector_hidden_act"] != "gelu" or not self.global_config["projector_bias"]
+ or self.global_config["projector_layer_norm_eps"] != 1e-6):
+ raise ValueError("unsupported projector_hidden_act / projector_bias / projector_layer_norm_eps")
+ self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.LFM3VL)
+ self.gguf_writer.add_vision_head_count_kv(self.hparams_vision["num_key_value_heads"])
+ self.gguf_writer.add_vision_spatial_merge_size(self.global_config["downsample_factor"])
+ self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision["norm_eps"])
+ self.gguf_writer.add_vision_use_silu(True)
+ size = self.preprocessor_config["size"]
+ self.gguf_writer.add_vision_min_pixels(size["shortest_edge"])
+ self.gguf_writer.add_vision_max_pixels(size["longest_edge"])
+ # PIL resample enum, the torchvision backend has no lanczos
+ resample = self.preprocessor_config["resample"]
+ resize_algo = {2: "bilinear", 3: "bicubic"}.get(resample)
+ if resize_algo is None:
+ raise ValueError(f"unsupported resample: {resample}")
+ self.gguf_writer.add_vision_image_resize_algo(resize_algo)
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ if "patch_embd" in new_name:
+ return gguf.GGMLQuantizationType.F16
+ if "conv2d_spatial" in new_name:
+ # CUDA CONV_2D_DW requires an f32 kernel; the tensor is tiny
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if not name.startswith(("model.vision_tower.", "model.multi_modal_projector.")):
+ return
+
+ if name == "model.vision_tower.patchifier.weight":
+ # qwen2vl-style layout [channel, temporal, ph, pw], one conv kernel per temporal slice
+ assert self.hparams_vision is not None
+ t = self.hparams_vision["temporal_patch_size"]
+ p = self.hparams_vision["patch_size"]
+ n_embd = data_torch.shape[0]
+ w = data_torch.view(n_embd, 3, t, p, p)
+ yield ("v.patch_embd.weight", w[:, :, 0].contiguous())
+ if t == 2:
+ yield ("v.patch_embd.weight.1", w[:, :, 1].contiguous())
+ else:
+ assert t == 1, f"unsupported vision_temporal_patch_size {t}"
+ return
+ if name == "model.vision_tower.embedding_norm.weight":
+ yield ("v.embedding_norm.weight", data_torch)
+ return
+ if "conv.in_proj.weight" in name:
+ yield (f"v.blk.{bid}.conv2d_in_proj.weight", data_torch)
+ return
+ if "conv.spatial_conv.weight" in name:
+ yield (f"v.blk.{bid}.conv2d_spatial.weight", data_torch)
+ return
+ if "conv.out_proj.weight" in name:
+ yield (f"v.blk.{bid}.conv2d_out_proj.weight", data_torch)
+ return
+
+ # projector: input LayerNorm -> linear_1 -> GELU -> linear_2
+ if name.startswith("model.multi_modal_projector."):
+ layer, suffix = name.removeprefix("model.multi_modal_projector.").split(".")
+ yield ({"layer_norm": "mm.input_norm", "linear_1": "mm.0", "linear_2": "mm.2"}[layer] + "." + suffix, data_torch)
+ return
+
+ yield (self.map_tensor_name(name), data_torch)
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 8648d6b7e..d94551a4f 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -691,6 +691,7 @@ class MODEL_ARCH(IntEnum):
GPT_OSS = auto()
LFM2 = auto()
LFM2MOE = auto()
+ LFM3 = auto()
DREAM = auto()
SMALLTHINKER = auto()
LLADA = auto()
@@ -1523,6 +1524,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.GPT_OSS: "gpt-oss",
MODEL_ARCH.LFM2: "lfm2",
MODEL_ARCH.LFM2MOE: "lfm2moe",
+ MODEL_ARCH.LFM3: "lfm3",
MODEL_ARCH.DREAM: "dream",
MODEL_ARCH.SMALLTHINKER: "smallthinker",
MODEL_ARCH.LLADA: "llada",
@@ -5522,6 +5524,29 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_UP_EXP,
MODEL_TENSOR.FFN_EXP_PROBS_B,
],
+ MODEL_ARCH.LFM3: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ROPE_FREQS,
+ MODEL_TENSOR.SHORTCONV_CONV,
+ MODEL_TENSOR.SHORTCONV_INPROJ,
+ MODEL_TENSOR.SHORTCONV_OUTPROJ,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_POST_NORM,
+ MODEL_TENSOR.ATTN_NORM, # operator_norm
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_GATE,
+ ],
MODEL_ARCH.SMALLTHINKER: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
@@ -6507,6 +6532,7 @@ class VisionProjectorType:
VOXTRAL_RT = "voxtral_rt"
MERALION = "meralion" # audio: Whisper + gated MLP adaptor
LFM2 = "lfm2"
+ LFM3VL = "lfm3vl"
KIMIVL = "kimivl"
PADDLEOCR = "paddleocr"
KIMIK25 = "kimik25"
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index 79c9add78..beacc5a67 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -119,6 +119,7 @@ class TensorNameMap:
"model.transformer.ln_f", # llada
"final_norm", # modern-bert
"model.norm", # cogvlm
+ "model.embedding_norm", # lfm3
),
# Rope frequencies
@@ -418,6 +419,7 @@ class TensorNameMap:
"model.layers.{bid}.post_self_attn_layernorm", # glm-4-0414
"model.layers.{bid}.post_attn_norm", # kolibri1
"model.layers.layers.{bid}.post_mixer_norm.weight", # plamo2
+ "model.layers.{bid}.post_operator_norm", # lfm3
),
# Rotary embeddings
@@ -504,6 +506,7 @@ class TensorNameMap:
"model.layers.layers.{bid}.post_mlp_norm.weight", # plamo2
"model.layers.{bid}.feed_forward.up_proj",
"model.layers.{bid}.post_moe_norm", # grok-2
+ "model.layers.{bid}.post_ffn_norm", # lfm3
),
MODEL_TENSOR.FFN_POST_NORM_1: (
@@ -1855,6 +1858,7 @@ class TensorNameMap:
"vision_model.model.layers.{bid}.self_attn.q_proj", # llama4
"vision_tower.transformer.layers.{bid}.attention.q_proj", # pixtral-hf
"vision_encoder.transformer.layers.{bid}.attention.wq", # pixtral
+ "model.vision_tower.layers.{bid}.self_attn.q_proj", # lfm3vl
"visual.blocks.{bid}.attn.q", # qwen2vl, generated
"vision_tower.encoder.blocks.{bid}.wq", # kimi-vl, generated
"siglip2.vision_model.encoder.layers.{bid}.self_attn.q_proj", # youtuvl
@@ -1870,6 +1874,7 @@ class TensorNameMap:
"model.vision_tower.encoder.layer.{bid}.attention.q_norm", # Intern-S1
"visual.blocks.{bid}.attn.q_norm", # GLM-OCR
"vision_model.model.layers.{bid}.self_attn.q_norm", # gemma4
+ "model.vision_tower.layers.{bid}.self_attn.q_layernorm", # lfm3vl
),
MODEL_TENSOR.V_ENC_ATTN_K: (
@@ -1884,6 +1889,7 @@ class TensorNameMap:
"vision_model.model.layers.{bid}.self_attn.k_proj", # llama4
"vision_tower.transformer.layers.{bid}.attention.k_proj", # pixtral-hf
"vision_encoder.transformer.layers.{bid}.attention.wk", # pixtral
+ "model.vision_tower.layers.{bid}.self_attn.k_proj", # lfm3vl
"visual.blocks.{bid}.attn.k", # qwen2vl, generated
"vision_tower.encoder.blocks.{bid}.wk", # kimi-vl, generated
"model.vision_model.transformer.layers.{bid}.self_attn.k_proj", # Deepseek-OCR CLIP, generated
@@ -1899,6 +1905,7 @@ class TensorNameMap:
"model.vision_tower.encoder.layer.{bid}.attention.k_norm", # Intern-S1
"visual.blocks.{bid}.attn.k_norm", # GLM-OCR
"vision_model.model.layers.{bid}.self_attn.k_norm", # gemma4
+ "model.vision_tower.layers.{bid}.self_attn.k_layernorm", # lfm3vl
),
MODEL_TENSOR.V_ENC_ATTN_V: (
@@ -1913,6 +1920,7 @@ class TensorNameMap:
"vision_model.model.layers.{bid}.self_attn.v_proj", # llama4
"vision_tower.transformer.layers.{bid}.attention.v_proj", # pixtral-hf
"vision_encoder.transformer.layers.{bid}.attention.wv", # pixtral
+ "model.vision_tower.layers.{bid}.self_attn.v_proj", # lfm3vl
"visual.blocks.{bid}.attn.v", # qwen2vl, generated
"vision_tower.encoder.blocks.{bid}.wv", # kimi-vl, generated
"siglip2.vision_model.encoder.layers.{bid}.self_attn.v_proj",
@@ -1935,6 +1943,7 @@ class TensorNameMap:
"vit.layers.{bid}.input_layernorm", # HunyuanVL
"vision_tower.transformer.layers.{bid}.attention_norm", # pixtral-hf
"vision_encoder.transformer.layers.{bid}.attention_norm", # pixtral
+ "model.vision_tower.layers.{bid}.operator_norm", # lfm3vl
"vision_model.model.layers.{bid}.input_layernorm", # llama4, gemma4
"visual.blocks.{bid}.norm1", # qwen2vl
"vision_tower.encoder.blocks.{bid}.norm0", # kimi-vl (norm0/norm1)
@@ -1964,6 +1973,7 @@ class TensorNameMap:
"vision_model.model.layers.{bid}.self_attn.o_proj", # llama4
"vision_tower.transformer.layers.{bid}.attention.o_proj", # pixtral-hf
"vision_encoder.transformer.layers.{bid}.attention.wo", # pixtral
+ "model.vision_tower.layers.{bid}.self_attn.out_proj", # lfm3vl
"visual.blocks.{bid}.attn.proj", # qwen2vl
"vision_tower.encoder.blocks.{bid}.wo", # kimi-vl
"model.vision.transformer.layers.{bid}.attention.dense", # cogvlm
@@ -1996,6 +2006,7 @@ class TensorNameMap:
"vision_model.model.layers.{bid}.post_attention_layernorm", # llama4
"vision_tower.transformer.layers.{bid}.ffn_norm", # pixtral-hf
"vision_encoder.transformer.layers.{bid}.ffn_norm", # pixtral
+ "model.vision_tower.layers.{bid}.ffn_norm", # lfm3vl
"visual.blocks.{bid}.norm2", # qwen2vl
"vision_tower.encoder.blocks.{bid}.norm1", # kimi-vl (norm0/norm1)
"model.vision.transformer.layers.{bid}.post_attention_layernorm", # cogvlm
@@ -2023,6 +2034,7 @@ class TensorNameMap:
"vit.layers.{bid}.mlp.dense_h_to_4h", # HunyuanVL
"vision_tower.transformer.layers.{bid}.feed_forward.up_proj", # pixtral-hf
"vision_encoder.transformer.layers.{bid}.feed_forward.w3", # pixtral
+ "model.vision_tower.layers.{bid}.feed_forward.w3", # lfm3vl
"vision_model.model.layers.{bid}.mlp.fc1", # llama4
"visual.blocks.{bid}.mlp.fc1", # qwen2vl
"visual.blocks.{bid}.mlp.up_proj", # qwen2.5vl
@@ -2043,6 +2055,7 @@ class TensorNameMap:
"vision_encoder.blocks.{bid}.mlp.fc1", # dots3note
"vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf
"vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral
+ "model.vision_tower.layers.{bid}.feed_forward.w1", # lfm3vl
"visual.blocks.{bid}.mlp.gate_proj", # qwen2.5vl
"vision_model.model.layers.{bid}.mlp.gate_proj", # gemma4
"model.qwen2_model.model.model.layers.{bid}.mlp.gate_proj", # Deepseek-OCR-2 qwen2
@@ -2061,6 +2074,7 @@ class TensorNameMap:
"vit.layers.{bid}.mlp.dense_4h_to_h", # HunyuanVL
"vision_tower.transformer.layers.{bid}.feed_forward.down_proj", # pixtral-hf
"vision_encoder.transformer.layers.{bid}.feed_forward.w2", # pixtral
+ "model.vision_tower.layers.{bid}.feed_forward.w2", # lfm3vl
"vision_model.model.layers.{bid}.mlp.fc2", # llama4
"visual.blocks.{bid}.mlp.fc2", # qwen2vl
"visual.blocks.{bid}.mlp.down_proj", # qwen2.5vl
diff --git a/models/templates/LFM3.jinja b/models/templates/LFM3.jinja
new file mode 100644
index 000000000..d0747b8b5
--- /dev/null
+++ b/models/templates/LFM3.jinja
@@ -0,0 +1,280 @@
+{{- bos_token -}}
+{%- set preserve_thinking = preserve_thinking | default(false) -%}
+{%- set enable_thinking = enable_thinking | default(true) -%}
+
+{%- macro render_value(v) -%}
+ {%- if v is string -%}
+ {{- v -}}
+ {%- elif v is boolean -%}
+ {{- "true" if v else "false" -}}
+ {%- elif v is none -%}
+ {{- "null" -}}
+ {%- elif v is mapping or (v is sequence and v is not string) -%}
+ {{- v | tojson -}}
+ {%- else -%}
+ {{- v -}}
+ {%- endif -%}
+{%- endmacro -%}
+
+{%- macro parse_content(content) -%}
+ {%- if content is string -%}
+ {{- content -}}
+ {%- elif content is mapping and content.get("type") == "image" -%}
+ {{- "<|vision_start|><|image|><|vision_end|>" -}}
+ {%- elif content is mapping and content.get("type") == "video" -%}
+ {{- "<|vision_start|><|video|><|vision_end|>" -}}
+ {%- elif content is mapping and content.get("type") == "text" -%}
+ {{- (content.get("text") or "") | string -}}
+ {%- elif content is mapping -%}
+ {{- content | tojson -}}
+ {%- elif content is iterable -%}
+ {%- set _ns = namespace(result="") -%}
+ {%- for item in content -%}
+ {%- if item is string -%}
+ {%- set _ns.result = _ns.result + item -%}
+ {%- elif item is mapping and item.get("type") == "image" -%}
+ {%- set _ns.result = _ns.result + "<|vision_start|><|image|><|vision_end|>" -%}
+ {%- elif item is mapping and item.get("type") == "video" -%}
+ {%- set _ns.result = _ns.result + "<|vision_start|><|video|><|vision_end|>" -%}
+ {%- elif item is mapping and item.get("type") == "text" -%}
+ {%- set _ns.result = _ns.result + ((item.get("text") or "") | string) -%}
+ {%- else -%}
+ {%- set _ns.result = _ns.result + (item | tojson) -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {{- _ns.result -}}
+ {%- elif content is not none -%}
+ {{- content | tojson -}}
+ {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_tool_calls(tool_calls) -%}
+ {%- for tool_call in tool_calls -%}
+ {%- if not loop.first -%}{{- "\n\n" -}}{%- endif -%}
+ {%- set func = tool_call["function"] if "function" in tool_call else tool_call -%}
+ {%- set func_args = func.get("arguments") -%}
+ {%- if func_args is string and (func_args | trim) not in ["", "{}", "null"] -%}
+ {{- raise_exception("Tool call arguments must be a mapping, got a JSON-encoded string: parse arguments with json.loads() before applying the chat template") -}}
+ {%- endif -%}
+ {{- "<tool_call>\n<function=" + func["name"] + ">\n" -}}
+ {%- if func_args is mapping -%}
+ {%- for arg_name, arg_value in func_args.items() -%}
+ {{- "<parameter=" + arg_name + ">\n" + render_value(arg_value) + "\n</parameter>\n" -}}
+ {%- endfor -%}
+ {%- endif -%}
+ {{- "</function>\n</tool_call>" -}}
+ {%- endfor -%}
+{%- endmacro -%}
+
+{%- macro tool_to_json(tool) -%}
+ {%- set fn = tool.function if tool.function is defined else tool -%}
+ {%- set ns_t = namespace(first=true) -%}
+ {{- "{" -}}
+ {%- for k, v in fn.items() -%}
+ {%- if k != "defer_loading" -%}
+ {%- if not ns_t.first -%}{{- ", " -}}{%- endif -%}
+ {%- set ns_t.first = false -%}
+ {{- k | tojson -}}{{- ": " -}}{{- v | tojson -}}
+ {%- endif -%}
+ {%- endfor -%}
+ {{- "}" -}}
+{%- endmacro -%}
+
+{%- macro is_tool_reference(item) -%}
+ {{- "1" if item is mapping and item.get("type") == "tool_reference" else "" -}}
+{%- endmacro -%}
+
+{%- macro referenced_tool_to_json(name) -%}
+ {%- set _ns = namespace(found=none) -%}
+ {%- for tool in tools or [] -%}
+ {%- set fn = tool.function if tool.function is defined else tool -%}
+ {%- if fn.name == name -%}
+ {%- set _ns.found = fn -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- if _ns.found is none -%}
+ {{- raise_exception("Tool reference '" + name | string + "' not found in available tools. Every tool referenced by a tool_reference must be in `tools` on every request.") -}}
+ {%- endif -%}
+ {{- tool_to_json(_ns.found) -}}
+{%- endmacro -%}
+
+{%- macro render_tool_response(content) -%}
+ {%- if content is string and "<tool_response>" in content -%}
+ {{- content -}}
+ {%- else -%}
+ {{- "<tool_response>\n" -}}
+ {%- if content is string -%}
+ {{- content -}}
+ {%- elif content is mapping -%}
+ {{- parse_content(content) -}}
+ {%- elif content is iterable -%}
+ {%- for item in content -%}
+ {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
+ {%- if is_tool_reference(item) -%}
+ {%- if loop.first or not is_tool_reference(loop.previtem) -%}
+ {{- "<additional_tools>\n" -}}
+ {%- endif -%}
+ {{- referenced_tool_to_json(item.name) -}}
+ {%- if loop.last or not is_tool_reference(loop.nextitem) -%}
+ {{- "\n</additional_tools>" -}}
+ {%- endif -%}
+ {%- elif item is mapping and item.result is defined and item.result is mapping and item.result.tools is defined -%}
+ {{- "<additional_tools>\n" -}}
+ {%- set _tl = item.result.tools -%}
+ {%- if _tl is string -%}
+ {{- _tl -}}
+ {%- elif _tl is mapping -%}
+ {{- tool_to_json(_tl) -}}
+ {%- else -%}
+ {%- for t in _tl -%}
+ {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
+ {{- tool_to_json(t) -}}
+ {%- endfor -%}
+ {%- endif -%}
+ {{- "\n</additional_tools>" -}}
+ {%- else -%}
+ {{- parse_content(item) -}}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- elif content is not none -%}
+ {{- content | tojson -}}
+ {%- endif -%}
+ {{- "\n</tool_response>" -}}
+ {%- endif -%}
+{%- endmacro -%}
+
+{%- set ns = namespace(system_prompt="", last_user_index=-1) -%}
+{%- if messages and messages[0]["role"] == "system" -%}
+ {%- if messages[0].get("content") -%}
+ {%- set ns.system_prompt = parse_content(messages[0]["content"]) -%}
+ {%- endif -%}
+ {%- set messages = messages[1:] -%}
+{%- endif -%}
+{%- if tools -%}
+ {%- set tools_ns = namespace(lines=[]) -%}
+ {%- for tool in tools -%}
+ {%- set fn = tool.function if tool.function is defined else tool -%}
+ {%- set _dyn = (fn.defer_loading is defined and fn.defer_loading) or (tool.defer_loading is defined and tool.defer_loading) -%}
+ {%- if not _dyn -%}
+ {%- set tools_ns.lines = tools_ns.lines + [tool_to_json(fn)] -%}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- if tools_ns.lines -%}
+ {%- set ns.system_prompt = ns.system_prompt + ("\n\n" if ns.system_prompt else "") + "<tools>\n" + (tools_ns.lines | join("\n")) + "\n</tools>" -%}
+ {%- endif -%}
+{%- endif -%}
+{%- if ns.system_prompt -%}
+ {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
+{%- endif -%}
+{%- for message in messages -%}
+ {%- if message["role"] == "user" -%}
+ {%- set ns.last_user_index = loop.index0 -%}
+ {%- endif -%}
+{%- endfor -%}
+{%- set carry = namespace(pending=[]) -%}
+{%- for message in messages -%}
+ {%- set _next = messages[loop.index0 + 1] if not loop.last else none -%}
+ {%- set _merge = message.role == "tool" and not message.get("additional_tools")
+ and _next is not none and _next.role == "tool" and not _next.get("additional_tools")
+ and message.get("tool_call_id") and _next.get("tool_call_id") == message.get("tool_call_id") -%}
+ {%- set _items = [] -%}
+ {%- if message.role == "tool" and not message.get("additional_tools") -%}
+ {%- set _c = message.get("content") -%}
+ {%- if _c is string -%}
+ {%- set _items = [{"type": "text", "text": _c}] if _c else [] -%}
+ {%- elif _c is mapping -%}
+ {%- set _items = [_c] -%}
+ {%- elif _c is iterable -%}
+ {%- set _items = _c | list -%}
+ {%- elif _c is not none -%}
+ {%- set _items = [_c] -%}
+ {%- endif -%}
+ {%- endif -%}
+ {%- if _merge -%}
+ {%- set carry.pending = carry.pending + _items -%}
+ {%- else -%}
+ {{- "<|im_start|>" + message.role + "\n" -}}
+ {%- if message.role == "assistant" -%}
+ {%- generation -%}
+ {%- set thinking = message.thinking if message.thinking is defined else message.reasoning if message.reasoning is defined else message.reasoning_content -%}
+ {%- set keep_thinking = preserve_thinking or loop.index0 > ns.last_user_index -%}
+ {%- if thinking is defined and keep_thinking -%}
+ {{- "<think>" + (thinking or "") + "</think>" -}}
+ {%- elif keep_thinking and "</think>" not in (message.get("content") if message.get("content") is string else "") -%}
+ {{- "<think></think>" -}}
+ {%- endif -%}
+ {%- set _cfm_tag = "CONTINUE_FINAL_MESSAGE_TAG " -%}
+ {%- set _has_cfm = false -%}
+ {%- set content = "" -%}
+ {%- if message.get("content") -%}
+ {%- set content = parse_content(message.content) -%}
+ {%- endif -%}
+ {%- if not keep_thinking -%}
+ {%- if "</think>" in content -%}
+ {%- set content = content.split("</think>")[-1] | trim -%}
+ {%- endif -%}
+ {%- set content = "<think></think>" + content -%}
+ {%- endif -%}
+ {%- if content.endswith(_cfm_tag) -%}
+ {%- set _has_cfm = true -%}
+ {%- set _trunc_len = (content | length) - (_cfm_tag | length) -%}
+ {%- set content = content[:_trunc_len] -%}
+ {%- endif -%}
+ {{- content -}}
+ {%- if message.tool_calls -%}
+ {{- render_tool_calls(message.tool_calls) -}}
+ {%- endif -%}
+ {%- if _has_cfm -%}
+ {{- _cfm_tag -}}
+ {%- endif -%}
+ {{- "<|im_end|>\n" -}}
+ {%- endgeneration -%}
+ {%- elif message.role == "tool" -%}
+ {%- if message.get("additional_tools") -%}
+ {%- set _at = message["additional_tools"] -%}
+ {%- if _at is mapping or (_at is sequence and _at is not string) -%}
+ {%- set _src = _at -%}
+ {%- else -%}
+ {%- set _src = message.get("content") -%}
+ {%- endif -%}
+ {%- if _src is string and "<tool_response>" in _src -%}
+ {{- _src -}}
+ {%- elif _src is string and "<additional_tools>" in _src -%}
+ {{- "<tool_response>\n" + _src + "\n</tool_response>" -}}
+ {%- elif _src -%}
+ {{- "<tool_response>\n<additional_tools>\n" -}}
+ {%- if _src is string -%}
+ {{- _src -}}
+ {%- elif _src is mapping -%}
+ {{- tool_to_json(_src) -}}
+ {%- else -%}
+ {%- for t in _src -%}
+ {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
+ {{- tool_to_json(t) -}}
+ {%- endfor -%}
+ {%- endif -%}
+ {{- "\n</additional_tools>\n</tool_response>" -}}
+ {%- endif -%}
+ {%- elif carry.pending -%}
+ {{- render_tool_response(carry.pending + _items) -}}
+ {%- set carry.pending = [] -%}
+ {%- else -%}
+ {{- render_tool_response("" if message.get("content") is none else message.get("content")) -}}
+ {%- endif -%}
+ {{- "<|im_end|>\n" -}}
+ {%- else %}
+ {%- if message.get("content") -%}
+ {{- parse_content(message["content"]) -}}
+ {%- endif -%}
+ {{- "<|im_end|>\n" -}}
+ {%- endif %}
+ {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+ {{- "<|im_start|>assistant\n" -}}
+ {%- if enable_thinking -%}
+ {{- "<think>" -}}
+ {%- else -%}
+ {{- "<think></think>" -}}
+ {%- endif -%}
+{%- endif -%}
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index fe3ad3eb7..497e42692 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -135,6 +135,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_OPENAI_MOE, "gpt-oss" },
{ LLM_ARCH_LFM2, "lfm2" },
{ LLM_ARCH_LFM2MOE, "lfm2moe" },
+ { LLM_ARCH_LFM3, "lfm3" },
{ LLM_ARCH_DREAM, "dream" },
{ LLM_ARCH_SMALLTHINKER, "smallthinker" },
{ LLM_ARCH_LLADA, "llada" },
@@ -1251,6 +1252,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
case LLM_ARCH_GRANITE_HYBRID:
case LLM_ARCH_LFM2:
case LLM_ARCH_LFM2MOE:
+ case LLM_ARCH_LFM3:
case LLM_ARCH_NEMOTRON_H:
case LLM_ARCH_NEMOTRON_H_MOE:
case LLM_ARCH_QWEN3NEXT:
@@ -1295,6 +1297,7 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) {
case LLM_ARCH_NEMOTRON_H_MOE:
case LLM_ARCH_LFM2:
case LLM_ARCH_LFM2MOE:
+ case LLM_ARCH_LFM3:
case LLM_ARCH_BAILINGMOE3:
return true;
default:
@@ -1341,6 +1344,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
case LLM_ARCH_NEMOTRON_H:
case LLM_ARCH_NEMOTRON_H_MOE:
case LLM_ARCH_GRANITE_HYBRID:
+ case LLM_ARCH_LFM3:
case LLM_ARCH_MINIMAX_01:
case LLM_ARCH_MINIMAX_M2:
case LLM_ARCH_MINIMAX_M3:
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 7996bdc30..eca6d18af 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -139,6 +139,7 @@ enum llm_arch {
LLM_ARCH_OPENAI_MOE,
LLM_ARCH_LFM2,
LLM_ARCH_LFM2MOE,
+ LLM_ARCH_LFM3,
LLM_ARCH_DREAM,
LLM_ARCH_SMALLTHINKER,
LLM_ARCH_LLADA,
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 0220c5ab8..c7b2fcc93 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -314,6 +314,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_lfm2(params);
case LLM_ARCH_LFM2MOE:
return new llama_model_lfm2moe(params);
+ case LLM_ARCH_LFM3:
+ return new llama_model_lfm3(params);
case LLM_ARCH_SMALLTHINKER:
return new llama_model_smallthinker(params);
case LLM_ARCH_GROVEMOE:
@@ -3637,6 +3639,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_HY_V3:
case LLM_ARCH_LFM2:
case LLM_ARCH_LFM2MOE:
+ case LLM_ARCH_LFM3:
case LLM_ARCH_SMALLTHINKER:
case LLM_ARCH_SEED_OSS:
case LLM_ARCH_GROVEMOE:
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
index bdc4f653f..5ebd57807 100644
--- a/src/llama-quant.cpp
+++ b/src/llama-quant.cpp
@@ -342,6 +342,9 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
quantize &= name.find("ssm_conv1d") == std::string::npos;
quantize &= name.find("shortconv.conv.weight") == std::string::npos;
+ // do not quantize LFM3-VL's depthwise conv kernels, CONV_2D_DW takes F32 kernels
+ quantize &= name.find("conv2d_spatial.weight") == std::string::npos;
+
// do not quantize MiniMax's indexer projection weights, they are tiny
quantize &= name.find("indexer.k_proj.weight") == std::string::npos;
quantize &= name.find("indexer.q_proj.weight") == std::string::npos;
diff --git a/src/models/lfm3.cpp b/src/models/lfm3.cpp
new file mode 100644
index 000000000..77eabc467
--- /dev/null
+++ b/src/models/lfm3.cpp
@@ -0,0 +1,197 @@
+#include "models.h"
+#include "../llama-memory-hybrid-iswa.h"
+
+void llama_model_lfm3::load_arch_hparams(llama_model_loader & ml) {
+ ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
+ hparams.is_recr_impl[il] = hparams.n_head_kv(il) == 0;
+ }
+
+ // LFM3 always mixes full and sliding window attention layers
+ ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+ if (!hparams.is_swa_any()) {
+ throw std::runtime_error("LFM3 requires sliding window attention layers");
+ }
+ hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
+ ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+
+ ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa);
+
+ switch (hparams.n_ff()) {
+ case 7680: type = LLM_TYPE_1_7B; break;
+ default: type = LLM_TYPE_UNKNOWN;
+ }
+}
+
+void llama_model_lfm3::load_arch_tensors(llama_model_loader &) {
+ LLAMA_LOAD_LOCALS;
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+ if (output == NULL) {
+ output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+ }
+
+ int rope_freqs_flag = 0;
+
+ for (int i = 0; i < n_layer; ++i) {
+ auto & layer = layers[i];
+
+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+ // absent when use_sandwich_norm is false
+ layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED);
+ layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED);
+ if ((layer.attn_post_norm == nullptr) != (layer.ffn_post_norm == nullptr)) {
+ throw std::runtime_error("LFM3 requires both post norms or neither");
+ }
+
+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0);
+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
+
+ if (!hparams.is_recr(i)) {
+ // note: n_embd_head_k * n_head may differ from n_embd
+ const int64_t n_embd_q = n_embd_head_k * n_head;
+ const int64_t n_embd_k = hparams.n_embd_k_gqa(i);
+ const int64_t n_embd_v = hparams.n_embd_v_gqa(i);
+ if (n_embd_k != n_embd_v) {
+ throw std::runtime_error("LFM3 requires equal key and value sizes");
+ }
+
+ layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
+ layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
+
+ create_tensor_qkv(layer, i, n_embd, n_embd_q, n_embd_k, n_embd_v, 0);
+
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q, n_embd}, 0);
+
+ layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head}, 0);
+
+ if (!hparams.is_swa(i)) {
+ layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_embd_head_k / 2}, rope_freqs_flag);
+ rope_freqs_flag = TENSOR_DUPLICATED;
+ }
+ } else {
+ layer.shortconv.conv = create_tensor(tn(LLM_TENSOR_SHORTCONV_CONV, "weight", i), {hparams.n_shortconv_l_cache, n_embd}, 0);
+ layer.shortconv.in_proj = create_tensor(tn(LLM_TENSOR_SHORTCONV_INPROJ, "weight", i), {n_embd, 3 * n_embd}, 0);
+ layer.shortconv.out_proj = create_tensor(tn(LLM_TENSOR_SHORTCONV_OUTPROJ, "weight", i), {n_embd, n_embd}, 0);
+ }
+ }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_lfm3::build_arch_graph(const llm_graph_params & params) const {
+ return std::make_unique<graph>(*this, params);
+}
+
+llama_model_lfm3::graph::graph(const llama_model & model, const llm_graph_params & params) :
+ llm_build_lfm_base(params) {
+ auto build_attn_block = [&model, this](ggml_tensor * cur_in,
+ ggml_tensor * inp_pos,
+ llm_graph_input_attn_kv_iswa * inp_attn,
+ int il) -> ggml_tensor * {
+ const int64_t n_embd_head = hparams.n_embd_head_k(il);
+ const int64_t n_head_il = hparams.n_head(il);
+ const int64_t n_head_kv = hparams.n_head_kv(il);
+
+ const int n_rot_l = hparams.n_rot(il);
+ const float freq_base_l = model.get_rope_freq_base (cparams, il);
+ const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
+
+ ggml_tensor * rope_factors = hparams.is_swa(il) ? nullptr : model.layers[il].rope_freqs;
+
+ auto [q, k, v] = build_qkv(model.layers[il], cur_in, n_embd_head, n_head_il, n_head_kv, il);
+
+ q = build_norm(q, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
+ cb(q, "model.layers.{}.self_attn.q_layernorm", il);
+ k = build_norm(k, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
+ cb(k, "model.layers.{}.self_attn.k_layernorm", il);
+
+ q = ggml_rope_ext(ctx0, q, inp_pos, rope_factors, n_rot_l, rope_type, n_ctx_orig,
+ freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow);
+ k = ggml_rope_ext(ctx0, k, inp_pos, rope_factors, n_rot_l, rope_type, n_ctx_orig,
+ freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow);
+
+ ggml_tensor * attn_out = build_attn(inp_attn,
+ NULL, NULL, NULL,
+ q, k, v, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
+ cb(attn_out, "attn_out", il);
+
+ ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, cur_in);
+ gate = ggml_sigmoid(ctx0, gate);
+ cb(gate, "model.layers.{}.self_attn.gate", il);
+
+ ggml_tensor * attn_3d = ggml_reshape_3d(ctx0, attn_out, n_embd_head, n_head_il, n_tokens);
+ ggml_tensor * gate_3d = ggml_reshape_3d(ctx0, gate, 1, n_head_il, n_tokens);
+ attn_3d = ggml_mul(ctx0, attn_3d, gate_3d);
+ attn_out = ggml_reshape_2d(ctx0, attn_3d, n_embd_head * n_head_il, n_tokens);
+
+ ggml_tensor * out = build_lora_mm(model.layers[il].wo, attn_out, model.layers[il].wo_s);
+ cb(out, "model.layers.{}.self_attn.out_proj", il);
+
+ return out;
+ };
+
+ ggml_tensor * cur = build_inp_embd(model.tok_embd);
+ cb(cur, "model.embed_tokens", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ auto * inp_hybrid = build_inp_mem_hybrid_iswa();
+
+ ggml_tensor * inp_pos = build_inp_pos();
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ for (int il = 0; il < n_layer; ++il) {
+ auto * prev_cur = cur;
+
+ cur = build_norm(cur, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
+ cb(cur, "model.layers.{}.operator_norm", il);
+
+ if (hparams.is_recr(il)) {
+ const auto * mctx_recr = static_cast<const llama_memory_hybrid_iswa_context *>(mctx)->get_recr();
+ cur = build_lfm_shortconv(model, mctx_recr, inp_hybrid->get_recr(), cur, il);
+ } else {
+ cur = build_attn_block(cur, inp_pos, inp_hybrid->get_attn(), il);
+ }
+
+ if (model.layers[il].attn_post_norm) {
+ cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);
+ cb(cur, "model.layers.{}.post_operator_norm", il);
+ }
+
+ if (il == n_layer - 1 && inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ prev_cur = ggml_get_rows(ctx0, prev_cur, inp_out_ids);
+ }
+
+ cur = ggml_add(ctx0, prev_cur, cur);
+
+ auto * ffn_norm_out = build_norm(cur, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);
+ cb(ffn_norm_out, "model.layers.{}.ffn_norm", il);
+
+ ggml_tensor * ffn_out = build_lfm_dense_ffn(model, ffn_norm_out, il);
+
+ if (model.layers[il].ffn_post_norm) {
+ ffn_out = build_norm(ffn_out, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il);
+ cb(ffn_out, "model.layers.{}.post_ffn_norm", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_out);
+
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+ }
+
+ cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ cur = build_lora_mm(model.output, cur, model.output_s);
+ cb(cur, "result_output", -1);
+
+ res->t_logits = cur;
+}
diff --git a/src/models/models.h b/src/models/models.h
index 8f2048a31..5cd69b59f 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -2433,6 +2433,19 @@ struct llama_model_lfm2moe : public llama_model_base {
};
+struct llama_model_lfm3 : public llama_model_base {
+ llama_model_lfm3(const struct llama_model_params & params) : llama_model_base(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ struct graph : public llm_build_lfm_base {
+ graph(const llama_model & model, const llm_graph_params & params);
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
struct llama_model_smallthinker : public llama_model_base {
llama_model_smallthinker(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv
index 2c57c2a14..42fa5c912 100644
--- a/tests/fusion/MTL.csv
+++ b/tests/fusion/MTL.csv
@@ -155,6 +155,7 @@ laguna ,0 ,any ,RMS_NORM+MUL , 9
lfm2 ,0 ,any ,RMS_NORM+MUL , 7
lfm2moe ,1 ,any ,MUL+ADD , 2
lfm2moe ,1 ,any ,RMS_NORM+MUL , 7
+lfm3 ,0 ,any ,RMS_NORM+MUL , 17
llada ,0 ,any ,RMS_NORM+MUL , 5
llada-moe ,1 ,any ,MUL+ADD , 4
llada-moe ,1 ,any ,RMS_NORM+MUL , 9
diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp
index 10a477fa4..9340c7d7d 100644
--- a/tests/test-chat.cpp
+++ b/tests/test-chat.cpp
@@ -5589,6 +5589,47 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
test_lfm2_parser(tmpl, detailed_debug);
}
+ {
+ auto tst = peg_tester("models/templates/LFM3.jinja", detailed_debug);
+
+ tst.test("Hello, world!\nWhat's up?")
+ .enable_thinking(false)
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .expect(message_assist)
+ .run();
+
+ tst.test("I'm\nthinking</think>Hello, world!\nWhat's up?")
+ .enable_thinking(true)
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .expect(message_assist_thoughts)
+ .run();
+
+ tst.test(
+ "<tool_call>\n"
+ "<function=special_function>\n"
+ "<parameter=arg1>\n1\n</parameter>\n"
+ "</function>\n"
+ "</tool_call>")
+ .enable_thinking(false)
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .tools({ special_function_tool })
+ .expect(message_assist_call)
+ .run();
+
+ tst.test(
+ "I'm\nthinking</think>"
+ "<tool_call>\n"
+ "<function=special_function>\n"
+ "<parameter=arg1>\n1\n</parameter>\n"
+ "</function>\n"
+ "</tool_call>")
+ .enable_thinking(true)
+ .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+ .tools({ special_function_tool })
+ .expect(message_assist_call_thoughts)
+ .run();
+ }
+
// Thinking cases only apply to LFM2.5-8B-A1B, the one LFM2 template that emits <think>
{
auto tst = peg_tester("models/templates/LFM2.5-8B-A1B.jinja", detailed_debug);
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 52d843f51..aff6a7c90 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -251,6 +251,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
n_head = 1;
n_ff = 96;
n_layer = 22; // hparams.n_layer_kv_from_start = 20 is hardcoded
+ } else if (arch == LLM_ARCH_LFM3) {
+ n_layer = 3;
} else if (arch == LLM_ARCH_DEEPSEEK4) {
// head size 64 so that GPU flash attention kernels support the model
n_embd = 512;
@@ -336,8 +338,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, uint32_t(2));
if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE ||
- arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_KIMI_LINEAR ||
- arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3 || arch == LLM_ARCH_GLM5_NEXT) {
+ arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_LFM3 ||
+ arch == LLM_ARCH_KIMI_LINEAR || arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3 || arch == LLM_ARCH_GLM5_NEXT
+ ) {
GGML_ASSERT(n_layer >= 2);
std::vector<uint32_t> n_head_per_layer;
n_head_per_layer.reserve(n_layer);
@@ -439,6 +442,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
if (arch == LLM_ARCH_FLINT) {
ms.add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, pattern);
}
+ } else if (arch == LLM_ARCH_LFM3) {
+ ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector<uint32_t>({0, 0, 1}));
+ ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f);
} else {
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(2));
}
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index 8ca4ae3a4..7c054c0c6 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -58,6 +58,7 @@ add_library(mtmd
models/sensenova-u1.cpp
models/minimax-m3.cpp
models/qwen3vl.cpp
+ models/lfm3vl.cpp
models/ling3vl.cpp
models/mimovl.cpp
models/qwen3a.cpp
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index 661f0c646..141954f10 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -173,6 +173,12 @@
#define TN_DEEPSTACK_FC1 "v.deepstack.%d.fc1.%s" // qwen3vl deepstack
#define TN_DEEPSTACK_FC2 "v.deepstack.%d.fc2.%s" // qwen3vl deepstack
+// lfm3vl
+#define TN_LFM3VL_EMBEDDING_NORM "%s.embedding_norm.%s"
+#define TN_LFM3VL_CONV2D_IN_PROJ "%s.blk.%d.conv2d_in_proj.%s"
+#define TN_LFM3VL_CONV2D_SPATIAL "%s.blk.%d.conv2d_spatial.%s"
+#define TN_LFM3VL_CONV2D_OUT_PROJ "%s.blk.%d.conv2d_out_proj.%s"
+
// mimicpmv
#define TN_MINICPMV_POS_EMBD_K "resampler.pos_embed_k"
#define TN_MINICPMV_QUERY "resampler.query"
@@ -517,6 +523,7 @@ enum projector_type {
PROJECTOR_TYPE_MERALION,
PROJECTOR_TYPE_MUSIC_FLAMINGO,
PROJECTOR_TYPE_LFM2,
+ PROJECTOR_TYPE_LFM3VL,
PROJECTOR_TYPE_D1OMNI_V,
PROJECTOR_TYPE_KIMIVL,
PROJECTOR_TYPE_PADDLEOCR,
@@ -595,6 +602,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_MERALION, "meralion"},
{ PROJECTOR_TYPE_MUSIC_FLAMINGO, "musicflamingo"},
{ PROJECTOR_TYPE_LFM2, "lfm2"},
+ { PROJECTOR_TYPE_LFM3VL, "lfm3vl"},
{ PROJECTOR_TYPE_D1OMNI_V, "d1omni_v"},
{ PROJECTOR_TYPE_KIMIVL, "kimivl"},
{ PROJECTOR_TYPE_PADDLEOCR, "paddleocr"},
diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h
index 1f319843b..2c5cf1fed 100644
--- a/tools/mtmd/clip-model.h
+++ b/tools/mtmd/clip-model.h
@@ -325,6 +325,11 @@ struct clip_layer {
ggml_tensor * deepstack_fc2_w = nullptr;
ggml_tensor * deepstack_fc2_b = nullptr;
+ // lfm3vl conv layers
+ ggml_tensor * conv2d_in_proj_w = nullptr;
+ ggml_tensor * conv2d_spatial_w = nullptr; // may be nullptr
+ ggml_tensor * conv2d_out_proj_w = nullptr;
+
// sam rel_pos
ggml_tensor * rel_pos_w = nullptr;
ggml_tensor * rel_pos_h = nullptr;
@@ -786,6 +791,9 @@ struct clip_model {
ggml_tensor * yasa_vision_pos_embed = nullptr;
std::vector<yasa2_stage> yasa_stages;
+ // lfm3vl
+ ggml_tensor * lfm3vl_embedding_norm_w = nullptr;
+
// pixtral, glm4v
ggml_tensor * token_embd_img_break = nullptr;
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index 204b16f3e..0ec65e4c1 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -1031,6 +1031,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_sensenova_u1>(ctx, img);
} break;
+ case PROJECTOR_TYPE_LFM3VL:
+ {
+ builder = std::make_unique<clip_graph_lfm3vl>(ctx, img);
+ } break;
case PROJECTOR_TYPE_LING3VL:
{
builder = std::make_unique<clip_graph_ling3vl>(ctx, img);
@@ -1876,6 +1880,39 @@ struct clip_model_loader {
}
hparams.set_warmup_n_tokens(16*16);
} break;
+ case PROJECTOR_TYPE_LFM3VL:
+ {
+ std::string resize_algo;
+ get_string(KEY_IMAGE_RESIZE_ALGO, resize_algo);
+ // Lfm3VlImageProcessor resizes with torchvision
+ if (resize_algo == "bilinear") {
+ hparams.set_image_resize_algo(RESIZE_ALGO_BILINEAR, true);
+ } else if (resize_algo == "bicubic") {
+ hparams.set_image_resize_algo(RESIZE_ALGO_BICUBIC, true);
+ } else {
+ throw std::runtime_error(string_format("%s: unsupported image resize algo: %s\n", __func__, resize_algo.c_str()));
+ }
+ // smart_resize stretches to the aligned size, no letterbox
+ hparams.image_resize_pad = PAD_NONE;
+ get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge);
+ // token counts and the projector graph assume a 2x2 merge
+ if (hparams.n_merge != 2) {
+ throw std::runtime_error(string_format("%s: unsupported spatial_merge_size %d for lfm3vl, only 2 is supported\n", __func__, hparams.n_merge));
+ }
+ get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
+ hparams.rope_theta = 1000000.0f; // vision_rope_theta
+ // HF processor size; the max is also capped at 4096 tokens as an OOM guard
+ int min_pixels = 0;
+ int max_pixels = 0;
+ get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels);
+ get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels);
+ if (min_pixels <= 0 || max_pixels < min_pixels) {
+ throw std::runtime_error(string_format("%s: invalid image min/max pixels %d/%d\n", __func__, min_pixels, max_pixels));
+ }
+ const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
+ hparams.set_limit_image_tokens(min_pixels / patch_area, std::min(4096, max_pixels / patch_area));
+ hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
+ } break;
case PROJECTOR_TYPE_MINIMAX_M3:
{
hparams.n_merge = 2; // spatial_merge_size
@@ -2515,6 +2552,9 @@ struct clip_model_loader {
model.position_embeddings = get_tensor(string_format(TN_POS_EMBD, prefix), false);
+ // lfm3vl
+ model.lfm3vl_embedding_norm_w = get_tensor(string_format(TN_LFM3VL_EMBEDDING_NORM, prefix, "weight"), false);
+
const bool has_standard_layers = (
model.proj_type != PROJECTOR_TYPE_GEMMA3NV &&
model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC &&
@@ -2527,10 +2567,12 @@ struct clip_model_loader {
model.layers.resize(n_layers_to_load);
for (int il = 0; il < n_layers_to_load; ++il) {
auto & layer = model.layers[il];
+ // lfm3vl conv layers have no attention tensors
+ const bool o_w_required = model.proj_type != PROJECTOR_TYPE_LFM3VL;
layer.k_w = get_tensor(string_format(TN_ATTN_K, prefix, il, "weight"), false);
layer.q_w = get_tensor(string_format(TN_ATTN_Q, prefix, il, "weight"), false);
layer.v_w = get_tensor(string_format(TN_ATTN_V, prefix, il, "weight"), false);
- layer.o_w = get_tensor(string_format(TN_ATTN_OUTPUT, prefix, il, "weight"));
+ layer.o_w = get_tensor(string_format(TN_ATTN_OUTPUT, prefix, il, "weight"), o_w_required);
layer.qkv_w = get_tensor(string_format(TN_ATTN_QKV, prefix, il, "weight"), false);
layer.k_norm = get_tensor(string_format(TN_ATTN_K_NORM, prefix, il, "weight"), false);
layer.q_norm = get_tensor(string_format(TN_ATTN_Q_NORM, prefix, il, "weight"), false);
@@ -2580,6 +2622,11 @@ struct clip_model_loader {
model.n_deepstack_layers++;
}
+ // lfm3vl conv layers
+ layer.conv2d_in_proj_w = get_tensor(string_format(TN_LFM3VL_CONV2D_IN_PROJ, prefix, il, "weight"), false);
+ layer.conv2d_spatial_w = get_tensor(string_format(TN_LFM3VL_CONV2D_SPATIAL, prefix, il, "weight"), false);
+ layer.conv2d_out_proj_w = get_tensor(string_format(TN_LFM3VL_CONV2D_OUT_PROJ, prefix, il, "weight"), false);
+
// some models already exported with legacy (incorrect) naming which is quite messy, let's fix it here
// note: Qwen model converted from the old surgery script has n_ff = 0, so we cannot use n_ff to check!
bool is_ffn_swapped = (
@@ -2764,6 +2811,24 @@ struct clip_model_loader {
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
+ case PROJECTOR_TYPE_LFM3VL:
+ {
+ model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);
+ model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B);
+ model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
+ model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
+ model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
+ model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
+ // each layer must be either a conv block or a complete attention block
+ for (size_t il = 0; il < model.layers.size(); ++il) {
+ const auto & layer = model.layers[il];
+ const bool is_conv = layer.conv2d_in_proj_w != nullptr;
+ const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w;
+ if (is_conv == is_attn) {
+ throw std::runtime_error(string_format("%s: lfm3vl layer %zu is neither a complete conv nor a complete attention block\n", __func__, il));
+ }
+ }
+ } break;
case PROJECTOR_TYPE_LING3VL:
{
model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM); // merger.norm
@@ -4452,6 +4517,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_HUNYUANVL:
case PROJECTOR_TYPE_YOUTUVL:
case PROJECTOR_TYPE_MUSE_GLIMMER:
+ case PROJECTOR_TYPE_LFM3VL:
return (img->nx() / params.patch_size) / 2;
case PROJECTOR_TYPE_STEP3VL:
case PROJECTOR_TYPE_MINICPMV4_6:
@@ -4484,6 +4550,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_HUNYUANVL:
case PROJECTOR_TYPE_YOUTUVL:
case PROJECTOR_TYPE_MUSE_GLIMMER:
+ case PROJECTOR_TYPE_LFM3VL:
return (img->ny() / params.patch_size) / 2;
case PROJECTOR_TYPE_STEP3VL:
case PROJECTOR_TYPE_MINICPMV4_6:
@@ -4571,6 +4638,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
case PROJECTOR_TYPE_GLM5V:
case PROJECTOR_TYPE_YOUTUVL:
case PROJECTOR_TYPE_MUSE_GLIMMER:
+ case PROJECTOR_TYPE_LFM3VL:
{
// dynamic size (2 conv, so double patch size)
int x_patch = img->nx() / (params.patch_size * 2);
@@ -5377,6 +5445,24 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
set_input_i32("merger_ds_idx_2", m_ds_2);
set_input_i32("merger_ds_idx_3", m_ds_3);
} break;
+ case PROJECTOR_TYPE_LFM3VL:
+ {
+ // patches in raster-scan order (no interleave before encoder layers)
+ const int pw = image_size_width / patch_size;
+ const int ph = image_size_height / patch_size;
+ std::vector<int> positions(n_pos * 4);
+ int ptr = 0;
+ for (int y = 0; y < ph; y++) {
+ for (int x = 0; x < pw; x++) {
+ positions[ ptr] = y;
+ positions[ num_patches + ptr] = x;
+ positions[2 * num_patches + ptr] = y;
+ positions[3 * num_patches + ptr] = x;
+ ptr++;
+ }
+ }
+ set_input_i32("positions", positions);
+ } break;
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_LING3VL:
@@ -6603,6 +6689,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
+ case PROJECTOR_TYPE_LFM3VL:
case PROJECTOR_TYPE_EXAONE4_5:
case PROJECTOR_TYPE_JANUS_PRO:
case PROJECTOR_TYPE_YOUTUVL:
diff --git a/tools/mtmd/models/lfm3vl.cpp b/tools/mtmd/models/lfm3vl.cpp
new file mode 100644
index 000000000..e8e455b11
--- /dev/null
+++ b/tools/mtmd/models/lfm3vl.cpp
@@ -0,0 +1,182 @@
+#include "models.h"
+
+ggml_cgraph * clip_graph_lfm3vl::build() {
+ GGML_ASSERT(model.patch_embeddings_0 != nullptr);
+ GGML_ASSERT(model.lfm3vl_embedding_norm_w != nullptr);
+ GGML_ASSERT(model.class_embedding == nullptr);
+ GGML_ASSERT(n_batch == 1);
+
+ const int batch_size = 1;
+ const int n_pos = n_patches;
+ const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position
+
+ const norm_type norm_t = NORM_TYPE_RMS;
+
+ // 2D rope: pairs (j, j+n_dims); first d_head/4 pairs rotate by y, next d_head/4 by x
+ int mrope_sections[4] = {d_head/4, d_head/4, 0, 0};
+
+ ggml_tensor * inp_raw = build_inp_raw();
+
+ // one patchifier kernel per temporal slice: a still image fills both slices, so both kernels apply to it
+ ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
+ if (model.patch_embeddings_1) {
+ inp = ggml_add(ctx0, inp, ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
+ }
+ inp = ggml_reshape_2d(ctx0, inp, n_pos, n_embd);
+ inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
+ cb(inp, "patchifier_out", -1);
+
+ // the 2x2 interleave waits for the projector, the conv layers need raster-scan order
+ inp = build_norm(inp, model.lfm3vl_embedding_norm_w, nullptr, norm_t, eps, -1);
+ cb(inp, "embedding_norm", -1);
+
+ ggml_tensor * inpL = inp;
+
+ // position IDs for 2D RoPE (same format as Qwen3VL)
+ ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);
+ ggml_set_name(positions, "positions");
+ ggml_set_input(positions);
+
+ const float rope_theta = hparams.rope_theta;
+
+ for (int il = 0; il < n_layer; il++) {
+ auto & layer = model.layers[il];
+
+ const bool is_conv_layer = (layer.conv2d_in_proj_w != nullptr);
+
+ ggml_tensor * cur = inpL;
+
+ cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
+ cb(cur, "ln1", il);
+
+ if (is_conv_layer) {
+ ggml_tensor * proj = build_mm(layer.conv2d_in_proj_w, cur);
+ cb(proj, "conv_in_proj", il);
+
+ ggml_tensor * first = ggml_view_2d(ctx0, proj, n_embd, n_pos,
+ proj->nb[1], 0);
+ ggml_tensor * middle = ggml_view_2d(ctx0, proj, n_embd, n_pos,
+ proj->nb[1], ggml_row_size(proj->type, n_embd));
+ ggml_tensor * third = ggml_view_2d(ctx0, proj, n_embd, n_pos,
+ proj->nb[1], ggml_row_size(proj->type, 2 * n_embd));
+
+ ggml_tensor * gated = ggml_mul(ctx0, first, third);
+ cb(gated, "conv_gated", il);
+
+ ggml_tensor * conv_out;
+ if (layer.conv2d_spatial_w) {
+ ggml_tensor * gated_3d = ggml_reshape_4d(ctx0, gated,
+ n_embd, n_patches_x, n_patches_y, batch_size);
+ ggml_tensor * gated_2d = ggml_cont(ctx0,
+ ggml_permute(ctx0, gated_3d, 2, 0, 1, 3));
+ cb(gated_2d, "conv_gated_2d", il);
+
+ conv_out = ggml_conv_2d_dw_direct(ctx0, layer.conv2d_spatial_w, gated_2d,
+ 1, 1, 1, 1, 1, 1);
+ cb(conv_out, "conv_spatial", il);
+
+ conv_out = ggml_cont(ctx0,
+ ggml_permute(ctx0, conv_out, 1, 2, 0, 3));
+ conv_out = ggml_reshape_2d(ctx0, conv_out, n_embd, n_pos);
+ } else {
+ conv_out = gated;
+ }
+
+ cur = ggml_mul(ctx0, middle, conv_out);
+ cb(cur, "conv_mul", il);
+
+ cur = build_mm(layer.conv2d_out_proj_w, cur);
+ cb(cur, "conv_out_proj", il);
+ } else {
+ ggml_tensor * Qcur = build_mm(layer.q_w, cur);
+ ggml_tensor * Kcur = build_mm(layer.k_w, cur);
+ ggml_tensor * Vcur = build_mm(layer.v_w, cur);
+
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+
+ Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head_kv, n_pos);
+ Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head_kv, n_pos);
+
+ if (layer.q_norm) {
+ Qcur = build_norm(Qcur, layer.q_norm, nullptr, norm_t, eps, il);
+ }
+ if (layer.k_norm) {
+ Kcur = build_norm(Kcur, layer.k_norm, nullptr, norm_t, eps, il);
+ }
+
+ Qcur = ggml_rope_multi(
+ ctx0, Qcur, positions, nullptr,
+ d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
+ 32768, rope_theta, 1, 0, 1, 32, 1);
+ Kcur = ggml_rope_multi(
+ ctx0, Kcur, positions, nullptr,
+ d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
+ 32768, rope_theta, 1, 0, 1, 32, 1);
+
+ cb(Qcur, "Qcur_rope", il);
+ cb(Kcur, "Kcur_rope", il);
+
+ cur = build_attn(layer.o_w, layer.o_b,
+ Qcur, Kcur, Vcur, nullptr, kq_scale, il);
+ cb(cur, "attn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, inpL);
+ inpL = cur;
+ cb(cur, "ffn_inp", il);
+
+ cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);
+ cb(cur, "ffn_inp_normed", il);
+
+ cur = build_ffn(cur,
+ layer.ff_up_w, layer.ff_up_b,
+ layer.ff_gate_w, layer.ff_gate_b,
+ layer.ff_down_w, layer.ff_down_b,
+ hparams.ffn_op, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, inpL, cur);
+ cb(cur, "layer_out", il);
+
+ inpL = cur;
+ }
+
+ // no post-layernorm for LFM3-VL
+
+ ggml_tensor * embeddings = inpL;
+
+ // projector input LayerNorm, eps is projector_layer_norm_eps (not the encoder eps)
+ embeddings = build_norm(embeddings, model.mm_input_norm_w, model.mm_input_norm_b, NORM_TYPE_NORMAL, 1e-6f, -1);
+ cb(embeddings, "proj_input_norm", -1);
+
+ // 2x2 patch interleave (same as Qwen3VL) so the reshape below merges spatial 2x2 blocks
+ {
+ embeddings = ggml_cont_4d(
+ ctx0, embeddings,
+ n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
+ embeddings = ggml_reshape_4d(
+ ctx0, embeddings,
+ n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
+ embeddings = ggml_permute(ctx0, embeddings, 0, 2, 1, 3);
+ embeddings = ggml_cont_3d(
+ ctx0, embeddings,
+ n_embd, n_patches_x * n_patches_y, batch_size);
+ }
+ cb(embeddings, "proj_interleaved", -1);
+
+ embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd * 4, n_pos / 4, batch_size);
+
+ // projector MLP: linear_1 -> GELU (erf, as HF "gelu") -> linear_2
+ embeddings = build_ffn(embeddings,
+ model.mm_0_w, model.mm_0_b,
+ nullptr, nullptr,
+ model.mm_1_w, model.mm_1_b,
+ ffn_op_type::FFN_GELU_ERF, -1);
+
+ ggml_build_forward_expand(gf, embeddings);
+
+ return gf;
+}
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 3b34548c4..ba5769c76 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -54,6 +54,11 @@ struct clip_graph_sensenova_u1 : clip_graph {
ggml_cgraph * build() override;
};
+struct clip_graph_lfm3vl : clip_graph {
+ clip_graph_lfm3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+ ggml_cgraph * build() override;
+};
+
struct clip_graph_ling3vl : clip_graph_qwen3vl {
clip_graph_ling3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen3vl(ctx, img) {}
ggml_cgraph * build() override;
diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp
index 551ea0c60..ffa483b25 100644
--- a/tools/mtmd/mtmd-image.cpp
+++ b/tools/mtmd/mtmd-image.cpp
@@ -809,8 +809,7 @@ clip_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const clip_hpa
mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const {
clip_image_u8 resized_image;
- const clip_image_size target_size = get_target_size(hparams, img.get_size());
- img_tool::resize(img, resized_image, target_size,
+ img_tool::resize(img, resized_image, target_size(img.get_size()),
hparams.image_resize_algo,
hparams.image_resize_pad,
hparams.image_pad_color);
@@ -819,6 +818,34 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_i
return output;
}
+//
+// mtmd_image_preprocessor_lfm3vl
+//
+
+// rounds before the pixel checks (an edge may round to 0) and computes in double, as transformers does
+clip_image_size mtmd_image_preprocessor_lfm3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels) {
+ const double h = height;
+ const double w = width;
+ int h_bar = static_cast<int>(std::nearbyint(h / factor)) * factor;
+ int w_bar = static_cast<int>(std::nearbyint(w / factor)) * factor;
+ if (static_cast<int64_t>(h_bar) * w_bar > max_pixels) {
+ const double beta = std::sqrt(h * w / max_pixels);
+ h_bar = static_cast<int>(std::floor(h / beta / factor)) * factor;
+ w_bar = static_cast<int>(std::floor(w / beta / factor)) * factor;
+ } else if (static_cast<int64_t>(h_bar) * w_bar < min_pixels) {
+ const double beta = std::sqrt(min_pixels / (h * w));
+ h_bar = static_cast<int>(std::ceil(h * beta / factor)) * factor;
+ w_bar = static_cast<int>(std::ceil(w * beta / factor)) * factor;
+ }
+
+ return {std::max(factor, w_bar), std::max(factor, h_bar)};
+}
+
+clip_image_size mtmd_image_preprocessor_lfm3vl::target_size(const clip_image_size & original_size) const {
+ return smart_resize(original_size.width, original_size.height, hparams.patch_size * hparams.n_merge,
+ hparams.image_min_pixels, hparams.image_max_pixels);
+}
+
//
// mtmd_image_preprocessor_glm5v
//
diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h
index 40d5b59cc..0e97a370a 100644
--- a/tools/mtmd/mtmd-image.h
+++ b/tools/mtmd/mtmd-image.h
@@ -135,6 +135,15 @@ struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor {
mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size);
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+ virtual clip_image_size target_size(const clip_image_size & original_size) const { return get_target_size(hparams, original_size); }
+};
+
+// LFM3-VL, same as dyn_size but sized exactly as the transformers Qwen2-VL smart_resize
+struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_dyn_size {
+ mtmd_image_preprocessor_lfm3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_dyn_size(ctx) {}
+ clip_image_size target_size(const clip_image_size & original_size) const override;
+
+ static clip_image_size smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels);
};
// GLM 5.3 flash, similar to dyn_size, but each edge is aligned up to patch_size*n_merge, and max budget is met by a search over the height with the width scaled proportionally
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index df32b8f36..37d5eb21d 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -772,6 +772,13 @@ struct mtmd_context {
img_end = "<|image_end|>";
image_preproc = std::make_unique<mtmd_image_preprocessor_muse_glimmer>(ctx_v);
} break;
+ case PROJECTOR_TYPE_LFM3VL:
+ {
+ // <|vision_start|> ... (image embeddings) ... <|vision_end|>
+ img_beg = "<|vision_start|>";
+ img_end = "<|vision_end|>";
+ image_preproc = std::make_unique<mtmd_image_preprocessor_lfm3vl>(ctx_v);
+ } break;
case PROJECTOR_TYPE_YOUTUVL:
{
// <|vision_start|> ... (image embeddings) ... <|vision_end|>