Commit abb76ab38 for llama.cpp
commit abb76ab380c7f3f731e365a859fb4cc64e939bb8
Author: Giuseppe Scrivano <giuseppe@scrivano.org>
Date: Thu Oct 29 09:13:41 2026 +0100
model : Support Limite 1B - Violetto (#29433)
* gguf-py : add Limite arch
Assisted-by: Claude Opus 5.5
Signed-off-by: Giuseppe Scrivano <giuseppe@scrivano.org>
* convert : add Limite support
Assisted-by: Claude Opus 5.5
Signed-off-by: Giuseppe Scrivano <giuseppe@scrivano.org>
* llama : add Limite support
Assisted-by: Claude Opus 5.5
Signed-off-by: Giuseppe Scrivano <giuseppe@scrivano.org>
---------
Signed-off-by: Giuseppe Scrivano <giuseppe@scrivano.org>
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 4e4fae608..d4e9f0441 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -176,6 +176,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"Lfm3ForCausalLM": "lfm3",
"Lfm3OmniForConditionalGeneration": "lfm3",
"Lfm3VlForConditionalGeneration": "lfm3",
+ "LimiteForCausalLM": "limite",
"Llama4ForCausalLM": "llama",
"Llama4ForConditionalGeneration": "llama",
"LlamaBidirectionalModel": "llama",
diff --git a/conversion/limite.py b/conversion/limite.py
new file mode 100644
index 000000000..7d413302b
--- /dev/null
+++ b/conversion/limite.py
@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+from typing import Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import ModelBase, TextModel, gguf
+
+
+@ModelBase.register("LimiteForCausalLM")
+@ModelBase.example("paradigma-inc/limite-1b-violetto")
+class LimiteModel(TextModel):
+ model_arch = gguf.MODEL_ARCH.LIMITE
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ hp = self.hparams
+
+ expected = {
+ "pos_mode": "rope",
+ "qk_norm": "rms_pre_rope",
+ "rope_style": "interleaved_pairs_odd_lane_sign_flip",
+ "rope_per_layer": False,
+ "rms_norm_has_weight": False,
+ "rms_norm_eps_mode": "torch_finfo_default",
+ "mlp_type": "swiglu",
+ "attn_gate_applied": "per_head_before_o_proj",
+ "ve_head_slice": "first_num_key_value_heads",
+ "ve_applied_before_qk_norm": True,
+ "global_nope": True,
+ "global_window": -1,
+ "mudd_r_site": "resid",
+ "tie_word_embeddings": True,
+ "final_softcap": 0.0,
+ }
+ for key, value in expected.items():
+ if hp.get(key) != value:
+ raise NotImplementedError(f"Limite: unsupported {key}={hp.get(key)!r}, expected {value!r}")
+ if hp["softcap_logits"]["kind"] != "sigmoid":
+ raise NotImplementedError(f"Limite: unsupported softcap_logits kind {hp['softcap_logits']['kind']!r}")
+
+ self.n_head = hp["num_attention_heads"]
+ self.n_head_kv = hp["num_key_value_heads"]
+ self.head_dim = hp["head_dim"]
+ self.n_pairs = hp["rope_n_pairs"]
+
+ self.mudd_taps: dict[int, list[int]] = {}
+ if hp.get("mudd"):
+ if not hp.get("mudd_mlp"):
+ raise NotImplementedError("Limite: mudd without mudd_mlp is not supported")
+ self.mudd_taps = {int(il): [int(t) for t in taps] for il, taps in hp["mudd_tap_idx"].items()}
+ assert all(len(taps) == hp["mudd_taps"] for taps in self.mudd_taps.values())
+
+ def set_vocab(self):
+ self._set_vocab_gpt2()
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ hp = self.hparams
+
+ self.gguf_writer.add_key_length(self.head_dim)
+ self.gguf_writer.add_value_length(self.head_dim)
+
+ # torch rms_norm with eps=None uses the fp32 epsilon, even for bf16 inputs
+ self.gguf_writer.add_layer_norm_rms_eps(torch.finfo(torch.float32).eps)
+ self.gguf_writer.add_attention_scale(hp["attention_softmax_scale"])
+ self.gguf_writer.add_xsa_eps(hp["xsa_normalize_eps"] if hp.get("xsa") else 0.0)
+
+ # HF freq_i = base^(-i/(n_pairs-1)), ggml uses freq_base^(-i/n_pairs)
+ self.gguf_writer.add_rope_dimension_count(2 * self.n_pairs)
+ self.gguf_writer.add_rope_freq_base(hp["rope_base_local"] ** (self.n_pairs / (self.n_pairs - 1)))
+
+ # HF keeps keys k >= q - window, so a window of W spans W + 1 tokens
+ global_layers = set(hp["global_layers"])
+ self.gguf_writer.add_sliding_window(hp["sliding_window"] + 1)
+ self.gguf_writer.add_sliding_window_pattern([il not in global_layers for il in range(self.block_count)])
+
+ cap = hp["softcap_logits"]
+ self.gguf_writer.add_final_logit_sigmoid_capping(cap["a"], cap["b"], cap["c"])
+
+ if self.mudd_taps:
+ n_taps = hp["mudd_taps"]
+ tap_indices = [-1] * (self.block_count * n_taps)
+ for il, taps in self.mudd_taps.items():
+ tap_indices[il * n_taps:(il + 1) * n_taps] = taps
+ self.gguf_writer.add_mudd_feed_forward_length(hp["mudd_inter"])
+ self.gguf_writer.add_mudd_tap_count(n_taps)
+ self.gguf_writer.add_mudd_tap_indices(tap_indices)
+
+ def tensor_force_quant(self, name, new_name, bid, n_dims):
+ # tiny mixing weights, kept at full precision as in the reference
+ # the mudd tensors come from a global source tensor, so take the block id from the new name
+ if new_name.startswith("blk."):
+ bid = int(new_name.split(".")[1])
+ for key in (gguf.MODEL_TENSOR.ATTN_GATE, gguf.MODEL_TENSOR.ATTN_VE_GATE, gguf.MODEL_TENSOR.MUDD_DOWN, gguf.MODEL_TENSOR.MUDD_UP):
+ if self.match_model_tensor_name(new_name, key, bid):
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def _swap_rope_pairs(self, w: Tensor, n_head: int) -> Tensor:
+ # HF rotates each pair by -theta; swapping the pair lanes gives the ggml +theta rotation.
+ # q and k are both swapped, so the attention scores do not change.
+ idx = torch.arange(self.head_dim)
+ idx[:2 * self.n_pairs] = idx[:2 * self.n_pairs].view(-1, 2).flip(-1).reshape(-1)
+ w = w.reshape(n_head, self.head_dim, -1)
+ return w[:, idx, :].reshape(n_head * self.head_dim, -1)
+
+ def _pad_cols(self, w: Tensor) -> Tensor:
+ # gates read only the first channels of the input, pad them to n_embd
+ return torch.nn.functional.pad(w, (0, self.hparams["hidden_size"] - w.shape[-1]))
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ hp = self.hparams
+ T = gguf.MODEL_TENSOR
+
+ def load(n: str) -> Tensor:
+ return self.model_tensors[n]().float()
+
+ if name == "model.value_embeds.weight":
+ ve = data_torch.reshape(data_torch.shape[0], -1, hp["ve_dim"])[:, :self.n_head_kv]
+ ve = torch.nn.functional.pad(ve, (0, self.head_dim - hp["ve_dim"]))
+ yield self.format_tensor_name(T.VALUE_EMBD), ve.reshape(ve.shape[0], -1) * hp["ve_gate_scale"]
+ return
+
+ if name == "model.mudd.dense1":
+ for il in self.mudd_taps:
+ yield self.format_tensor_name(T.MUDD_DOWN, il), data_torch
+ return
+ if name == "model.mudd.dense2":
+ # rows [0, n) mix the attention input, rows [n, 2n) mix the residual
+ dense2_mlp = load("model.mudd.dense2_mlp")
+ bias = load("model.mudd.bias")
+ bias_mlp = load("model.mudd.bias_mlp")
+ for il, taps in self.mudd_taps.items():
+ n = len(taps)
+ w = torch.cat([data_torch[il, :n], dense2_mlp[il, :n]], 0)
+ b = torch.cat([bias[il, :n], bias_mlp[il, :n]], 0)
+ yield self.format_tensor_name(T.MUDD_UP, il), w
+ yield self.format_tensor_name(T.MUDD_UP, il, ".bias"), b
+ return
+ if name.startswith("model.mudd."):
+ return
+
+ if bid is None:
+ yield from super().modify_tensors(data_torch, name, bid)
+ return
+
+ prefix = f"model.layers.{bid}."
+ name = name.removeprefix(prefix)
+
+ if name in ("self_attn.q_proj.weight", "self_attn.k_proj.weight", "self_attn.v_proj.weight"):
+ data_torch = data_torch * load(prefix + "self_attn.qkv_scale")
+ if name == "self_attn.q_proj.weight":
+ data_torch = self._swap_rope_pairs(data_torch, self.n_head)
+ elif name == "self_attn.k_proj.weight":
+ data_torch = self._swap_rope_pairs(data_torch, self.n_head_kv)
+ elif name == "self_attn.o_proj.weight":
+ data_torch = data_torch * load(prefix + "self_attn.o_scale") * load(prefix + "post_lambda_attn") * hp["attn_gate_scale"]
+ elif name == "mlp.down_proj.weight":
+ data_torch = data_torch * load(prefix + "post_lambda_mlp")
+ elif name == "self_attn.attn_gate":
+ yield self.format_tensor_name(T.ATTN_GATE, bid), self._pad_cols(data_torch)
+ return
+ elif name == "self_attn.ve_gate":
+ yield self.format_tensor_name(T.ATTN_VE_GATE, bid), self._pad_cols(data_torch[:self.n_head_kv])
+ return
+ elif name == "self_attn.xsa_alpha":
+ if hp.get("xsa") and bid in hp["xsa_layers"]:
+ yield self.format_tensor_name(T.ATTN_XSA_ALPHA, bid), torch.tanh(data_torch)
+ return
+ elif name == "resid_lambda_attn":
+ yield self.format_tensor_name(T.ATTN_RESID_SCALE, bid), data_torch.reshape(1)
+ return
+ elif name == "resid_lambda_mlp":
+ yield self.format_tensor_name(T.FFN_RESID_SCALE, bid), data_torch.reshape(1)
+ return
+ elif name in ("self_attn.qkv_scale", "self_attn.o_scale", "post_lambda_attn", "post_lambda_mlp"):
+ # folded into the projections above
+ return
+
+ yield from super().modify_tensors(data_torch, prefix + name, bid)
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index e28fdf605..1ca2415cd 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -143,6 +143,7 @@ class Keys:
ATTN_LOGIT_SOFTCAPPING = "{arch}.attn_logit_softcapping"
ROUTER_LOGIT_SOFTCAPPING = "{arch}.router_logit_softcapping"
FINAL_LOGIT_SOFTCAPPING = "{arch}.final_logit_softcapping"
+ FINAL_LOGIT_SIGMOID_CAPPING = "{arch}.final_logit_sigmoid_capping"
SWIN_NORM = "{arch}.swin_norm"
RESCALE_EVERY_N_LAYERS = "{arch}.rescale_every_n_layers"
TIME_MIX_EXTRA_DIM = "{arch}.time_mix_extra_dim"
@@ -227,6 +228,7 @@ class Keys:
ROPE_PATTERN = "{arch}.attention.rope_pattern"
VALUE_EXPERT_COUNT = "{arch}.attention.value_expert_count"
VALUE_EXPERT_USED_COUNT = "{arch}.attention.value_expert_used_count"
+ XSA_EPS = "{arch}.attention.xsa_epsilon"
class Indexer:
HEAD_COUNT = "{arch}.attention.indexer.head_count"
@@ -301,6 +303,11 @@ class Keys:
LLM_KV_SPLIT_COUNT = "split.count"
LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count"
+ class MUDD:
+ FEED_FORWARD_LENGTH = "{arch}.mudd.feed_forward_length"
+ TAP_COUNT = "{arch}.mudd.tap_count"
+ TAP_INDICES = "{arch}.mudd.tap_indices"
+
class HRM:
LAYERS_PER_STACK = "{arch}.hrm.layers_per_stack"
H_CYCLES = "{arch}.hrm.h_cycles"
@@ -731,6 +738,7 @@ class MODEL_ARCH(IntEnum):
POCKETTTS = auto()
K2HORIZON = auto()
KOLIBRI1 = auto()
+ LIMITE = auto()
class VISION_PROJECTOR_TYPE(IntEnum):
@@ -865,6 +873,13 @@ class MODEL_TENSOR(IntEnum):
FFN_ROUTED_DOWN = auto() # Kimi K3 (latent MoE: hidden -> latent)
FFN_ROUTED_UP = auto() # Kimi K3 (latent MoE: latent -> hidden)
FFN_ROUTED_NORM = auto() # Kimi K3 (latent MoE: norm on expert output)
+ VALUE_EMBD = auto() # limite
+ ATTN_VE_GATE = auto() # limite
+ ATTN_XSA_ALPHA = auto() # limite
+ ATTN_RESID_SCALE = auto() # limite
+ FFN_RESID_SCALE = auto() # limite
+ MUDD_DOWN = auto() # limite
+ MUDD_UP = auto() # limite
TIME_MIX_W0 = auto()
TIME_MIX_W1 = auto()
TIME_MIX_W2 = auto()
@@ -1577,6 +1592,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.POCKETTTS: "pockettts",
MODEL_ARCH.K2HORIZON: "k2-horizon",
MODEL_ARCH.KOLIBRI1: "kolibri1",
+ MODEL_ARCH.LIMITE: "limite",
}
VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = {
@@ -1709,6 +1725,13 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.FFN_ROUTED_DOWN: "blk.{bid}.ffn_routed_down", # Kimi K3
MODEL_TENSOR.FFN_ROUTED_UP: "blk.{bid}.ffn_routed_up", # Kimi K3
MODEL_TENSOR.FFN_ROUTED_NORM: "blk.{bid}.ffn_routed_norm", # Kimi K3
+ MODEL_TENSOR.VALUE_EMBD: "value_embd", # limite
+ MODEL_TENSOR.ATTN_VE_GATE: "blk.{bid}.attn_ve_gate", # limite
+ MODEL_TENSOR.ATTN_XSA_ALPHA: "blk.{bid}.attn_xsa_alpha", # limite
+ MODEL_TENSOR.ATTN_RESID_SCALE: "blk.{bid}.attn_resid_scale", # limite
+ MODEL_TENSOR.FFN_RESID_SCALE: "blk.{bid}.ffn_resid_scale", # limite
+ MODEL_TENSOR.MUDD_DOWN: "blk.{bid}.mudd_down", # limite
+ MODEL_TENSOR.MUDD_UP: "blk.{bid}.mudd_up", # limite
MODEL_TENSOR.TIME_MIX_W0: "blk.{bid}.time_mix_w0",
MODEL_TENSOR.TIME_MIX_W1: "blk.{bid}.time_mix_w1",
MODEL_TENSOR.TIME_MIX_W2: "blk.{bid}.time_mix_w2",
@@ -6288,6 +6311,25 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_UP_SHEXP,
MODEL_TENSOR.FFN_DOWN_SHEXP,
],
+ MODEL_ARCH.LIMITE: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.VALUE_EMBD,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_VE_GATE,
+ MODEL_TENSOR.ATTN_XSA_ALPHA,
+ MODEL_TENSOR.ATTN_RESID_SCALE,
+ MODEL_TENSOR.FFN_RESID_SCALE,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.MUDD_DOWN,
+ MODEL_TENSOR.MUDD_UP,
+ ],
}
# tensors that will not be serialized
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index efdf8f09c..b3dabc0bd 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -876,6 +876,9 @@ class GGUFWriter:
def add_final_logit_softcapping(self, value: float) -> None:
self.add_float32(Keys.LLM.FINAL_LOGIT_SOFTCAPPING.format(arch=self.arch), value)
+ def add_final_logit_sigmoid_capping(self, scale: float, shift: float, temperature: float) -> None:
+ self.add_array(Keys.LLM.FINAL_LOGIT_SIGMOID_CAPPING.format(arch=self.arch), [scale, shift, temperature])
+
def add_expert_count(self, count: int) -> None:
self.add_uint32(Keys.LLM.EXPERT_COUNT.format(arch=self.arch), count)
@@ -948,6 +951,15 @@ class GGUFWriter:
def add_embedding_scale(self, value: float) -> None:
self.add_float32(Keys.LLM.EMBEDDING_SCALE.format(arch=self.arch), value)
+ def add_mudd_feed_forward_length(self, value: int) -> None:
+ self.add_uint32(Keys.MUDD.FEED_FORWARD_LENGTH.format(arch=self.arch), value)
+
+ def add_mudd_tap_count(self, value: int) -> None:
+ self.add_uint32(Keys.MUDD.TAP_COUNT.format(arch=self.arch), value)
+
+ def add_mudd_tap_indices(self, value: Sequence[int]) -> None:
+ self.add_array(Keys.MUDD.TAP_INDICES.format(arch=self.arch), value)
+
def add_hrm_layers_per_stack(self, value: int) -> None:
self.add_uint32(Keys.HRM.LAYERS_PER_STACK.format(arch=self.arch), value)
@@ -1151,6 +1163,9 @@ class GGUFWriter:
def add_attention_scale(self, value: float) -> None:
self.add_float32(Keys.Attention.SCALE.format(arch=self.arch), value)
+ def add_xsa_eps(self, value: float) -> None:
+ self.add_float32(Keys.Attention.XSA_EPS.format(arch=self.arch), value)
+
def add_attn_output_scale(self, value: float) -> None:
self.add_float32(Keys.Attention.OUTPUT_SCALE.format(arch=self.arch), value)
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 497e42692..3a19646eb 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -145,6 +145,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_APERTUS, "apertus" },
{ LLM_ARCH_MINIMAX_01, "minimax-01" },
{ LLM_ARCH_HRM_TEXT, "hrm_text" },
+ { LLM_ARCH_LIMITE, "limite" },
{ LLM_ARCH_MINIMAX_M2, "minimax-m2" },
{ LLM_ARCH_MINIMAX_M3, "minimax-m3" },
{ LLM_ARCH_COGVLM, "cogvlm" },
@@ -248,6 +249,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_ATTN_LOGIT_SOFTCAPPING, "%s.attn_logit_softcapping" },
{ LLM_KV_ROUTER_LOGIT_SOFTCAPPING, "%s.router_logit_softcapping" },
{ LLM_KV_FINAL_LOGIT_SOFTCAPPING, "%s.final_logit_softcapping" },
+ { LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, "%s.final_logit_sigmoid_capping" },
{ LLM_KV_SWIN_NORM, "%s.swin_norm" },
{ LLM_KV_RESCALE_EVERY_N_LAYERS, "%s.rescale_every_n_layers" },
{ LLM_KV_TIME_MIX_EXTRA_DIM, "%s.time_mix_extra_dim" },
@@ -268,6 +270,9 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_HRM_H_CYCLES, "%s.hrm.h_cycles" },
{ LLM_KV_HRM_L_CYCLES, "%s.hrm.l_cycles" },
{ LLM_KV_HRM_PREFIX_LM, "%s.hrm.prefix_lm" },
+ { LLM_KV_MUDD_FEED_FORWARD_LENGTH, "%s.mudd.feed_forward_length" },
+ { LLM_KV_MUDD_TAP_COUNT, "%s.mudd.tap_count" },
+ { LLM_KV_MUDD_TAP_INDICES, "%s.mudd.tap_indices" },
{ LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" },
{ LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" },
@@ -292,6 +297,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_ATTENTION_ROPE_PATTERN, "%s.attention.rope_pattern" },
{ LLM_KV_ATTENTION_VALUE_EXPERT_COUNT, "%s.attention.value_expert_count" },
{ LLM_KV_ATTENTION_VALUE_EXPERT_USED_COUNT, "%s.attention.value_expert_used_count" },
+ { LLM_KV_ATTENTION_XSA_EPS, "%s.attention.xsa_epsilon" },
{ LLM_KV_ATTENTION_SCALE, "%s.attention.scale" },
{ LLM_KV_ATTENTION_OUTPUT_SCALE, "%s.attention.output_scale" },
@@ -605,6 +611,13 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_FFN_ROUTED_DOWN, "blk.%d.ffn_routed_down" },
{ LLM_TENSOR_FFN_ROUTED_UP, "blk.%d.ffn_routed_up" },
{ LLM_TENSOR_FFN_ROUTED_NORM, "blk.%d.ffn_routed_norm" },
+ { LLM_TENSOR_VALUE_EMBD, "value_embd" },
+ { LLM_TENSOR_ATTN_VE_GATE, "blk.%d.attn_ve_gate" },
+ { LLM_TENSOR_ATTN_XSA_ALPHA, "blk.%d.attn_xsa_alpha" },
+ { LLM_TENSOR_ATTN_RESID_SCALE, "blk.%d.attn_resid_scale" },
+ { LLM_TENSOR_FFN_RESID_SCALE, "blk.%d.ffn_resid_scale" },
+ { LLM_TENSOR_MUDD_DOWN, "blk.%d.mudd_down" },
+ { LLM_TENSOR_MUDD_UP, "blk.%d.mudd_up" },
{ LLM_TENSOR_SSM_G_B, "blk.%d.ssm_g_b" },
{ LLM_TENSOR_SSM_NORM, "blk.%d.ssm_norm" },
{ LLM_TENSOR_ATTN_Q_A_NORM, "blk.%d.attn_q_a_norm" },
@@ -998,6 +1011,13 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_FFN_ROUTED_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_FFN_ROUTED_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_FFN_ROUTED_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_VALUE_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},
+ {LLM_TENSOR_ATTN_VE_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_ATTN_XSA_ALPHA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_ATTN_RESID_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_FFN_RESID_SCALE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_MUDD_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_MUDD_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_SSM_G_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_TIME_MIX_LERP_X, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_TIME_MIX_LN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
@@ -1315,6 +1335,7 @@ bool llm_arch_supports_mixed_batch(const llm_arch & arch) {
case LLM_ARCH_EAGLE3:
case LLM_ARCH_DFLASH:
case LLM_ARCH_GEMMA4_ASSISTANT:
+ case LLM_ARCH_LIMITE:
return false;
default:
return true;
@@ -1357,6 +1378,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
case LLM_ARCH_KIMI_K3:
case LLM_ARCH_GLM5_NEXT:
case LLM_ARCH_QWEN3TTS:
+ case LLM_ARCH_LIMITE:
return false;
default:
return true;
diff --git a/src/llama-arch.h b/src/llama-arch.h
index eca6d18af..554156813 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -176,6 +176,7 @@ enum llm_arch {
LLM_ARCH_K2_HORIZON,
LLM_ARCH_BEAM,
LLM_ARCH_KOLIBRI1,
+ LLM_ARCH_LIMITE,
LLM_ARCH_UNKNOWN,
};
@@ -252,6 +253,7 @@ enum llm_kv {
LLM_KV_ATTN_LOGIT_SOFTCAPPING,
LLM_KV_ROUTER_LOGIT_SOFTCAPPING,
LLM_KV_FINAL_LOGIT_SOFTCAPPING,
+ LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING,
LLM_KV_SWIN_NORM,
LLM_KV_RESCALE_EVERY_N_LAYERS,
LLM_KV_TIME_MIX_EXTRA_DIM,
@@ -273,6 +275,9 @@ enum llm_kv {
LLM_KV_HRM_L_CYCLES,
LLM_KV_HRM_PREFIX_LM,
LLM_KV_REALTIME_DELAY_TOKENS,
+ LLM_KV_MUDD_FEED_FORWARD_LENGTH,
+ LLM_KV_MUDD_TAP_COUNT,
+ LLM_KV_MUDD_TAP_INDICES,
LLM_KV_ATTENTION_HEAD_COUNT,
LLM_KV_ATTENTION_HEAD_COUNT_KV,
@@ -298,6 +303,7 @@ enum llm_kv {
LLM_KV_ATTENTION_ROPE_PATTERN,
LLM_KV_ATTENTION_VALUE_EXPERT_COUNT,
LLM_KV_ATTENTION_VALUE_EXPERT_USED_COUNT,
+ LLM_KV_ATTENTION_XSA_EPS,
LLM_KV_ATTENTION_OUTPUT_SCALE,
LLM_KV_ATTENTION_VALUE_SCALE,
@@ -612,6 +618,13 @@ enum llm_tensor {
LLM_TENSOR_FFN_ROUTED_DOWN, // kimi-k3: latent MoE down
LLM_TENSOR_FFN_ROUTED_UP, // kimi-k3: latent MoE up
LLM_TENSOR_FFN_ROUTED_NORM, // kimi-k3: latent MoE norm
+ LLM_TENSOR_VALUE_EMBD, // limite: value embeddings
+ LLM_TENSOR_ATTN_VE_GATE, // limite: value embedding gate
+ LLM_TENSOR_ATTN_XSA_ALPHA, // limite: exclusive self attention
+ LLM_TENSOR_ATTN_RESID_SCALE, // limite: residual scale (attn)
+ LLM_TENSOR_FFN_RESID_SCALE, // limite: residual scale (ffn)
+ LLM_TENSOR_MUDD_DOWN, // limite: dense connection mixer
+ LLM_TENSOR_MUDD_UP, // limite: dense connection mixer
LLM_TENSOR_TIME_MIX_W0,
LLM_TENSOR_TIME_MIX_W1,
LLM_TENSOR_TIME_MIX_W2,
diff --git a/src/llama-hparams.h b/src/llama-hparams.h
index 2c464ad30..2ac7bdeae 100644
--- a/src/llama-hparams.h
+++ b/src/llama-hparams.h
@@ -227,6 +227,13 @@ struct llama_hparams {
uint32_t n_hrm_l_cycles = 0;
bool hrm_prefix_lm = false;
+ // limite
+ float f_xsa_eps = 0.0f;
+ std::array<float, 3> final_logit_sigmoid_capping = {}; // scale, shift, temperature
+ uint32_t n_mudd_ff = 0;
+ uint32_t n_mudd_taps = 0;
+ std::array<int32_t, LLAMA_MAX_LAYERS> mudd_tap_idx; // [n_layer][n_mudd_taps], -1 = no mixing
+
bool ssm_dt_b_c_rms = false;
float f_clamp_kqv = 0.0f;
diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp
index a963f2480..a71e3017a 100644
--- a/src/llama-model-loader.cpp
+++ b/src/llama-model-loader.cpp
@@ -411,6 +411,7 @@ namespace GGUFMeta {
template bool llama_model_loader::get_arr<std::vector<std::string>>(enum llm_kv kid, std::vector<std::string> & result, bool required);
template bool llama_model_loader::get_arr<std::array<int32_t, 512>>(enum llm_kv kid, std::array<int32_t, 512> & result, bool required);
+ template bool llama_model_loader::get_arr<std::array<float, 3>>(enum llm_kv kid, std::array<float, 3> & result, bool required);
template bool llama_model_loader::get_arr<std::vector<int32_t>>(enum llm_kv kid, std::vector<int32_t> & result, bool required);
template bool llama_model_loader::get_arr<std::array<uint32_t, LLAMA_MAX_LAYERS>>(enum llm_kv kid, std::array<uint32_t, LLAMA_MAX_LAYERS> & result, bool required);
template bool llama_model_loader::get_arr<std::vector<uint32_t>>(enum llm_kv kid, std::vector<uint32_t> & result, bool required);
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index 1160e88a2..93836ed96 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -244,6 +244,16 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_HRM_H_CYCLES, hparams.n_hrm_h_cycles);
add_kv(LLM_KV_HRM_L_CYCLES, hparams.n_hrm_l_cycles);
add_kv(LLM_KV_HRM_PREFIX_LM, hparams.hrm_prefix_lm);
+ if (hparams.final_logit_sigmoid_capping[0] != 0.0f) {
+ add_kv(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, std::vector<float>(
+ hparams.final_logit_sigmoid_capping.begin(), hparams.final_logit_sigmoid_capping.end()));
+ }
+ add_kv(LLM_KV_MUDD_FEED_FORWARD_LENGTH, hparams.n_mudd_ff);
+ add_kv(LLM_KV_MUDD_TAP_COUNT, hparams.n_mudd_taps);
+ if (hparams.n_mudd_taps > 0) {
+ add_kv(LLM_KV_MUDD_TAP_INDICES, std::vector<int32_t>(
+ hparams.mudd_tap_idx.begin(), hparams.mudd_tap_idx.begin() + hparams.n_layer()*hparams.n_mudd_taps));
+ }
add_kv(LLM_KV_TOKEN_SHIFT_COUNT, hparams.token_shift_count);
add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, hparams.n_moe_layer_step);
// add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, ???); // saved as LLM_KV_ATTENTION_RECURRENT_LAYERS instead
@@ -277,6 +287,7 @@ void llama_model_saver::add_kv_from_model() {
hparams.is_swa_impl.begin(), hparams.is_swa_impl.begin() + hparams.n_layer_all));
}
add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale);
+ add_kv(LLM_KV_ATTENTION_XSA_EPS, hparams.f_xsa_eps);
add_kv(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale);
add_kv(LLM_KV_ATTENTION_VALUE_SCALE, hparams.f_attn_value_scale);
add_kv(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.attn_temp_length);
@@ -501,6 +512,7 @@ void llama_model_saver::add_tensors_from_model() {
add_tensor(model->hc_head_base);
add_tensor(model->hc_head_scale);
add_tensor(model->per_layer_tok_embd);
+ add_tensor(model->value_embd);
add_tensor(model->hc_head_norm);
add_tensor(model->hc_head_down);
add_tensor(model->hc_head_up);
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 81af8903f..8966bc946 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -332,6 +332,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_hrm_text(params);
case LLM_ARCH_BEAM:
return new llama_model_beam(params);
+ case LLM_ARCH_LIMITE:
+ return new llama_model_limite(params);
case LLM_ARCH_COGVLM:
return new llama_model_cogvlm(params);
case LLM_ARCH_PANGU_EMBED:
@@ -3575,6 +3577,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_DOTS3NOTE:
case LLM_ARCH_NANBEIGE:
case LLM_ARCH_POCKETTTS:
+ case LLM_ARCH_LIMITE:
return LLAMA_ROPE_TYPE_NORM;
case LLM_ARCH_BAILINGMOE3:
// VL files carry mrope sections; text-only files keep NORM rope
diff --git a/src/llama-model.h b/src/llama-model.h
index 222238efe..3af8b054f 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -638,6 +638,15 @@ struct llama_layer {
// gemma4 layer output scale, reused for talkie embedding skip scale
struct ggml_tensor * out_scale = nullptr;
+ // limite
+ struct ggml_tensor * attn_ve_gate = nullptr;
+ struct ggml_tensor * attn_xsa_alpha = nullptr;
+ struct ggml_tensor * attn_resid_scale = nullptr;
+ struct ggml_tensor * ffn_resid_scale = nullptr;
+ struct ggml_tensor * mudd_down = nullptr;
+ struct ggml_tensor * mudd_up = nullptr;
+ struct ggml_tensor * mudd_up_b = nullptr;
+
struct llama_layer_posnet posnet;
struct llama_layer_convnext convnext;
@@ -783,6 +792,9 @@ struct llama_model {
struct ggml_tensor * altup_unembd_proj = nullptr;
struct ggml_tensor * per_layer_tok_embd = nullptr;
+ // limite value embeddings
+ struct ggml_tensor * value_embd = nullptr;
+
struct ggml_tensor * hc_head_norm = nullptr;
struct ggml_tensor * hc_head_down = nullptr;
struct ggml_tensor * hc_head_up = nullptr;
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
index 9ddd6a63a..9f8aa8984 100644
--- a/src/llama-quant.cpp
+++ b/src/llama-quant.cpp
@@ -333,6 +333,13 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
// these are not too big so keep them as it is
quantize &= name.find("per_layer_model_proj") == std::string::npos;
+ // limite: very small gates and dense connection mixers
+ if (arch == LLM_ARCH_LIMITE) {
+ quantize &= name.find("attn_gate.weight") == std::string::npos;
+ quantize &= name.find("attn_ve_gate.weight") == std::string::npos;
+ quantize &= name.find("mudd_") == std::string::npos;
+ }
+
// do not quantize positional embeddings and token types (BERT)
quantize &= name != LLM_TN(arch)(LLM_TENSOR_POS_EMBD, "weight");
quantize &= name != LLM_TN(arch)(LLM_TENSOR_TOKEN_TYPES, "weight");
diff --git a/src/models/limite.cpp b/src/models/limite.cpp
new file mode 100644
index 000000000..63f7c8288
--- /dev/null
+++ b/src/models/limite.cpp
@@ -0,0 +1,252 @@
+#include "models.h"
+
+// Limite: pre-norm transformer with value embeddings, exclusive self attention (XSA),
+// per-head output gates, learned residual scales and dense (MUDD) cross-layer connections.
+// ref: https://github.com/paradigma-inc/limite-violetto
+
+void llama_model_limite::load_arch_hparams(llama_model_loader & ml) {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale);
+ ml.get_key(LLM_KV_ATTENTION_XSA_EPS, hparams.f_xsa_eps, false);
+ ml.get_arr(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, hparams.final_logit_sigmoid_capping, false);
+
+ hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
+ ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+ load_swa_pattern(ml, 4);
+
+ std::fill(hparams.mudd_tap_idx.begin(), hparams.mudd_tap_idx.end(), -1);
+ ml.get_key(LLM_KV_MUDD_TAP_COUNT, hparams.n_mudd_taps, false);
+ if (hparams.n_mudd_taps > 0) {
+ ml.get_key(LLM_KV_MUDD_FEED_FORWARD_LENGTH, hparams.n_mudd_ff);
+ ml.get_arr(LLM_KV_MUDD_TAP_INDICES, hparams.mudd_tap_idx);
+ }
+
+ switch (hparams.n_layer()) {
+ case 48: type = LLM_TYPE_1B; break;
+ default: type = LLM_TYPE_UNKNOWN;
+ }
+}
+
+void llama_model_limite::load_arch_tensors(llama_model_loader &) {
+ LLAMA_LOAD_LOCALS;
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+ value_embd = create_tensor(tn(LLM_TENSOR_VALUE_EMBD, "weight"), {n_embd_v_gqa, n_vocab}, 0);
+
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+ // if output is NULL, init from the input tok embed
+ if (output == NULL) {
+ output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+ }
+
+ const int64_t n_taps = hparams.n_mudd_taps;
+ const int64_t n_mudd = hparams.n_mudd_ff;
+
+ for (int i = 0; i < n_layer; ++i) {
+ auto & layer = layers[i];
+
+ create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0);
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0);
+
+ layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head}, TENSOR_NOT_REQUIRED);
+ layer.attn_ve_gate = create_tensor(tn(LLM_TENSOR_ATTN_VE_GATE, "weight", i), {n_embd, n_head_kv}, TENSOR_NOT_REQUIRED);
+ layer.attn_xsa_alpha = create_tensor(tn(LLM_TENSOR_ATTN_XSA_ALPHA, "weight", i), {n_head}, TENSOR_NOT_REQUIRED);
+
+ layer.attn_resid_scale = create_tensor(tn(LLM_TENSOR_ATTN_RESID_SCALE, "weight", i), {1}, 0);
+ layer.ffn_resid_scale = create_tensor(tn(LLM_TENSOR_FFN_RESID_SCALE, "weight", i), {1}, 0);
+
+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
+
+ if (n_taps > 0 && hparams.mudd_tap_idx[i*n_taps] >= 0) {
+ // rows [0, n_taps) mix the attention input, rows [n_taps, 2*n_taps) mix the residual
+ layer.mudd_down = create_tensor(tn(LLM_TENSOR_MUDD_DOWN, "weight", i), {n_embd, n_mudd}, 0);
+ layer.mudd_up = create_tensor(tn(LLM_TENSOR_MUDD_UP, "weight", i), {n_mudd, 2*n_taps}, 0);
+ layer.mudd_up_b = create_tensor(tn(LLM_TENSOR_MUDD_UP, "bias", i), {2*n_taps}, 0);
+ }
+ }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_limite::build_arch_graph(const llm_graph_params & params) const {
+ return std::make_unique<graph>(*this, params);
+}
+
+llama_model_limite::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
+ const int64_t n_embd_head = hparams.n_embd_head_k();
+ const int64_t n_taps = hparams.n_mudd_taps;
+
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_v());
+
+ ggml_tensor * cur;
+ ggml_tensor * inpL;
+
+ inpL = build_inp_embd(model.tok_embd);
+
+ // value embeddings are looked up by token id, embeddings input is not supported
+ ggml_tensor * inp_ve = ggml_get_rows(ctx0, model.value_embd, res->t_inp_tokens);
+ inp_ve = ggml_reshape_3d(ctx0, inp_ve, n_embd_head, n_head_kv, n_tokens);
+ cb(inp_ve, "inp_ve", -1);
+
+ inpL = build_norm(inpL, nullptr, nullptr, LLM_NORM_RMS, -1);
+ cb(inpL, "inp_norm", -1);
+
+ ggml_tensor * inp_pos = build_inp_pos();
+
+ auto * inp_attn = build_attn_inp_kv_iswa();
+
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ // hist[il] is the input of layer il, read by the dense connections
+ std::vector<ggml_tensor *> hist(n_layer, nullptr);
+
+ for (int il = 0; il < n_layer; ++il) {
+ const auto & layer = model.layers[il];
+
+ hist[il] = inpL;
+
+ ggml_tensor * resid = inpL;
+
+ if (layer.mudd_up) {
+ const int32_t * taps = hparams.mudd_tap_idx.data() + il*n_taps;
+
+ ggml_tensor * w = build_norm(inpL, nullptr, nullptr, LLM_NORM_RMS, il);
+ w = ggml_gelu_erf(ctx0, build_lora_mm(layer.mudd_down, w));
+ w = ggml_add(ctx0, build_lora_mm(layer.mudd_up, w), layer.mudd_up_b);
+ cb(w, "mudd_w", il);
+
+ ggml_tensor * stack = nullptr;
+ for (int64_t j = 0; j < n_taps; ++j) {
+ GGML_ASSERT(taps[j] >= 0 && taps[j] <= il);
+ ggml_tensor * h = ggml_reshape_3d(ctx0, hist[taps[j]], n_embd, 1, n_tokens);
+ stack = stack ? ggml_concat(ctx0, stack, h, 1) : h;
+ }
+
+ auto mix = [&](size_t offset) {
+ if (n_taps == 4) {
+ return ggml_dsv4_hc_pre(ctx0, stack, ggml_cont(ctx0, ggml_view_2d(ctx0, w, n_taps, n_tokens, w->nb[1], offset)));
+ }
+ ggml_tensor * wj = ggml_view_3d(ctx0, w, 1, n_taps, n_tokens, w->nb[0], w->nb[1], offset);
+ ggml_tensor * m = ggml_cont(ctx0, ggml_permute(ctx0, ggml_mul(ctx0, stack, wj), 1, 0, 2, 3));
+ return ggml_reshape_2d(ctx0, ggml_sum_rows(ctx0, m), n_embd, n_tokens);
+ };
+
+ cur = mix(0);
+ resid = mix(n_taps*w->nb[0]);
+ cb(cur, "mudd_attn", il);
+ cb(resid, "mudd_resid", il);
+ } else {
+ cur = inpL;
+ }
+
+ ggml_tensor * attn_in = build_norm(cur, nullptr, nullptr, LLM_NORM_RMS, il);
+ cb(attn_in, "attn_norm", il);
+
+ // self-attention
+ {
+ auto [Qcur, Kcur, Vcur] = build_qkv(layer, attn_in, n_embd_head, n_head, n_head_kv, il);
+
+ if (layer.attn_ve_gate) {
+ ggml_tensor * gate = ggml_sigmoid(ctx0, build_lora_mm(layer.attn_ve_gate, attn_in));
+ gate = ggml_reshape_3d(ctx0, gate, 1, n_head_kv, n_tokens);
+
+ Vcur = ggml_add(ctx0, Vcur, ggml_mul(ctx0, inp_ve, gate));
+ cb(Vcur, "Vcur_ve", il);
+ }
+
+ Qcur = ggml_rms_norm(ctx0, Qcur, hparams.f_norm_rms_eps);
+ Kcur = ggml_rms_norm(ctx0, Kcur, hparams.f_norm_rms_eps);
+ cb(Qcur, "Qcur_normed", il);
+ cb(Kcur, "Kcur_normed", il);
+
+ // the full attention layers use no positional encoding
+ if (hparams.is_swa(il)) {
+ Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+ Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ }
+
+ cur = build_attn(inp_attn,
+ nullptr, nullptr, nullptr,
+ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, hparams.f_attention_scale, il);
+ cb(cur, "attn_raw", il);
+
+ if (layer.attn_xsa_alpha) {
+ // remove from each head output its projection on the token's own value
+ const int64_t n_group = n_head/n_head_kv;
+
+ ggml_tensor * y = ggml_reshape_4d(ctx0, cur, n_embd_head, n_group, n_head_kv, n_tokens);
+ ggml_tensor * vn = ggml_l2_norm(ctx0, Vcur, hparams.f_xsa_eps);
+ vn = ggml_reshape_4d(ctx0, vn, n_embd_head, 1, n_head_kv, n_tokens);
+ vn = ggml_repeat(ctx0, vn, y);
+
+ ggml_tensor * proj = ggml_sum_rows(ctx0, ggml_mul(ctx0, y, vn));
+ proj = ggml_mul(ctx0, proj, ggml_reshape_4d(ctx0, layer.attn_xsa_alpha, 1, n_group, n_head_kv, 1));
+
+ cur = ggml_sub(ctx0, y, ggml_mul(ctx0, vn, proj));
+ cb(cur, "attn_xsa", il);
+ }
+
+ cur = ggml_reshape_3d(ctx0, cur, n_embd_head, n_head, n_tokens);
+
+ if (layer.wqkv_gate) {
+ ggml_tensor * gate = ggml_sigmoid(ctx0, build_lora_mm(layer.wqkv_gate, attn_in));
+ cur = ggml_mul(ctx0, cur, ggml_reshape_3d(ctx0, gate, 1, n_head, n_tokens));
+ cb(cur, "attn_gated", il);
+ }
+
+ cur = ggml_reshape_2d(ctx0, cur, n_embd_head*n_head, n_tokens);
+ cur = build_lora_mm(layer.wo, cur, layer.wo_s);
+ cb(cur, "attn_out", il);
+ }
+
+ if (il == n_layer - 1 && inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ resid = ggml_get_rows(ctx0, resid, inp_out_ids);
+ }
+
+ cur = ggml_add(ctx0, ggml_mul(ctx0, resid, layer.attn_resid_scale), cur);
+ cb(cur, "attn_resid", il);
+
+ ggml_tensor * ffn_inp = cur;
+
+ cur = build_norm(cur, nullptr, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = build_ffn(cur,
+ layer.ffn_up, nullptr, nullptr,
+ layer.ffn_gate, nullptr, nullptr,
+ layer.ffn_down, nullptr, nullptr,
+ nullptr,
+ LLM_FFN_SILU, LLM_FFN_PAR, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, ggml_mul(ctx0, ffn_inp, layer.ffn_resid_scale), cur);
+
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
+ inpL = cur;
+ }
+
+ cur = build_norm(inpL, nullptr, nullptr, LLM_NORM_RMS, -1);
+
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ cur = build_lora_mm(model.output, cur, model.output_s);
+
+ // scale * sigmoid((x + shift) / temperature)
+ const auto & cap = hparams.final_logit_sigmoid_capping;
+ if (cap[0] != 0.0f && cap[2] != 0.0f) {
+ cur = ggml_scale_bias(ctx0, cur, 1.0f/cap[2], cap[1]/cap[2]);
+ cur = ggml_sigmoid(ctx0, cur);
+ cur = ggml_scale(ctx0, cur, cap[0]);
+ }
+
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+}
diff --git a/src/models/models.h b/src/models/models.h
index 6c938c458..0fe40f4d0 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -2056,6 +2056,19 @@ struct llama_model_plm : public llama_model_base {
};
+struct llama_model_limite : public llama_model_base {
+ llama_model_limite(const struct llama_model_params & params) : llama_model_base(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ struct graph : public llm_graph_context {
+ graph(const llama_model & model, const llm_graph_params & params);
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
struct llama_model_hrm_text : public llama_model_base {
llama_model_hrm_text(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index aff6a7c90..333551157 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -269,6 +269,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
n_embd = 160; // exercise per-head tensor split granularity with head size 80
} else if (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE) {
n_head = 4;
+ } else if (arch == LLM_ARCH_LIMITE) {
+ n_head = 4;
+ n_layer = 4; // dense connections on the last two layers
} else if (arch == LLM_ARCH_DEEPSEEK2
|| arch == LLM_ARCH_DEEPSEEK32
|| arch == LLM_ARCH_GLM_DSA
@@ -297,7 +300,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
if (arch == LLM_ARCH_QWEN3) {
n_head_kv = 1; // MQA coverage
} else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER ||
- arch == LLM_ARCH_KOLIBRI1) {
+ arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE) {
n_head_kv = 2; // GQA coverage
}
if (arch == LLM_ARCH_BEAM) {
@@ -458,7 +461,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
}
- if (arch == LLM_ARCH_KOLIBRI1) {
+ if (arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE) {
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, false);
}
@@ -594,6 +597,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f);
}
+ if (arch == LLM_ARCH_LIMITE) {
+ ms.add_kv(LLM_KV_ATTENTION_SCALE, 0.1f);
+ ms.add_kv(LLM_KV_ATTENTION_XSA_EPS, 1e-4f);
+ ms.add_kv(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, std::vector<float>({23.0f, 5.0f, 7.5f}));
+ ms.add_kv(LLM_KV_MUDD_FEED_FORWARD_LENGTH, uint32_t(8));
+ ms.add_kv(LLM_KV_MUDD_TAP_COUNT, uint32_t(2));
+ ms.add_kv(LLM_KV_MUDD_TAP_INDICES, std::vector<int32_t>({-1, -1, -1, -1, 0, 2, 1, 3}));
+ }
+
// dummy tokenizer: token ids are derived from fixed-size chunks and detokenized as hex ids
{
std::vector<std::string> tokenizer_list(n_vocab);