Commit a68799c60 for llama.cpp
commit a68799c6094020a020cee72895b8df36b3b81098
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Sun Nov 15 17:39:59 2026 +0100
model: support MiniMax-M3.1 (#32301)
diff --git a/conversion/__init__.py b/conversion/__init__.py
index afdd73e4c..fb6b8464c 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -220,6 +220,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"MiniMaxM2ForCausalLM": "minimax",
"MiniMaxM3SparseForCausalLM": "minimax",
"MiniMaxM3SparseForConditionalGeneration": "minimax",
+ "DSparkMiniMaxDraftModel": "minimax",
"Ministral3ForCausalLM": "mistral3",
"Ministral3Model": "mistral3",
"Mistral3ForConditionalGeneration": "mistral3",
diff --git a/conversion/base.py b/conversion/base.py
index ac231ac94..7558fdcb4 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -398,6 +398,9 @@ class ModelBase:
return data / scale.float()
def dequant_simple(weight: Tensor, scale: Tensor, block_size: Sequence[int] | None = None) -> Tensor:
+ if scale.dtype == torch.uint8:
+ # E8M0 exponent stored as raw bytes (MXFP8)
+ scale = (scale.float() - 127.0).exp2()
scale = scale.float()
# ModelOpt block scales are pre-shaped for broadcasting, e.g. [out/128, 1, in/128, 1]
@@ -538,7 +541,7 @@ class ModelBase:
s = self.model_tensors[name]
self.model_tensors[weight_name] = lambda w=w, s=s: dequant_bitnet(w(), s())
tensors_to_remove.append(name)
- elif quant_method in ("fp8", "bsgemm-moe"): # bsgemm-moe: drama-3, same block layout
+ elif quant_method in ("fp8", "bsgemm-moe", "mxfp8"): # bsgemm-moe: drama-3, same block layout
block_size = quant_config.get("weight_block_size")
for name in self.model_tensors.keys():
if name.endswith("_scale_inv"):
@@ -1357,10 +1360,10 @@ class ModelBase:
consumed.append(input_scale_name)
# Check if this is a per-expert tensor
- m = re.search(r'\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight$', name)
+ m = re.search(r'\.experts\.(\d+)\.(gate_proj|up_proj|down_proj|w1|w2|w3)\.weight$', name)
if m:
expert_id = int(m.group(1))
- proj_type = m.group(2)
+ proj_type = {"w1": "gate_proj", "w2": "down_proj", "w3": "up_proj"}.get(m.group(2), m.group(2))
bid_m = re.search(r'\.layers\.(\d+)\.', name)
bid = int(bid_m.group(1)) if bid_m else 0
key = (bid, proj_type)
@@ -1451,6 +1454,8 @@ class ModelBase:
quant_layers = quant_config.get("quantized_layers", quant_layers) or {}
nvfp4_compressed_tensors = self._is_nvfp4_compressed_tensors(quant_method, quant_format, quant_groups)
+ # e.g. MXFP8 dense weights with compressed-tensors NVFP4 experts
+ nvfp4_compressed_tensors |= quantization_config.get("moe_quant_format") == "nvfp4-pack-quantized"
self._nvfp4_global_algo = quant_algo
diff --git a/conversion/minimax.py b/conversion/minimax.py
index 4d856fab8..c9f512f5f 100644
--- a/conversion/minimax.py
+++ b/conversion/minimax.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+import json
+
from typing import Iterable, Sequence, TYPE_CHECKING
import torch
@@ -8,6 +10,7 @@ if TYPE_CHECKING:
from torch import Tensor
from .base import ModelBase, TextModel, MmprojModel, gguf, logger
+from .qwen import DSparkModel
@ModelBase.register("MiniMaxText01ForCausalLM")
@@ -170,6 +173,15 @@ class MiniMaxM2Model(TextModel):
class MiniMaxM3Model(MiniMaxM2Model):
model_arch = gguf.MODEL_ARCH.MINIMAXM3
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ # transformers 5.x folds sparse_attention_config and moe_layer_freq into other keys
+ with open(self.dir_model / "config.json", "r", encoding="utf-8") as f:
+ raw_hparams = json.load(f)
+ for key, value in raw_hparams.get("text_config", raw_hparams).items():
+ self.hparams.setdefault(key, value)
+
def tensor_force_quant(self, name, new_name, bid, n_dims):
if ".indexer." in new_name:
return gguf.GGMLQuantizationType.F32
@@ -188,6 +200,7 @@ class MiniMaxM3Model(MiniMaxM2Model):
self.gguf_writer.add_indexer_top_k(sac["sparse_topk_blocks"])
self.gguf_writer.add_indexer_block_size(sac["sparse_block_size"])
self.gguf_writer.add_indexer_local_blocks(sac["sparse_local_block"])
+ self.gguf_writer.add_indexer_types([bool(v) for v in sac["sparse_attention_freq"]])
moe_layer_freq = self.find_hparam(["moe_layer_freq"])
n_dense = 0
@@ -201,7 +214,47 @@ class MiniMaxM3Model(MiniMaxM2Model):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
# Gemma-style (1 + w) RMSNorm: bake the +1 in so llama.cpp can use plain RMSNorm
if name.endswith("norm.weight"):
- data_torch = data_torch + 1.0
+ data_torch = data_torch.float() + 1.0
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
+@ModelBase.register("DSparkMiniMaxDraftModel")
+class MiniMaxM3DSparkModel(DSparkModel):
+ model_arch = gguf.MODEL_ARCH.DFLASH
+
+ def __init__(self, dir_model, *args, **kwargs):
+ # transformers drops the draft keys when it parses the minimax_m3_vl config
+ with open(dir_model / "config.json", "r", encoding="utf-8") as f:
+ kwargs["hparams"] = json.load(f)
+ super().__init__(dir_model, *args, **kwargs)
+
+ # draft layers use a dense FFN, the MoE keys are copied from the target config
+ self.hparams["intermediate_size"] = self.hparams["dspark_ffn_hidden_size"]
+ for key in ("num_local_experts", "num_experts_per_tok", "scoring_func"):
+ self.hparams.pop(key, None)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+
+ self.gguf_writer.add_hidden_act(self.hparams["hidden_act"])
+
+ layer_types = self.hparams["layer_types"]
+ self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
+ self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in layer_types])
+
+ @classmethod
+ def filter_tensors(cls, item):
+ name, gen = item
+ name = name.removeprefix("language_model.model.dspark.").replace(".decoder_layer.", ".")
+ if name == "final_norm.weight":
+ name = "norm.weight"
+ return super().filter_tensors((name, gen))
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
+ # Gemma-style (1 + w) RMSNorm, same as the target
+ if name.endswith("norm.weight"):
+ data_torch = data_torch.float() + 1.0
yield from super().modify_tensors(data_torch, name, bid)
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 42924b938..194bbb92b 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -2970,9 +2970,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
} break;
case LLM_ARCH_MINIMAX_M3:
{
- // sparse (MSA) layers carry an indexer key cache, but leading dense layers do not
+ // only sparse (MSA) layers carry an indexer key cache
llama_kv_cache::layer_filter_cb filter_idx =
- [&](int32_t il) { return (uint32_t) il >= hparams.n_layer_dense_lead; };
+ [&](int32_t il) { return (uint32_t) il < hparams.n_layer() && hparams.is_indexer_full(il); };
res = new llama_kv_cache_msa(
*this,
diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp
index 96360e404..000222f4c 100644
--- a/src/models/dflash.cpp
+++ b/src/models/dflash.cpp
@@ -15,6 +15,8 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) {
if (ml.get_key(LLM_KV_HIDDEN_ACT, hidden_act, false)) {
if (hidden_act == "gelu" || hidden_act == "gelu_pytorch_tanh") {
hparams.llm_ffn_op = LLM_FFN_GELU;
+ } else if (hidden_act == "swigluoai") {
+ hparams.llm_ffn_op = LLM_FFN_SWIGLU_OAI_MOE;
} else if (hidden_act != "silu") {
throw std::runtime_error("unsupported DFlash hidden activation: " + hidden_act);
}
diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp
index 466985cc4..4f1184590 100644
--- a/src/models/minimax-m3.cpp
+++ b/src/models/minimax-m3.cpp
@@ -24,6 +24,12 @@ void llama_model_minimax_m3::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, hparams.indexer_block_size);
ml.get_key(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, hparams.indexer_local_blocks);
+ // BC for MiniMax-M3 GGUFs without indexer_types metadata: leading dense layers have no indexer
+ for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
+ hparams.is_indexer_full_impl[il] = il >= hparams.n_layer_dense_lead;
+ }
+ ml.get_key_or_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, hparams.n_layer(), false);
+
msa_p = {
/*.blk =*/ (int) hparams.indexer_block_size,
/*.topk_blocks =*/ (int) hparams.indexer_top_k,
@@ -79,8 +85,9 @@ void llama_model_minimax_m3::load_arch_tensors(llama_model_loader &) {
layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), { n_ff_exp * n_expert_shared, n_embd}, 0);
layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
+ }
- // indexer
+ if (hparams.is_indexer_full(i)) {
layer.index_q_proj = create_tensor(tn(LLM_TENSOR_INDEXER_Q_PROJ, "weight", i), {n_embd, hparams.indexer_n_head * hparams.indexer_head_size}, 0);
layer.index_k_proj = create_tensor(tn(LLM_TENSOR_INDEXER_K_PROJ, "weight", i), {n_embd, hparams.indexer_head_size}, 0);
layer.index_q_norm = create_tensor(tn(LLM_TENSOR_INDEXER_Q_NORM, "weight", i), {hparams.indexer_head_size}, 0);
@@ -300,7 +307,11 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap
ggml_tensor * inp_out_ids = build_inp_out_ids();
+ const bool extract_last = (size_t) n_layer < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[n_layer];
+
for (int il = 0; il < n_layer; ++il) {
+ res->t_layer_inp[il] = inpL;
+
ggml_tensor * inpSA = inpL;
// self-attention
@@ -325,7 +336,7 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
- const bool is_sparse = msa_enabled && il >= (int) hparams.n_layer_dense_lead;
+ const bool is_sparse = msa_enabled && hparams.is_indexer_full(il);
if (!is_sparse) {
cur = build_attn(inp_attn, model.layers[il].wo, NULL, model.layers[il].wo_s,
@@ -529,7 +540,7 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap
}
}
- if (il == n_layer - 1 && inp_out_ids) {
+ if (il == n_layer - 1 && inp_out_ids && !extract_last) {
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
}
@@ -589,6 +600,13 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap
inpL = cur;
}
+ if (extract_last) {
+ res->t_layer_inp[n_layer] = inpL;
+ if (inp_out_ids) {
+ inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+ }
+ }
+
cur = inpL;
cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);