Commit a07421535 for llama.cpp
commit a074215352c382e66a108597bcf7f4f31e00d111
Author: Aman Gupta <amangupta052@gmail.com>
Date: Mon Oct 26 15:52:02 2026 +0800
deepseekv4.1: follow-up review fixes (#31137)
* kv-cache : use ggml_backend_tensor_copy to refill the DSV4 K mirrors
* llama : move the Engram pad id to llama_hparams and the token map to llama_model
The saver no longer needs the deepseek41 model class.
* deepseek41 : drop the F32 src1 request on the HC mixes matmul
* ggml : add softplus TODOs in the backends that still use log(1 + exp(x))
* convert : make the DSV4/DSV4.1 modify_tensors generators, drop the redundant vision filter
* deepseek41 : clean up comments
* deepseek41 : drop the F32 src1 requests on the router and indexer matmuls
* convert : use format_tensor_name directly in DeepseekV4Model
* s/comp_kv_src/comp_il_src/g
* cont : fix TODOs for log1p
---------
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
diff --git a/conversion/deepseek.py b/conversion/deepseek.py
index 946de5347..0f8e2ffee 100644
--- a/conversion/deepseek.py
+++ b/conversion/deepseek.py
@@ -803,9 +803,6 @@ class DeepseekV4Model(TextModel):
self._dsv4_mxfp4_generated = True
return ()
- def _format_dsv4_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None, suffix: str = ".weight") -> str:
- return self.format_tensor_name(key, bid, suffix)
-
def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]:
root_map: dict[str, tuple[gguf.MODEL_TENSOR, str]] = {
"embed.weight": (gguf.MODEL_TENSOR.TOKEN_EMBD, ".weight"),
@@ -879,17 +876,17 @@ class DeepseekV4Model(TextModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if re.match(r"layers\.\d+\.ffn\.experts\.\d+\.w[123]\.(weight|scale)$", name):
- return []
+ return
# hash layers route text tokens via tid2eid and image tokens via bias_vl; gate.bias is unused
if name.endswith(".ffn.gate.bias") and bid is not None and bid < self.hparams["num_hash_layers"]:
- return []
+ return
tensor_key, suffix = self._map_dsv4_tensor_name(name, bid)
if tensor_key == gguf.MODEL_TENSOR.FFN_GATE_TID2EID:
- return []
+ return
- return [(self._format_dsv4_tensor_name(tensor_key, bid, suffix), data_torch)]
+ yield self.format_tensor_name(tensor_key, bid, suffix), data_torch
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
del bid # unused
diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py
index 4b87d5c28..53980c1d0 100644
--- a/conversion/deepseek41.py
+++ b/conversion/deepseek41.py
@@ -145,12 +145,6 @@ class DeepseekV41Model(DeepseekV4Model):
# num_hidden_layers lives in text_config, which is merged only after indexing; MTP stages are skipped anyway
return TextModel.index_tensors(self, remote_hf_model_id=remote_hf_model_id)
- @classmethod
- def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
- if item[0].startswith("vision."):
- return None
- return super().filter_tensors(item)
-
def set_gguf_parameters(self):
super().set_gguf_parameters()
hp = self.hparams
@@ -282,12 +276,13 @@ class DeepseekV41Model(DeepseekV4Model):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if name.endswith(".engram.k_weight"):
- return [] # folded into engram_gate
+ return # folded into engram_gate
if name.endswith(".engram.q_weight"):
# Engram gate weight = q_weight * k_weight (bf16 * bf16 is exact in f32)
k = self.model_tensors[name.removesuffix("q_weight") + "k_weight"]
- return [(self.format_tensor_name(gguf.MODEL_TENSOR.ENGRAM_GATE, bid), data_torch.float() * k().float())]
- return super().modify_tensors(data_torch, name, bid)
+ yield self.format_tensor_name(gguf.MODEL_TENSOR.ENGRAM_GATE, bid), data_torch.float() * k().float()
+ return
+ yield from super().modify_tensors(data_torch, name, bid)
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
if new_name.endswith((".engram_gate.weight", ".ffn_gate_inp.weight")) or name in self._dsv4_f32_tensors:
@@ -411,13 +406,15 @@ class DeepseekV41DSparkModel(DeepseekV41Model):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
if name.endswith(".ffn.gate.bias_vl"):
- return []
+ return
if name in self._DSPARK_ROOT:
- return [(self.format_tensor_name(self._DSPARK_ROOT[name], None), data_torch)]
+ yield self.format_tensor_name(self._DSPARK_ROOT[name], None), data_torch
+ return
m = re.match(r"layers\.\d+\.(main_proj\.weight|main_norm\.weight|norm\.weight)$", name)
if m is not None:
- return [(self.format_tensor_name(self._DSPARK_ROOT[m.group(1)], None), data_torch)]
- return super().modify_tensors(data_torch, name, bid)
+ yield self.format_tensor_name(self._DSPARK_ROOT[m.group(1)], None), data_torch
+ return
+ yield from super().modify_tensors(data_torch, name, bid)
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
if new_name == "conf_proj.weight":
diff --git a/src/llama-hparams.h b/src/llama-hparams.h
index a3bf9e688..760bb8ac6 100644
--- a/src/llama-hparams.h
+++ b/src/llama-hparams.h
@@ -358,6 +358,8 @@ struct llama_hparams {
llama_token ple_embd_token = LLAMA_TOKEN_NULL;
// hash constant sets, stored back to back: 1 shared by all PLE layers (qwen4exp), or 1 per PLE layer in layer order (deepseek41)
uint32_t ple_n_modules = 1;
+ // deepseek41 Engram: the token that pads the n-gram history, an index into llama_model::engram_token_map
+ uint32_t engram_pad_id = 0;
// the file lists PLE layer indices, so this is never a per-layer gguf array and can hold one bit per layer
std::bitset<LLAMA_MAX_LAYERS> is_ple_impl;
// the hash multipliers reach ~2e13 and have to stay 64-bit
diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp
index c18d913da..6186f9d34 100644
--- a/src/llama-kv-cache-dsv4.cpp
+++ b/src/llama-kv-cache-dsv4.cpp
@@ -428,8 +428,11 @@ static void dsv4_state_read_k_cache(
dsv4_state_read_tensor_streams(io, kv->get_k_storage(il), kv_size, n_rows_ref, s0, ns);
}
- // refill the mirrors from their source rows just read
- std::vector<uint8_t> buf;
+ if (n_rows_ref == 0) {
+ return;
+ }
+
+ // refill the mirrors from their sources; a mirror gets the same rows as its source, so a whole copy is exact
for (uint32_t il : kv->get_layer_ids()) {
const int32_t src = kv_src[il];
if (src == (int32_t) il) {
@@ -437,20 +440,11 @@ static void dsv4_state_read_k_cache(
}
ggml_tensor * t_src = kv->get_k_storage(src);
ggml_tensor * t_dst = kv->get_k_storage(il);
- GGML_ASSERT(t_src->type == t_dst->type && t_src->ne[0] == t_dst->ne[0]);
-
- const size_t row_size = ggml_row_size(t_src->type, t_src->ne[0]);
- const size_t stream_stride = (size_t) kv_size*row_size;
- const size_t size = (size_t) n_rows_ref*row_size;
- if (size == 0) {
- continue;
- }
- buf.resize(size);
- for (uint32_t s = 0; s < ns; ++s) {
- const size_t offset = (size_t) (s0 + s)*stream_stride;
- ggml_backend_tensor_get(t_src, buf.data(), offset, size);
- ggml_backend_tensor_set(t_dst, buf.data(), offset, size);
+ if (t_src->type != t_dst->type || !ggml_are_same_shape(t_src, t_dst)) {
+ throw std::runtime_error("DSV4 K-cache mirror layout mismatch");
}
+
+ ggml_backend_tensor_copy(t_src, t_dst);
}
}
@@ -1352,10 +1346,10 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
// crosses devices. no mirrors without offload
const uint32_t n_layer = model.hparams.n_layer();
comp_read.assign(n_layer, -1);
- comp_kv_src.resize(n_layer);
+ comp_il_src.resize(n_layer);
for (uint32_t il = 0; il < n_layer; ++il) {
const int32_t src = is_v41 ? model.hparams.dsv41_kv_src[il] : (int32_t) il;
- comp_kv_src[il] = src >= 0 ? src : (int32_t) il;
+ comp_il_src[il] = src >= 0 ? src : (int32_t) il;
if (model.hparams.dsv4_compress_ratios[il] == 0) {
continue;
}
@@ -1366,7 +1360,7 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
// the first layer of this source on this device holds the mirror
comp_read[il] = il;
for (int32_t j = src + 1; j < (int32_t) il; ++j) {
- if (comp_read[j] == j && comp_kv_src[j] == src && model.dev_layer(j) == model.dev_layer(il)) {
+ if (comp_read[j] == j && comp_il_src[j] == src && model.dev_layer(j) == model.dev_layer(il)) {
comp_read[il] = j;
break;
}
@@ -1811,9 +1805,9 @@ void llama_kv_cache_dsv4::state_write(llama_io_write_i & io, llama_seq_id seq_id
const uint32_t n_rows_lid = seq_id >= 0 ?
dsv4_state_n_used_k_rows(pos_max, lid_state->get_ratio(), kv_lid->get_size()) : kv_lid->get_size();
- dsv4_state_write_k_cache(io, kv_csa.get(), seq_id, flags, n_rows_csa, comp_kv_src);
- dsv4_state_write_k_cache(io, kv_hca.get(), seq_id, flags, n_rows_hca, comp_kv_src);
- dsv4_state_write_k_cache(io, kv_lid.get(), seq_id, flags, n_rows_lid, comp_kv_src);
+ dsv4_state_write_k_cache(io, kv_csa.get(), seq_id, flags, n_rows_csa, comp_il_src);
+ dsv4_state_write_k_cache(io, kv_hca.get(), seq_id, flags, n_rows_hca, comp_il_src);
+ dsv4_state_write_k_cache(io, kv_lid.get(), seq_id, flags, n_rows_lid, comp_il_src);
}
csa_state->state_write(io, seq_id, flags, rs_idx);
@@ -1851,9 +1845,9 @@ void llama_kv_cache_dsv4::state_read(llama_io_read_i & io, llama_seq_id seq_id,
if (!partial_only) {
clear_compressed(seq_id, true);
- dsv4_state_read_k_cache(io, kv_csa.get(), seq_id, flags, comp_kv_src);
- dsv4_state_read_k_cache(io, kv_hca.get(), seq_id, flags, comp_kv_src);
- dsv4_state_read_k_cache(io, kv_lid.get(), seq_id, flags, comp_kv_src);
+ dsv4_state_read_k_cache(io, kv_csa.get(), seq_id, flags, comp_il_src);
+ dsv4_state_read_k_cache(io, kv_hca.get(), seq_id, flags, comp_il_src);
+ dsv4_state_read_k_cache(io, kv_lid.get(), seq_id, flags, comp_il_src);
}
csa_state->state_read(io, seq_id, flags);
diff --git a/src/llama-kv-cache-dsv4.h b/src/llama-kv-cache-dsv4.h
index 99ac73ddb..bfc52947f 100644
--- a/src/llama-kv-cache-dsv4.h
+++ b/src/llama-kv-cache-dsv4.h
@@ -170,7 +170,7 @@ private:
std::vector<uint32_t> rs_idx;
// [n_layer] storage layer -> layer whose rows it holds (itself, or the source of a mirror)
- std::vector<int32_t> comp_kv_src;
+ std::vector<int32_t> comp_il_src;
// [n_layer] layer -> layer whose compressed-KV storage it reads (itself, its V4.1 kv source or a mirror), -1 = none
std::vector<int32_t> comp_read;
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index 752e6361f..56a42c756 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -8,7 +8,6 @@
#include "llama-hparams.h"
#include "llama-model.h"
#include "llama-vocab.h"
-#include "models/models.h"
#include <cstdint>
#include <string>
@@ -342,8 +341,6 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_HYPER_CONNECTION_LOW_RANK, hparams.hc_low_rank);
if (model->arch == LLM_ARCH_DEEPSEEK41) {
- const auto * m41 = dynamic_cast<const llama_model_deepseek41 *>(model);
- GGML_ASSERT(m41 != nullptr);
std::vector<uint32_t> kv_src;
std::vector<uint32_t> idx_src;
for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
@@ -362,8 +359,8 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE, hparams.dsv41_cand_block_size);
}
if (hparams.ple_n_heads > 0) {
- add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, m41->engram_pad_id);
- add_kv(LLM_KV_ENGRAM_TOKEN_MAP, m41->engram_token_map);
+ add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, hparams.engram_pad_id);
+ add_kv(LLM_KV_ENGRAM_TOKEN_MAP, model->engram_token_map);
}
}
diff --git a/src/llama-model.h b/src/llama-model.h
index 01fbda319..ccc2602e5 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -734,6 +734,9 @@ struct llama_model {
std::vector<std::string> classifier_labels;
std::vector<std::string> decision_labels;
+ // deepseek41 Engram: token id -> id of its normalized form, hashed into the n-grams
+ std::vector<int32_t> engram_token_map;
+
struct ggml_tensor * tok_embd = nullptr;
struct ggml_tensor * type_embd = nullptr;
struct ggml_tensor * pos_embd = nullptr;
diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp
index 7dc5ee3a5..13f6a5acc 100644
--- a/src/models/deepseek4.cpp
+++ b/src/models/deepseek4.cpp
@@ -350,7 +350,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_pre(
ggml_tensor ** post,
ggml_tensor ** comb,
int il) const {
- return build_hc_pre(x, build_hc_mixes(x, hc_fn, hc_scale, hc_base, post, comb, false, il), il);
+ return build_hc_pre(x, build_hc_mixes(x, hc_fn, hc_scale, hc_base, post, comb, il), il);
}
ggml_tensor * llama_model_deepseek4::graph::build_hc_mixes(
@@ -360,7 +360,6 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_mixes(
ggml_tensor * hc_base,
ggml_tensor ** post,
ggml_tensor ** comb,
- bool f32_src,
int il) const {
const int64_t hc = hparams.dsv4_hc_mult;
const int64_t hc_dim = hc*n_embd;
@@ -373,10 +372,6 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_mixes(
ggml_tensor * flat = ggml_reshape_2d(ctx0, x, hc_dim, nt);
ggml_tensor * flat_norm = ggml_rms_norm(ctx0, flat, norm_rms_eps);
ggml_tensor * mixes = ggml_mul_mat(ctx0, hc_fn, flat_norm);
- if (f32_src && !ggml_is_quantized(hc_fn->type)) {
- // F32 weights (model.py hc_pre runs in fp32): request F32 inputs, the mixes are near-tie sensitive
- ggml_prec_set_src(mixes, GGML_PREC_F32, 1);
- }
cb(mixes, "hc_mixes", il);
ggml_tensor * scale_pre = dsv4_view_1d(ctx0, hc_scale, 1, 0);
diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp
index 49e95f54c..6a9003d42 100644
--- a/src/models/deepseek41.cpp
+++ b/src/models/deepseek41.cpp
@@ -11,9 +11,7 @@
#include <string>
// ref: deepseek-ai/DeepSeek-V4.1-Flash-1017 inference/model.py, engram.py
-// compressed layers attend over [window | all compressed rows] with top-k as a mask. with flash attention the default
-// is sparse: fused indexer, FA bounded to window + top-k, decode-sized ubatches gather only the selected rows (same
-// keys and mask, different key order). LLAMA_DSV41_SPARSE=0 restores the reference path
+// LLAMA_DSV41_SPARSE=0 disables the sparse attention path (see build_attention41)
static constexpr int64_t DSV41_ENGRAM_DEAD = -1;
@@ -96,7 +94,7 @@ void llama_model_deepseek41::load_arch_hparams(llama_model_loader & ml) {
ml.get_arr(LLM_KV_PLE_LAYER_MULTIPLIERS, mult);
ml.get_arr(LLM_KV_PLE_HEAD_OFFSETS, offsets);
ml.get_arr(LLM_KV_PLE_HEAD_VOCAB_SIZES, primes);
- ml.get_key(LLM_KV_ENGRAM_PAD_TOKEN_ID, engram_pad_id);
+ ml.get_key(LLM_KV_ENGRAM_PAD_TOKEN_ID, hparams.engram_pad_id);
ml.get_arr(LLM_KV_ENGRAM_TOKEN_MAP, engram_token_map);
const size_t n_e = e_layers.size();
@@ -105,7 +103,7 @@ void llama_model_deepseek41::load_arch_hparams(llama_model_loader & ml) {
if (n_cols == 0 || hparams.ple_head_dim == 0 ||
mult.size() != n_e*n_gram || primes.size() != n_e*n_cols || offsets.size() != n_e*n_cols ||
mult.size() > LLAMA_MAX_PLE_NGRAM || primes.size() > LLAMA_MAX_PLE_HEADS ||
- engram_pad_id >= engram_token_map.size()) {
+ hparams.engram_pad_id >= engram_token_map.size()) {
throw std::runtime_error("DeepSeek-V4.1 engram metadata is inconsistent");
}
for (size_t e = 0; e < n_e; ++e) {
@@ -247,7 +245,7 @@ void llama_model_deepseek41::engram_hash(int e, const llama_ubatch & ubatch, con
const uint64_t * mult = hparams.ple_layer_multipliers.data() + (size_t) e*n_gram;
const uint32_t * primes = hparams.ple_head_vocab_sizes.data() + (size_t) e*n_cols;
- const int64_t pad = engram_token_map[engram_pad_id];
+ const int64_t pad = engram_token_map[hparams.engram_pad_id];
const uint32_t * offs = hparams.ple_head_offsets.data() + (size_t) e*n_cols;
const auto cid = [&](llama_token t) -> int64_t {
@@ -279,8 +277,7 @@ void llama_model_deepseek41::engram_hash(int e, const llama_ubatch & ubatch, con
}
}
-// cand_pin: +1e30 (the reference uses +inf) on the block holding each query's newest compressed row, so a partly
-// filled block always survives; a finite pin keeps fully masked blocks at -inf instead of NaN
+// cand_pin: +1e30 on the block with the newest compressed row of each query, so a partly filled block is always kept; the reference uses +inf, but a finite pin keeps fully masked blocks at -inf instead of NaN
class llm_graph_input_dsv41 : public llm_graph_input_i {
public:
llm_graph_input_dsv41(const llama_model_deepseek41 & model, const llama_kv_cache_dsv4_context * mctx) :
@@ -535,8 +532,7 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41(
if (hparams.dsv41_topk_src[il] == il) {
ggml_tensor * ik = ggml_view_4d(ctx0, comp_all, n_idx_embd, 1, n_comp, n_stream,
comp_all->nb[1], comp_all->nb[2], comp_all->nb[3], ggml_row_size(comp_all->type, n_embd_head));
- // the fused indexer reads the strided view in place, the unfused path needs it contiguous; quantized
- // caches are dequantized straight from the strided view (CUDA has no same-type copy of quantized views)
+ // the fused indexer reads the strided view in place, the unfused path needs it contiguous; CUDA has no same-type copy of quantized views, so dequantize them
const bool fused_idx = sparse && cparams.fused_lid && ci.kq_mask->type == GGML_TYPE_F16;
if (ggml_is_quantized(ik->type)) {
ik = ggml_cast(ctx0, ik, GGML_TYPE_F32);
@@ -551,8 +547,7 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41(
ggml_tensor * iw = ggml_scale(ctx0, build_lora_mm(layer.indexer_proj, cur), 1.0f/sqrtf(float(n_idx_embd*n_idx_head)));
iw = ggml_view_4d(ctx0, iw, iw->ne[0], nt/n_stream, 1, n_stream, iw->nb[1], iw->nb[1]*(nt/n_stream), iw->nb[1]*(nt/n_stream), 0);
- // chunk queries so the [n_ih, tokens, n_comp] scores stay near 2^26 elements (at most 64 chunks); the fused
- // indexer has no such intermediate and runs as one chunk
+ // chunk queries so the [n_ih, tokens, n_comp] scores stay near 2^26 elements (at most 64 chunks); the fused indexer runs as one chunk
const int64_t n_tok = nt/n_stream;
GGML_ASSERT(ci.kq_mask->ne[1] == n_tok);
const int64_t chunk = fused_idx ? n_tok :
@@ -582,9 +577,6 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41(
ggml_tensor * iqc = ggml_view_4d(ctx0, iq, iq->ne[0], iq->ne[1], nc, n_stream, iq->nb[1], iq->nb[2], iq->nb[3], t0*iq->nb[2]);
ggml_tensor * kq = ggml_mul_mat(ctx0, ik_p, ggml_permute(ctx0, iqc, 0, 2, 1, 3));
- if (!ggml_is_quantized(ik->type)) {
- ggml_prec_set_src(kq, GGML_PREC_F32, 1);
- }
kq = ggml_cont(ctx0, ggml_permute(ctx0, kq, 2, 1, 0, 3));
score = ggml_sum_rows(ctx0, ggml_mul(ctx0, ggml_relu(ctx0, kq), rows(iw)));
ggml_tensor * mask = rows(ci.kq_mask);
@@ -656,14 +648,12 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41(
const int64_t k_sel = top_k->ne[0];
const int64_t n_raw = raw_k->ne[2];
- // few queries per stream: gather each query's top-k rows and attend over [window | selected], one FA stream
- // per query, instead of a full n_comp mask per layer. same keys and mask as dense, different key order
+ // few queries per stream: attend over [window | top-k rows of the query], one FA stream per query; same keys and mask as dense, different key order
const bool gather = sparse && cparams.flash_attn && raw_mask->type == GGML_TYPE_F16 && ci.kq_mask->type == GGML_TYPE_F16 &&
((n_stream == 1 && nt <= 32) || nt == n_stream);
if (gather) {
- // ggml_top_k returns the rows in no particular order (CUB DeviceTopK can vary between runs) and FA sums the
- // gathered keys in that order: sort the rows once per top-k source so the result is deterministic
+ // ggml_top_k returns the rows in no fixed order and FA sums the keys in that order: sort them so the result is deterministic
ggml_tensor *& sel = gather_sel_of[topk_src];
if (!sel) {
GGML_ASSERT(n_comp <= (INT64_C(1) << 24)); // row ids are exact in F32
@@ -714,8 +704,7 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41(
}
ggml_tensor * k_all = ggml_concat(ctx0, raw_k, comp_k, 2);
- // mask: window row, then comp mask at the selected rows (-inf where top-k picked an invisible row);
- // one per top-k source, shared by its layers
+ // window mask, then the comp mask at the selected rows (-inf where top-k picked an invisible row); one per top-k source
ggml_tensor *& kq_mask = gather_mask_of[topk_src];
if (!kq_mask) {
ggml_tensor * sel_mask = ggml_get_rows(ctx0, ggml_reshape_3d(ctx0, ci.kq_mask, 1, n_comp, nt), sel);
@@ -730,8 +719,7 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41(
ggml_tensor * comp_k = ggml_view_4d(ctx0, comp_all, n_embd_head, 1, n_comp, n_stream,
comp_all->nb[1], comp_all->nb[2], comp_all->nb[3], 0);
if (k_rot) {
- // compressed rows are stored unrotated: rotate in F32, then cast to the cache type (dequantized from
- // the strided view: CUDA has no same-type quantized copy)
+ // compressed rows are stored unrotated: rotate in F32, then cast to the cache type
comp_k = comp_k->type != GGML_TYPE_F32 ? ggml_cast(ctx0, comp_k, GGML_TYPE_F32) : ggml_cont(ctx0, comp_k);
comp_k = llama_mul_mat_hadamard(ctx0, comp_k, k_rot);
if (comp_k->type != raw_k->type) {
@@ -862,7 +850,7 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g
ggml_tensor * post = nullptr;
ggml_tensor * comb = nullptr;
- ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, true, il);
+ ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il);
// the first layer's attention collapses with the identity pre-mix: copy 0
cur = pre_prev ? build_hc_pre(inpL, pre_prev, il) : ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd, n_tokens, inpL->nb[2], 0));
@@ -877,7 +865,7 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g
cb(inpL, "hc_attn_post", il);
residual = inpL;
- pre_prev = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, true, il);
+ pre_prev = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il);
cur = build_hc_pre(inpL, attn_pre, il);
cb(cur, "hc_ffn_pre", il);
@@ -889,12 +877,9 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "ffn_norm", il);
- // F32 router inputs: expert selection is near-tie sensitive
+ // F32 router accumulation: expert selection is near-tie sensitive
ggml_tensor * logits = build_lora_mm(layer.ffn_gate_inp, cur);
ggml_prec_set_acc(logits, GGML_PREC_F32);
- if (!ggml_is_quantized(layer.ffn_gate_inp->type)) {
- ggml_prec_set_src(logits, GGML_PREC_F32, 1);
- }
cb(logits, "ffn_moe_logits", il);
// image (embedding) tokens use the vision router bias, as DeepSeek-V4
diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp
index 8b6d6abdb..6bbb534f0 100644
--- a/src/models/dflash.cpp
+++ b/src/models/dflash.cpp
@@ -1406,7 +1406,7 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll
ggml_tensor * post = nullptr;
ggml_tensor * comb = nullptr;
- ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, false, il);
+ ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il);
ggml_tensor * cur;
if (pre_prev == nullptr) {
@@ -1465,7 +1465,7 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll
cb(inpL, "hc_attn_post", il);
residual = inpL;
- ggml_tensor * ffn_pre = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, false, il);
+ ggml_tensor * ffn_pre = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il);
cur = build_hc_pre(inpL, attn_pre, il);
cb(cur, "hc_ffn_pre", il);
diff --git a/src/models/models.h b/src/models/models.h
index 59a4e5703..37dbacdb9 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -1376,7 +1376,6 @@ struct llama_model_deepseek4 : public llama_model_base {
ggml_tensor * hc_base,
ggml_tensor ** post,
ggml_tensor ** comb,
- bool f32_src,
int il) const;
ggml_tensor * build_hc_post(
@@ -1498,11 +1497,7 @@ struct llama_model_deepseek41 : public llama_model_deepseek4 {
void load_arch_hparams(llama_model_loader & ml) override;
void load_arch_tensors(llama_model_loader & ml) override;
- // Engram: the hash constants are hparams.ple_* (one module per Engram layer), hashed on the host (ggml has no
- // int64 mul/xor/mod); the token map and its pad id are V4.1 only
- uint32_t engram_pad_id = 0;
- std::vector<int32_t> engram_token_map;
-
+ // Engram: the hash constants are hparams.ple_* (one module per Engram layer), hashed on the host (ggml has no int64 mul/xor/mod)
int engram_index(int il) const;
void engram_hash(int e, const llama_ubatch & ubatch, const std::vector<llama_token> & prev, std::vector<int32_t> & out) const;
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index fe3a67c64..3e9439252 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -9996,6 +9996,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
std::default_random_engine rng(0);
// unary ops
+ // TODO: add GGML_UNARY_OP_SOFTPLUS test that is sensitive to using `log1p(exp(x))` vs `log(exp(x) + 1)`
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) {
for (int v : {0, 1}) {
for (int op = 0; op < GGML_UNARY_OP_COUNT; op++) {