Commit 4b851fbee for llama.cpp
commit 4b851fbee8867136a0c561c8a3521f2fe7d53252
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Mon Oct 26 16:19:47 2026 +0200
models : replace ggml_tensor** out-params with output structs (#31142)
The hyper-connection and DSA graph builders wrote their extra outputs
(post / comb gates, carried top-k selections, inject gates) through
ggml_tensor** out-params. Return them in a small struct next to the
primary output instead.
Assisted-by: dsh:llama.cpp/ggml-org/Qwen3.8-Flash-Next-GGUF (MacBook, M5 Max, 128GB)
diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp
index 13f6a5acc..4d58b2fd2 100644
--- a/src/models/deepseek4.cpp
+++ b/src/models/deepseek4.cpp
@@ -342,24 +342,23 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_sinkhorn(
return comb;
}
-ggml_tensor * llama_model_deepseek4::graph::build_hc_pre(
+llama_model_deepseek4::graph::hc_pre
+llama_model_deepseek4::graph::build_hc_pre(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
- ggml_tensor ** post,
- ggml_tensor ** comb,
int il) const {
- return build_hc_pre(x, build_hc_mixes(x, hc_fn, hc_scale, hc_base, post, comb, il), il);
+ hc_mixes mixes = build_hc_mixes(x, hc_fn, hc_scale, hc_base, il);
+ return { build_hc_pre(x, mixes.pre, il), mixes.post, mixes.comb };
}
-ggml_tensor * llama_model_deepseek4::graph::build_hc_mixes(
+llama_model_deepseek4::graph::hc_mixes
+llama_model_deepseek4::graph::build_hc_mixes(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
- ggml_tensor ** post,
- ggml_tensor ** comb,
int il) const {
const int64_t hc = hparams.dsv4_hc_mult;
const int64_t hc_dim = hc*n_embd;
@@ -386,28 +385,29 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_mixes(
pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps);
cb(pre, "hc_pre", il);
- *post = dsv4_view_2d(ctx0, mixes, hc, nt, hc);
- *post = dsv4_hc_affine(ctx0, *post, scale_post, base_post);
- *post = ggml_sigmoid(ctx0, *post);
- *post = ggml_scale(ctx0, *post, 2.0f);
- cb(*post, "hc_post", il);
+ ggml_tensor * post = dsv4_view_2d(ctx0, mixes, hc, nt, hc);
+ post = dsv4_hc_affine(ctx0, post, scale_post, base_post);
+ post = ggml_sigmoid(ctx0, post);
+ post = ggml_scale(ctx0, post, 2.0f);
+ cb(post, "hc_post", il);
+ ggml_tensor * comb = nullptr;
if (cparams.fused_dsv4_hc_comb) {
- *comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps,
+ comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps,
(int32_t) hparams.dsv4_hc_sinkhorn_iters);
- res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, *comb, il});
+ res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, comb, il});
} else {
ggml_tensor * scale_comb = dsv4_view_1d(ctx0, hc_scale, 1, 2);
ggml_tensor * base_comb = dsv4_view_1d(ctx0, hc_base, hc*hc, 2*hc);
- *comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc);
- *comb = dsv4_hc_affine(ctx0, *comb, scale_comb, base_comb);
- *comb = ggml_reshape_3d(ctx0, *comb, hc, hc, nt);
- *comb = build_hc_sinkhorn(*comb, il);
+ comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc);
+ comb = dsv4_hc_affine(ctx0, comb, scale_comb, base_comb);
+ comb = ggml_reshape_3d(ctx0, comb, hc, hc, nt);
+ comb = build_hc_sinkhorn(comb, il);
}
- cb(*comb, "hc_comb", il);
+ cb(comb, "hc_comb", il);
- return pre;
+ return { pre, post, comb };
}
ggml_tensor * llama_model_deepseek4::graph::build_hc_post(
@@ -1252,14 +1252,12 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p
}
ggml_tensor * residual = inpL;
- ggml_tensor * post = nullptr;
- ggml_tensor * comb = nullptr;
-
- cur = build_hc_pre(inpL,
+ const hc_pre attn_hc = build_hc_pre(inpL,
model.layers[il].hc_attn_fn,
model.layers[il].hc_attn_scale,
model.layers[il].hc_attn_base,
- &post, &comb, il);
+ il);
+ cur = attn_hc.cur;
cb(cur, "hc_attn_pre", il);
cur = build_norm(cur, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);
@@ -1267,20 +1265,21 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p
cur = build_attention(model, inp_dsv4, cur, inp_pos, il);
- inpL = build_hc_post(cur, residual, post, comb, il);
+ inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il);
cb(inpL, "hc_attn_post", il);
residual = inpL;
- cur = build_hc_pre(inpL,
+ const hc_pre ffn_hc = build_hc_pre(inpL,
model.layers[il].hc_ffn_fn,
model.layers[il].hc_ffn_scale,
model.layers[il].hc_ffn_base,
- &post, &comb, il);
+ il);
+ cur = ffn_hc.cur;
cb(cur, "hc_ffn_pre", il);
ggml_build_forward_expand(gf, residual);
- ggml_build_forward_expand(gf, post);
- ggml_build_forward_expand(gf, comb);
+ ggml_build_forward_expand(gf, ffn_hc.post);
+ ggml_build_forward_expand(gf, ffn_hc.comb);
cur = build_norm(cur, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "ffn_norm", il);
@@ -1330,7 +1329,7 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p
cur = ggml_add(ctx0, moe_out, ffn_shexp);
cb(cur, "ffn_out", il);
- inpL = build_hc_post(cur, residual, post, comb, il);
+ inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il);
inpL = build_cvec(inpL, il);
cb(inpL, "l_last", il);
}
@@ -1430,14 +1429,12 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm
cb(inpL, "mtp_eh_proj", il);
ggml_tensor * residual = inpL;
- ggml_tensor * post = nullptr;
- ggml_tensor * comb = nullptr;
-
- ggml_tensor * cur = build_hc_pre(inpL,
+ hc_pre attn_hc = build_hc_pre(inpL,
layer.hc_attn_fn,
layer.hc_attn_scale,
layer.hc_attn_base,
- &post, &comb, il);
+ il);
+ ggml_tensor * cur = attn_hc.cur;
cb(cur, "mtp_hc_attn_pre", il);
cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
@@ -1445,15 +1442,16 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm
cur = build_attention(model, inp_attn, cur, inp_pos, il);
- inpL = build_hc_post(cur, residual, post, comb, il);
+ inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il);
cb(inpL, "mtp_hc_attn_post", il);
residual = inpL;
- cur = build_hc_pre(inpL,
+ hc_pre ffn_hc = build_hc_pre(inpL,
layer.hc_ffn_fn,
layer.hc_ffn_scale,
layer.hc_ffn_base,
- &post, &comb, il);
+ il);
+ cur = ffn_hc.cur;
cb(cur, "mtp_hc_ffn_pre", il);
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
@@ -1483,7 +1481,7 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm
cur = ggml_add(ctx0, moe_out, ffn_shexp);
cb(cur, "mtp_ffn_out", il);
- inpL = build_hc_post(cur, residual, post, comb, il);
+ inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il);
inpL = build_cvec(inpL, il);
cb(inpL, "mtp_l_out", il);
diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp
index 6a9003d42..97b2cbabe 100644
--- a/src/models/deepseek41.cpp
+++ b/src/models/deepseek41.cpp
@@ -847,10 +847,7 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g
}
ggml_tensor * residual = inpL;
- ggml_tensor * post = nullptr;
- ggml_tensor * comb = nullptr;
-
- ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il);
+ const hc_mixes attn_mix = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il);
// the first layer's attention collapses with the identity pre-mix: copy 0
cur = pre_prev ? build_hc_pre(inpL, pre_prev, il) : ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd, n_tokens, inpL->nb[2], 0));
@@ -861,18 +858,19 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g
cur = build_attention41(model, inp_dsv4, cur, inp_pos, cand_pin, il);
- inpL = build_hc_post(cur, residual, post, comb, il);
+ inpL = build_hc_post(cur, residual, attn_mix.post, attn_mix.comb, il);
cb(inpL, "hc_attn_post", il);
residual = inpL;
- pre_prev = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il);
+ const hc_mixes ffn_mix = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il);
+ pre_prev = ffn_mix.pre;
- cur = build_hc_pre(inpL, attn_pre, il);
+ cur = build_hc_pre(inpL, attn_mix.pre, il);
cb(cur, "hc_ffn_pre", il);
ggml_build_forward_expand(gf, residual);
- ggml_build_forward_expand(gf, post);
- ggml_build_forward_expand(gf, comb);
+ ggml_build_forward_expand(gf, ffn_mix.post);
+ ggml_build_forward_expand(gf, ffn_mix.comb);
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "ffn_norm", il);
@@ -912,7 +910,7 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g
cur = ggml_add(ctx0, moe_out, ffn_shexp);
cb(cur, "ffn_out", il);
- inpL = build_hc_post(cur, residual, post, comb, il);
+ inpL = build_hc_post(cur, residual, ffn_mix.post, ffn_mix.comb, il);
inpL = build_cvec(inpL, il);
cb(inpL, "l_out", il);
}
diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp
index 6bbb534f0..e3c2f1f29 100644
--- a/src/models/dflash.cpp
+++ b/src/models/dflash.cpp
@@ -1204,14 +1204,12 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
const auto & layer = model.layers[il];
ggml_tensor * residual = inpL;
- ggml_tensor * post = nullptr;
- ggml_tensor * comb = nullptr;
-
- ggml_tensor * cur = build_hc_pre(inpL,
+ hc_pre attn_hc = build_hc_pre(inpL,
layer.hc_attn_fn,
layer.hc_attn_scale,
layer.hc_attn_base,
- &post, &comb, il);
+ il);
+ ggml_tensor * cur = attn_hc.cur;
cb(cur, "hc_attn_pre", il);
cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
@@ -1219,15 +1217,16 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
cur = build_attention(model, inp_attn, cur, inp_pos, il);
- inpL = build_hc_post(cur, residual, post, comb, il);
+ inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il);
cb(inpL, "hc_attn_post", il);
residual = inpL;
- cur = build_hc_pre(inpL,
+ hc_pre ffn_hc = build_hc_pre(inpL,
layer.hc_ffn_fn,
layer.hc_ffn_scale,
layer.hc_ffn_base,
- &post, &comb, il);
+ il);
+ cur = ffn_hc.cur;
cb(cur, "hc_ffn_pre", il);
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
@@ -1256,7 +1255,7 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
cur = ggml_add(ctx0, moe_out, ffn_shexp);
cb(cur, "ffn_out", il);
- inpL = build_hc_post(cur, residual, post, comb, il);
+ inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il);
cb(inpL, "l_out", il);
}
@@ -1403,10 +1402,7 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll
const auto & layer = model.layers[il];
ggml_tensor * residual = inpL;
- ggml_tensor * post = nullptr;
- ggml_tensor * comb = nullptr;
-
- ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il);
+ const hc_mixes attn_mix = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il);
ggml_tensor * cur;
if (pre_prev == nullptr) {
@@ -1461,13 +1457,14 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll
cb(cur, "attn_out", il);
}
- inpL = build_hc_post(cur, residual, post, comb, il);
+ inpL = build_hc_post(cur, residual, attn_mix.post, attn_mix.comb, il);
cb(inpL, "hc_attn_post", il);
residual = inpL;
- ggml_tensor * ffn_pre = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il);
+ const hc_mixes ffn_mix = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il);
+ pre_prev = ffn_mix.pre;
- cur = build_hc_pre(inpL, attn_pre, il);
+ cur = build_hc_pre(inpL, attn_mix.pre, il);
cb(cur, "hc_ffn_pre", il);
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
@@ -1496,13 +1493,11 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll
cur = ggml_add(ctx0, moe_out, ffn_shexp);
cb(cur, "ffn_out", il);
- inpL = build_hc_post(cur, residual, post, comb, il);
+ inpL = build_hc_post(cur, residual, ffn_mix.post, ffn_mix.comb, il);
ggml_tensor * l_out = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens);
cb(l_out, "l_out", il);
ggml_build_forward_expand(gf, l_out);
-
- pre_prev = ffn_pre;
}
// no hc_head: collapse with the last FFN's pre-mix
diff --git a/src/models/glm5-next.cpp b/src/models/glm5-next.cpp
index 395560990..d1e9b503d 100644
--- a/src/models/glm5-next.cpp
+++ b/src/models/glm5-next.cpp
@@ -442,13 +442,12 @@ ggml_tensor * llama_model_glm5_next::graph::build_hc_sinkhorn(
return comb;
}
-ggml_tensor * llama_model_glm5_next::graph::build_hc_pre(
+llama_model_glm5_next::graph::hc_pre
+llama_model_glm5_next::graph::build_hc_pre(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
- ggml_tensor ** post,
- ggml_tensor ** comb,
int il) const {
const int64_t hc = hparams.dsv4_hc_mult;
const int64_t hc_dim = hc*n_embd;
@@ -475,29 +474,29 @@ ggml_tensor * llama_model_glm5_next::graph::build_hc_pre(
pre = ggml_scale_bias(ctx0, pre, 1.0f, hparams.dsv4_hc_eps);
cb(pre, "hc_pre", il);
- *post = dsv4_view_2d(ctx0, mixes, hc, nt, hc);
- *post = dsv4_hc_affine(ctx0, *post, scale_post, base_post);
- *post = ggml_sigmoid(ctx0, *post);
- *post = ggml_scale(ctx0, *post, 2.0f);
- cb(*post, "hc_post", il);
+ ggml_tensor * post = dsv4_view_2d(ctx0, mixes, hc, nt, hc);
+ post = dsv4_hc_affine(ctx0, post, scale_post, base_post);
+ post = ggml_sigmoid(ctx0, post);
+ post = ggml_scale(ctx0, post, 2.0f);
+ cb(post, "hc_post", il);
+ ggml_tensor * comb = nullptr;
if (cparams.fused_dsv4_hc_comb) {
- *comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps,
+ comb = ggml_dsv4_hc_comb(ctx0, mixes, hc_scale, hc_base, hparams.dsv4_hc_eps,
(int32_t) hparams.dsv4_hc_sinkhorn_iters);
- res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, *comb, il});
+ res->add_fused_node({LLM_FUSED_OP_DSV4_HC_COMB, comb, il});
} else {
ggml_tensor * scale_comb = dsv4_view_1d(ctx0, hc_scale, 1, 2);
ggml_tensor * base_comb = dsv4_view_1d(ctx0, hc_base, hc*hc, 2*hc);
- *comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc);
- *comb = dsv4_hc_affine(ctx0, *comb, scale_comb, base_comb);
- *comb = ggml_reshape_3d(ctx0, *comb, hc, hc, nt);
- *comb = build_hc_sinkhorn(*comb, il);
+ comb = dsv4_view_2d(ctx0, mixes, hc*hc, nt, 2*hc);
+ comb = dsv4_hc_affine(ctx0, comb, scale_comb, base_comb);
+ comb = ggml_reshape_3d(ctx0, comb, hc, hc, nt);
+ comb = build_hc_sinkhorn(comb, il);
}
- cb(*comb, "hc_comb", il);
+ cb(comb, "hc_comb", il);
- ggml_tensor * result = build_hc_pre(x, pre, il);
- return result;
+ return { build_hc_pre(x, pre, il), post, comb };
}
ggml_tensor * llama_model_glm5_next::graph::build_hc_post(
@@ -609,8 +608,9 @@ llama_model_glm5_next::graph_mtp::graph_mtp(const llama_model & model, const llm
cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "mtp_attn_norm", il);
- ggml_tensor * prev_sel = nullptr;
- cur = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, &prev_sel, il);
+ // the NextN block has a full indexer, nothing to carry in
+ dsa_out dsa = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, nullptr, il);
+ cur = dsa.cur;
cb(cur, "mtp_attn_out", il);
// narrow to the output tokens before the position-wise FFN; unmasked nextn embeddings need all rows
@@ -711,10 +711,8 @@ llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_p
const auto & layer = model.layers[il];
ggml_tensor * residual = inpL;
- ggml_tensor * post = nullptr;
- ggml_tensor * comb = nullptr;
-
- cur = build_hc_pre(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, il);
+ hc_pre attn_hc = build_hc_pre(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, il);
+ cur = attn_hc.cur;
cb(cur, "hc_attn_pre", il);
cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
@@ -725,14 +723,17 @@ llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_p
cur = build_kda_layer(cur, layer, inp_rs, d_conv, head_dim, n_head_kda,
d_inner, n_seq_tokens, n_seqs, il);
} else {
- cur = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, &prev_sel, il);
+ dsa_out dsa = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, prev_sel, il);
+ cur = dsa.cur;
+ prev_sel = dsa.sel;
}
- inpL = build_hc_post(cur, residual, post, comb, il);
+ inpL = build_hc_post(cur, residual, attn_hc.post, attn_hc.comb, il);
cb(inpL, "hc_attn_post", il);
residual = inpL;
- cur = build_hc_pre(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, il);
+ hc_pre ffn_hc = build_hc_pre(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, il);
+ cur = ffn_hc.cur;
cb(cur, "hc_ffn_pre", il);
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
@@ -770,7 +771,7 @@ llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_p
cb(cur, "ffn_out", il);
}
- inpL = build_hc_post(cur, residual, post, comb, il);
+ inpL = build_hc_post(cur, residual, ffn_hc.post, ffn_hc.comb, il);
inpL = build_cvec(inpL, il);
cb(inpL, "l_out", il);
}
@@ -1039,10 +1040,11 @@ ggml_tensor * llama_model_glm5_next::graph::build_kpool_select(
// Nope MLA layer with sparse attention over the indexer selection
-ggml_tensor * llama_model_glm5_next::graph::build_dsa_layer(
+llama_model_glm5_next::graph::dsa_out
+llama_model_glm5_next::graph::build_dsa_layer(
ggml_tensor * cur, const llama_layer & layer,
const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_attn_k * inp_attn,
- llm_graph_input_kpool * inp_kpool, ggml_tensor ** prev_sel, int il) {
+ llm_graph_input_kpool * inp_kpool, ggml_tensor * prev_sel, int il) {
const auto * mctx_mla = mctx_hyb->get_attn();
@@ -1073,13 +1075,11 @@ ggml_tensor * llama_model_glm5_next::graph::build_dsa_layer(
ggml_tensor * kq_mask = inp_attn->get_kq_mask();
- ggml_tensor * sel = nullptr;
+ ggml_tensor * sel = prev_sel;
if (il >= (int) hparams.n_layer() || hparams.is_indexer_full(il)) { // the NextN block always has a full indexer
sel = build_kpool_select(cur, qr, kq_mask, layer, mctx_hyb, inp_kpool, il);
- *prev_sel = sel;
} else {
- GGML_ASSERT(*prev_sel != nullptr && "shared indexer layer must follow a full indexer layer");
- sel = *prev_sel;
+ GGML_ASSERT(sel != nullptr && "shared indexer layer must follow a full indexer layer");
}
ggml_build_forward_expand(gf, q_absorbed);
@@ -1099,5 +1099,5 @@ ggml_tensor * llama_model_glm5_next::graph::build_dsa_layer(
out = ggml_mul_mat(ctx0, layer.wo, out);
cb(out, "attn_out", il);
- return out;
+ return { out, sel };
}
diff --git a/src/models/hy-v4.cpp b/src/models/hy-v4.cpp
index 25e40abf7..3d456fe85 100644
--- a/src/models/hy-v4.cpp
+++ b/src/models/hy-v4.cpp
@@ -172,12 +172,12 @@ static ggml_tensor * hy_v4_hc_reduce(ggml_context * ctx0, ggml_tensor * x, ggml_
return ggml_cast(ctx0, result, out_type);
}
-ggml_tensor * llama_model_hy_v4::graph::build_hc_pre(
+llama_model_hy_v4::graph::hc_pre
+llama_model_hy_v4::graph::build_hc_pre(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
- ggml_tensor ** post,
int il) const {
const int64_t hc = hparams.dsv4_hc_mult;
const int64_t nt = x->ne[2];
@@ -202,16 +202,15 @@ ggml_tensor * llama_model_hy_v4::graph::build_hc_pre(
cb(pre, "hc_pre", il);
// post = magnitude*sigmoid(mixes[hc:2hc]*scale_post + base_post) + eps
- ggml_tensor * po = hy_v4_view_2d(ctx0, mixes, hc, nt, hc);
- po = ggml_mul(ctx0, po, scale_post);
- po = ggml_add(ctx0, po, base_post);
- po = ggml_sigmoid(ctx0, po);
- po = ggml_scale(ctx0, po, hparams.hc_magnitude);
- po = ggml_scale_bias(ctx0, po, 1.0f, hparams.dsv4_hc_eps);
- *post = po;
- cb(po, "hc_post_gate", il);
-
- return hy_v4_hc_reduce(ctx0, x, pre, hc, n_embd, nt, x->type);
+ ggml_tensor * post = hy_v4_view_2d(ctx0, mixes, hc, nt, hc);
+ post = ggml_mul(ctx0, post, scale_post);
+ post = ggml_add(ctx0, post, base_post);
+ post = ggml_sigmoid(ctx0, post);
+ post = ggml_scale(ctx0, post, hparams.hc_magnitude);
+ post = ggml_scale_bias(ctx0, post, 1.0f, hparams.dsv4_hc_eps);
+ cb(post, "hc_post_gate", il);
+
+ return { hy_v4_hc_reduce(ctx0, x, pre, hc, n_embd, nt, x->type), post };
}
ggml_tensor * llama_model_hy_v4::graph::build_hc_post(
@@ -418,12 +417,13 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k(
return ggml_cont(ctx0, ggml_top_k(ctx0, score, n_top_k));
}
-ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa(
+llama_model_hy_v4::graph::attn_dsa
+llama_model_hy_v4::graph::build_attention_dsa(
const llama_model & model,
llm_graph_input_attn_k_dsa * inp_attn_dsa,
ggml_tensor * cur,
ggml_tensor * inp_pos,
- ggml_tensor ** last_top_k,
+ ggml_tensor * last_top_k,
float kq_scale,
int il) const {
const auto & layer = model.layers[il];
@@ -436,11 +436,12 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa(
ggml_tensor * qr = ggml_mul_mat(ctx0, layer.wq_a, cur);
qr = build_norm(qr, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il);
+ ggml_tensor * top_k = last_top_k;
if (hparams.is_indexer_full(il)) {
- *last_top_k = build_indexer_top_k(model, inp_attn_dsa, cur, qr, inp_pos, il);
- cb(*last_top_k, "top_k", il);
+ top_k = build_indexer_top_k(model, inp_attn_dsa, cur, qr, inp_pos, il);
+ cb(top_k, "top_k", il);
}
- GGML_ASSERT(*last_top_k != nullptr);
+ GGML_ASSERT(top_k != nullptr);
ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq_b, qr);
@@ -480,7 +481,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa(
ggml_tensor * attn = build_attn(inp_attn_dsa,
nullptr, nullptr, nullptr,
- Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, layer.wv_b, *last_top_k, kq_scale, il);
+ Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, layer.wv_b, top_k, kq_scale, il);
cb(attn, "attn_kqv", il);
ggml_tensor * gate = ggml_mul_mat(ctx0, layer.wqkv_gate, cur);
@@ -491,7 +492,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention_dsa(
ggml_tensor * out = build_lora_mm(layer.wo, attn);
cb(out, "attn_out", il);
- return out;
+ return { out, top_k };
}
llama_model_hy_v4::graph::graph(const llama_model & model, const llm_graph_params & params) :
@@ -520,23 +521,27 @@ llama_model_hy_v4::graph::graph(const llama_model & model, const llm_graph_param
for (int il = 0; il < n_layer; ++il) {
ggml_tensor * residual = inpL;
- ggml_tensor * post = nullptr;
-
- cur = build_hc_pre(inpL, model.layers[il].hc_attn_fn, model.layers[il].hc_attn_scale,
- model.layers[il].hc_attn_base, &post, il);
+ hc_pre attn_hc = build_hc_pre(inpL, model.layers[il].hc_attn_fn, model.layers[il].hc_attn_scale,
+ model.layers[il].hc_attn_base, il);
+ cur = attn_hc.cur;
cur = build_norm(cur, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "attn_norm", il);
- cur = is_dsa
- ? build_attention_dsa(model, inp_attn_dsa, cur, inp_pos, &last_top_k, kq_scale, il)
- : build_attention(model, inp_attn, cur, inp_pos, kq_scale, il);
+ if (is_dsa) {
+ attn_dsa attn = build_attention_dsa(model, inp_attn_dsa, cur, inp_pos, last_top_k, kq_scale, il);
+ cur = attn.cur;
+ last_top_k = attn.top_k;
+ } else {
+ cur = build_attention(model, inp_attn, cur, inp_pos, kq_scale, il);
+ }
- inpL = build_hc_post(cur, residual, post, il);
+ inpL = build_hc_post(cur, residual, attn_hc.post, il);
cb(inpL, "hc_attn_out", il);
residual = inpL;
- cur = build_hc_pre(inpL, model.layers[il].hc_ffn_fn, model.layers[il].hc_ffn_scale,
- model.layers[il].hc_ffn_base, &post, il);
+ hc_pre ffn_hc = build_hc_pre(inpL, model.layers[il].hc_ffn_fn, model.layers[il].hc_ffn_scale,
+ model.layers[il].hc_ffn_base, il);
+ cur = ffn_hc.cur;
cur = build_norm(cur, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "ffn_norm", il);
@@ -575,7 +580,7 @@ llama_model_hy_v4::graph::graph(const llama_model & model, const llm_graph_param
cb(cur, "ffn_out", il);
}
- inpL = build_hc_post(cur, residual, post, il);
+ inpL = build_hc_post(cur, residual, ffn_hc.post, il);
cb(inpL, "l_out", il);
}
diff --git a/src/models/models.h b/src/models/models.h
index 37dbacdb9..5d9b6077a 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -1358,24 +1358,33 @@ struct llama_model_deepseek4 : public llama_model_base {
// mean over the hyper-connection streams: [n_embd, hc, n_tokens] -> [n_embd, n_tokens]
ggml_tensor * build_hc_mean(ggml_tensor * x) const;
- // returns the collapsed input and fills the post / comb weights
- ggml_tensor * build_hc_pre(
+ // all outputs of the hc mix builders:
+ // build_hc_pre - cur is the collapsed input, post / comb feed build_hc_post
+ // build_hc_mixes - pre is the pre weights, post / comb feed build_hc_post
+ struct hc_pre {
+ ggml_tensor * cur;
+ ggml_tensor * post;
+ ggml_tensor * comb;
+ };
+
+ struct hc_mixes {
+ ggml_tensor * pre;
+ ggml_tensor * post;
+ ggml_tensor * comb;
+ };
+
+ hc_pre build_hc_pre(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
- ggml_tensor ** post,
- ggml_tensor ** comb,
int il) const;
- // returns the pre weights and fills post / comb
- ggml_tensor * build_hc_mixes(
+ hc_mixes build_hc_mixes(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
- ggml_tensor ** post,
- ggml_tensor ** comb,
int il) const;
ggml_tensor * build_hc_post(
@@ -2263,12 +2272,16 @@ struct llama_model_hy_v4 : public llama_model_base {
// iHC (independent Hyper-Connections): pre reduces the hc streams to one and returns the
// per-stream post gates, post writes the sublayer output back into the streams, head
// collapses the streams before the final norm.
- ggml_tensor * build_hc_pre(
+ struct hc_pre {
+ ggml_tensor * cur; // the collapsed stream
+ ggml_tensor * post; // per-stream post gates, feeds build_hc_post
+ };
+
+ hc_pre build_hc_pre(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
- ggml_tensor ** post,
int il) const;
ggml_tensor * build_hc_post(
@@ -2301,12 +2314,18 @@ struct llama_model_hy_v4 : public llama_model_base {
ggml_tensor * inp_pos,
int il) const;
- ggml_tensor * build_attention_dsa(
+ // the attention output and the top-k to carry to the following "shared" layers
+ struct attn_dsa {
+ ggml_tensor * cur;
+ ggml_tensor * top_k;
+ };
+
+ attn_dsa build_attention_dsa(
const llama_model & model,
llm_graph_input_attn_k_dsa * inp_attn_dsa,
ggml_tensor * cur,
ggml_tensor * inp_pos,
- ggml_tensor ** last_top_k,
+ ggml_tensor * last_top_k,
float kq_scale,
int il) const;
};
@@ -2730,14 +2749,18 @@ struct llama_model_qwen4exp : public llama_model_base {
llm_build_delta_net_base(params), model(model) {}
// HC replaces every layer norm: residual is [n_embd, hc, n_tokens]
- ggml_tensor * build_hc_mix(
+ struct hc_mix {
+ ggml_tensor * cur; // the mixed [n_embd, T] stream
+ ggml_tensor * inject; // [hc, T] scatter weights, null when the inject weights are null
+ };
+
+ hc_mix build_hc_mix(
ggml_tensor * x,
ggml_tensor * w_norm,
ggml_tensor * w_down,
ggml_tensor * w_up,
ggml_tensor * w_inject,
ggml_tensor * w_down_inject, // [w_down | w_inject] in one weight, can be null
- ggml_tensor ** inject,
int il);
ggml_tensor * build_hc_combine(
@@ -2969,14 +2992,18 @@ struct llama_model_glm5_next : public llama_model_base {
// mean over the hyper-connection streams: [n_embd, hc, n_tokens] -> [n_embd, n_tokens]
ggml_tensor * build_hc_mean(ggml_tensor * x) const;
- // returns the collapsed input and fills the post / comb weights
- ggml_tensor * build_hc_pre(
+ // all outputs of the hc mix builder: cur is the collapsed input, post / comb feed build_hc_post
+ struct hc_pre {
+ ggml_tensor * cur;
+ ggml_tensor * post;
+ ggml_tensor * comb;
+ };
+
+ hc_pre build_hc_pre(
ggml_tensor * x,
ggml_tensor * hc_fn,
ggml_tensor * hc_scale,
ggml_tensor * hc_base,
- ggml_tensor ** post,
- ggml_tensor ** comb,
int il) const;
ggml_tensor * build_hc_post(
@@ -3002,9 +3029,15 @@ struct llama_model_glm5_next : public llama_model_base {
ggml_tensor * build_kpool_select(ggml_tensor * cur, ggml_tensor * qr, ggml_tensor * kq_mask, const llama_layer & layer,
const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_kpool * inp_kpool, int il);
- ggml_tensor * build_dsa_layer(ggml_tensor * cur, const llama_layer & layer,
- const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_attn_k * inp_attn,
- llm_graph_input_kpool * inp_kpool, ggml_tensor ** prev_sel, int il);
+ // attention output (cur) and the selection to carry to the next shared layer (sel)
+ struct dsa_out {
+ ggml_tensor * cur;
+ ggml_tensor * sel;
+ };
+
+ dsa_out build_dsa_layer(ggml_tensor * cur, const llama_layer & layer,
+ const llama_memory_hybrid_idx_context * mctx_hyb, llm_graph_input_attn_k * inp_attn,
+ llm_graph_input_kpool * inp_kpool, ggml_tensor * prev_sel, int il);
};
diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp
index 1f3b1cdef..f5a81f8f6 100644
--- a/src/models/qwen4exp.cpp
+++ b/src/models/qwen4exp.cpp
@@ -343,15 +343,15 @@ static ggml_tensor * hc_stream_mean(ggml_context * ctx, ggml_tensor * x) {
}
// Hyper-connections keep hc parallel residual streams [n_embd, hc, T] in place of layer norms.
-// Returns the mixed [n_embd, T] stream; `inject` gets the [hc, T] scatter weights.
-ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix(
+// build_hc_mix returns the mixed [n_embd, T] stream and the [hc, T] scatter weights.
+llama_model_qwen4exp::graph::hc_mix
+llama_model_qwen4exp::graph::build_hc_mix(
ggml_tensor * x,
ggml_tensor * w_norm,
ggml_tensor * w_down,
ggml_tensor * w_up,
ggml_tensor * w_inject,
ggml_tensor * w_down_inject,
- ggml_tensor ** inject,
int il) {
const int64_t hc = hparams.dsv4_hc_mult;
const int64_t hc_dim = hc * n_embd;
@@ -364,15 +364,16 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix(
cb(xn, "hc_norm", il);
ggml_tensor * lo = nullptr;
- if (inject && w_down_inject) {
+ ggml_tensor * inject = nullptr;
+ if (w_down_inject) {
// one matmul for both: the hc_down rows, then the hc_inject rows
const int64_t n_lo = w_down_inject->ne[1] - hc;
ggml_tensor * lo_inject = build_lora_mm(w_down_inject, xn);
// scale the combined rows before the split; both lo and inject use 1/hc
lo_inject = ggml_scale(ctx0, lo_inject, 1.0f / (float) hc);
- lo = ggml_view_2d(ctx0, lo_inject, n_lo, nt, lo_inject->nb[1], 0);
- *inject = ggml_view_2d(ctx0, lo_inject, hc, nt, lo_inject->nb[1], ggml_row_size(lo_inject->type, n_lo));
- cb(*inject, "hc_inject", il);
+ lo = ggml_view_2d(ctx0, lo_inject, n_lo, nt, lo_inject->nb[1], 0);
+ inject = ggml_view_2d(ctx0, lo_inject, hc, nt, lo_inject->nb[1], ggml_row_size(lo_inject->type, n_lo));
+ cb(inject, "hc_inject", il);
} else {
lo = ggml_scale(ctx0, build_lora_mm(w_down, xn), 1.0f / (float) hc);
}
@@ -405,12 +406,12 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix(
}
cb(mixed, "hc_mixed", il);
- if (inject && !w_down_inject) {
- *inject = ggml_scale(ctx0, build_lora_mm(w_inject, xn), 1.0f / (float) hc);
- cb(*inject, "hc_inject", il);
+ if (!w_down_inject && w_inject) {
+ inject = ggml_scale(ctx0, build_lora_mm(w_inject, xn), 1.0f / (float) hc);
+ cb(inject, "hc_inject", il);
}
- return mixed;
+ return { mixed, inject };
}
ggml_tensor * llama_model_qwen4exp::graph::build_hc_combine(
@@ -504,14 +505,14 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa
res_hc = build_ple(inp->get_recr(), ple_emb, res_hc, il);
}
- ggml_tensor * inject = nullptr;
- ggml_tensor * cur = build_hc_mix(res_hc,
+ hc_mix attn_mix = build_hc_mix(res_hc,
model.layers[il].hc_attn_norm,
model.layers[il].hc_attn_down,
model.layers[il].hc_attn_up,
model.layers[il].hc_attn_inject,
model.layers[il].hc_attn_down_inject,
- &inject, il);
+ il);
+ ggml_tensor * cur = attn_mix.cur;
ggml_build_forward_expand(gf, cur);
@@ -523,28 +524,29 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa
if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {
// everything below is per token, so drop the rows that produce no output
- cur = ggml_get_rows(ctx0, cur, inp_out_ids);
- inject = ggml_get_rows(ctx0, inject, inp_out_ids);
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ attn_mix.inject = ggml_get_rows(ctx0, attn_mix.inject, inp_out_ids);
res_hc = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, res_hc->ne[2]);
res_hc = ggml_get_rows(ctx0, res_hc, inp_out_ids);
res_hc = ggml_reshape_3d(ctx0, res_hc, n_embd, hc, res_hc->ne[1]);
}
- res_hc = build_hc_combine(res_hc, cur, inject, il);
+ res_hc = build_hc_combine(res_hc, cur, attn_mix.inject, il);
- cur = build_hc_mix(res_hc,
+ hc_mix ffn_mix = build_hc_mix(res_hc,
model.layers[il].hc_ffn_norm,
model.layers[il].hc_ffn_down,
model.layers[il].hc_ffn_up,
model.layers[il].hc_ffn_inject,
model.layers[il].hc_ffn_down_inject,
- &inject, il);
+ il);
+ cur = ffn_mix.cur;
cur = build_layer_ffn(cur, il);
cb(cur, "ffn_out", il);
- res_hc = build_hc_combine(res_hc, cur, inject, il);
+ res_hc = build_hc_combine(res_hc, cur, ffn_mix.inject, il);
// "l_last" is the layer output name that build_cvec and imatrix look for
cb(res_hc, "l_last", il);
@@ -562,9 +564,10 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa
}
// the final mixer is the output norm: there is no separate one
- ggml_tensor * cur = build_hc_mix(res_hc,
+ hc_mix head_mix = build_hc_mix(res_hc,
model.hc_head_norm, model.hc_head_down, model.hc_head_up,
- nullptr, nullptr, nullptr, -1);
+ nullptr, nullptr, -1);
+ ggml_tensor * cur = head_mix.cur;
cb(cur, "result_norm", -1);
res->t_embd = cur;
@@ -641,14 +644,15 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_
ggml_tensor * res_hc = build_lora_mm(layer.nextn.eh_proj, ggml_concat(ctx0, e_norm, h_norm, 0)); // [n_embd, hc, n_tokens]
cb(res_hc, "mtp_eh_proj", il);
- ggml_tensor * inject = nullptr;
- ggml_tensor * cur = build_hc_mix(res_hc, layer.hc_attn_norm, layer.hc_attn_down, layer.hc_attn_up, layer.hc_attn_inject, layer.hc_attn_down_inject, &inject, il);
+ hc_mix attn_mix = build_hc_mix(res_hc, layer.hc_attn_norm, layer.hc_attn_down, layer.hc_attn_up, layer.hc_attn_inject, layer.hc_attn_down_inject, il);
+ ggml_tensor * cur = attn_mix.cur;
cur = build_layer_attn(inp_hyb->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, sections, il);
- res_hc = build_hc_combine(res_hc, cur, inject, il);
+ res_hc = build_hc_combine(res_hc, cur, attn_mix.inject, il);
- cur = build_hc_mix(res_hc, layer.hc_ffn_norm, layer.hc_ffn_down, layer.hc_ffn_up, layer.hc_ffn_inject, layer.hc_ffn_down_inject, &inject, il);
+ hc_mix ffn_mix = build_hc_mix(res_hc, layer.hc_ffn_norm, layer.hc_ffn_down, layer.hc_ffn_up, layer.hc_ffn_inject, layer.hc_ffn_down_inject, il);
+ cur = ffn_mix.cur;
cur = build_layer_ffn(cur, il);
- res_hc = build_hc_combine(res_hc, cur, inject, il);
+ res_hc = build_hc_combine(res_hc, cur, ffn_mix.inject, il);
// the next draft step reads this residual as its h
ggml_tensor * flat = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, n_tokens);
@@ -657,9 +661,10 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_
cb(res->t_h_nextn, "h_nextn", il);
ggml_build_forward_expand(gf, res->t_h_nextn);
- cur = build_hc_mix(ggml_reshape_3d(ctx0, flat_out, n_embd, hc, flat_out->ne[1]),
+ hc_mix head_mix = build_hc_mix(ggml_reshape_3d(ctx0, flat_out, n_embd, hc, flat_out->ne[1]),
layer.nextn.hc_head_norm, layer.nextn.hc_head_down, layer.nextn.hc_head_up,
- nullptr, nullptr, nullptr, il);
+ nullptr, nullptr, il);
+ cur = head_mix.cur;
cb(cur, "result_norm", -1);
res->t_embd = cur;