Commit 79bce0c88 for llama.cpp
commit 79bce0c8812ddfe4716922d061b67edc2ed517e9
Author: Niklas Wenzel <dev@nikwen.de>
Date: Sat Oct 31 23:55:02 2026 +0100
tests : enable Gemma 4 in `test-llama-archs` (#31401)
* tests : enable Gemma 4 in `test-llama-archs`
* tests : use GQA for Gemma 4 in `test-llama-archs`
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index a08e173e9..36f3850d5 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -199,6 +199,9 @@ void llama_model_saver::add_kv_from_model() {
if (hparams.n_embd_out_impl > 0) {
add_kv(LLM_KV_EMBEDDING_LENGTH_OUT, hparams.n_embd_out_impl);
}
+ if (hparams.n_embd_per_layer > 0) {
+ add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.n_embd_per_layer);
+ }
add_kv(LLM_KV_BLOCK_COUNT, hparams.n_layer_all);
add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);
add_kv(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, true);
@@ -307,7 +310,9 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_ATTENTION_INDEXER_KPOOL_SELECT_TAIL, hparams.indexer_kpool_select_tail);
add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, hparams.indexer_local_blocks);
add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, true);
- if (hparams.n_layer_kv_from_start < 0) {
+ if (hparams.n_layer_kv_from_start >= 0) {
+ add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS, uint32_t(hparams.n_layer_all - hparams.n_layer_kv_from_start));
+ } else {
for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
if (hparams.kv_source(il) != il) {
add_kv(LLM_KV_ATTENTION_KV_SOURCE_MAP, hparams.kv_source_impl, true);
@@ -520,6 +525,8 @@ void llama_model_saver::add_tensors_from_model() {
add_tensor(model->hc_head_base);
add_tensor(model->hc_head_scale);
add_tensor(model->per_layer_tok_embd);
+ add_tensor(model->per_layer_model_proj);
+ add_tensor(model->per_layer_proj_norm);
add_tensor(model->value_embd);
add_tensor(model->hc_head_norm);
add_tensor(model->hc_head_down);
diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv
index e08d0c81b..569947a7f 100644
--- a/tests/fusion/MTL.csv
+++ b/tests/fusion/MTL.csv
@@ -81,6 +81,8 @@ gemma2 ,0 ,any ,RMS_NORM+MUL , 5
gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4
gemma3 ,0 ,any ,RMS_NORM+MUL , 9
gemma3 ,0 ,any ,RMS_NORM+MUL+ADD , 4
+gemma4 ,0 ,any ,RMS_NORM+MUL , 47
+gemma4 ,0 ,any ,RMS_NORM+MUL+ADD , 20
glm-dsa ,0 ,any ,ADD+ADD , 1
glm-dsa ,0 ,any ,MUL+ADD , 2
glm-dsa ,0 ,any ,NORM+MUL+ADD , 2
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index d82d6db79..312733ecd 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -243,10 +243,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
n_head = 4;
n_layer = 4; // dense and MoE layers, each with local and full attention
} else if (arch == LLM_ARCH_GEMMA4) {
- n_embd = 128;
- n_head = 2;
- n_ff = 192;
- n_layer = 5; // need at least 5 for swa_pattern (every 5th is full_attention)
+ // TODO: expand coverage to both GQA and MQA (see https://github.com/ggml-org/llama.cpp/pull/31401#pullrequestreview-5710165130)
+ n_head = 4;
+ n_layer = 10; // last 5 layers reuse KV, both SWA and full attention
} else if (arch == LLM_ARCH_GEMMA3N) {
n_embd = 64;
n_head = 1;
@@ -305,7 +304,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
if (arch == LLM_ARCH_QWEN3) {
n_head_kv = 1; // MQA coverage
} else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER ||
- arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE) {
+ arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE || arch == LLM_ARCH_GEMMA4) {
n_head_kv = 2; // GQA coverage
}
if (arch == LLM_ARCH_BEAM) {
@@ -433,12 +432,17 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);
} else if (arch == LLM_ARCH_GEMMA4) {
ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, n_embd/2);
- ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS, uint32_t(0));
+ ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS, uint32_t(5));
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, n_embd_head);
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, n_embd_head);
ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f);
// SWA pattern: every 5th layer is full attention (matches E2B layer_types)
- ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5));
+ std::vector<uint32_t> pattern;
+ pattern.reserve(n_layer);
+ for (uint32_t il = 0; il < n_layer; il++) {
+ pattern.push_back((il + 1) % 5 != 0);
+ }
+ ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);
} else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 ||
arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE ||
arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT) {
@@ -1001,9 +1005,6 @@ static bool arch_supported(const llm_arch arch) {
if (arch == LLM_ARCH_WAVTOKENIZER_DEC) {
return false; // FIXME CUDA backend crashes.
}
- if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
- return false; // FIXME @ngxson
- }
if (arch == LLM_ARCH_GRANITE_SWITCH) {
return false; // FIXME adapter fixture
}
@@ -1075,10 +1076,7 @@ static int save_models(const std::string & arch_filter, const size_t seed, const
if (!arch_matches(arch_filter, arch)) {
continue;
}
- if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
- continue; // FIXME: ISWA KV cache initialization needs more fixture params
- }
- if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH) {
+ if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
continue;
}
for (bool moe : {false, true}) {
@@ -1235,10 +1233,7 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con
if (!arch_matches(arch_filter, arch)) {
continue;
}
- if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
- continue; // FIXME: ISWA KV cache initialization needs more fixture params
- }
- if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH) {
+ if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
continue;
}