Commit 6023692b1 for llama.cpp
commit 6023692b1e13a6d4ee6b93f989c4713e474e6165
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Thu Nov 5 21:10:36 2026 +0100
llama: fix kv shifting, add tests, add per-model build_rope (#31683)
* llama: fix kv shifting and add tests
* llama: add graph build_rope (#31693)
* llama: add graph build_rope
* improve
* migrate new models
* update review skill
* fix ci
* add --kv-shift to CI
diff --git a/.github/workflows/models-check.yml b/.github/workflows/models-check.yml
index 66a150aff..2774cefd3 100644
--- a/.github/workflows/models-check.yml
+++ b/.github/workflows/models-check.yml
@@ -11,7 +11,8 @@ on:
'tests/fusion/**',
'tests/test-fusion.cpp',
'tests/test-llama-archs.cpp',
- 'src/llama-graph.cpp',
+ 'src/llama-graph*',
+ 'src/llama-kv-cache*',
'src/llama-model*',
'src/models/**'
]
@@ -24,7 +25,8 @@ on:
'tests/fusion/**',
'tests/test-fusion.cpp',
'tests/test-llama-archs.cpp',
- 'src/llama-graph.cpp',
+ 'src/llama-graph*',
+ 'src/llama-kv-cache*',
'src/llama-model*',
'src/models/**'
]
@@ -145,6 +147,11 @@ jobs:
rm -rf build-ci-models && mkdir -p build-ci-models
./build/bin/test-llama-archs -o build-ci-models
+ - name: Test KV shift
+ id: test_kv_shift
+ run: |
+ ./build/bin/test-llama-archs --kv-shift
+
- name: Test fusion
id: test_fusion
run: |
@@ -293,6 +300,11 @@ jobs:
# run: |
# ./build/bin/test-fusion --models build-ci-models --device Vulkan0 --check tests/fusion/Vulkan.csv
+ - name: Test KV shift
+ id: test_kv_shift
+ run: |
+ ./build/bin/test-llama-archs --kv-shift
+
- name: Test archs
id: test_archs
run: |
diff --git a/skills/code-review/SKILL.md b/skills/code-review/SKILL.md
index 4c8e4bbcc..d72b46712 100644
--- a/skills/code-review/SKILL.md
+++ b/skills/code-review/SKILL.md
@@ -85,6 +85,7 @@ See the `add-new-model` skill and `docs/development/HOWTO-add-model.md` for the
- Hparams that the model can't run correctly without must be mandatory (hard-error if missing), not read with a silent default fallback. Only genuinely-optional-across-configs values get a fallback accessor.
- New/optional weight tensors (scales, etc.) must route through `build_lora_mm` and the existing helpers, matching convention - don't leave raw matmuls copied from another arch.
- Don't hack RoPE with a custom sin/cos implementation. If `ggml_rope_ext` genuinely can't express it, that's an issue for discussion, not a PR.
+- In `src/models/`, apply RoPE with `build_rope(cur, pos, il)`, not a direct `ggml_rope_ext`/`ggml_rope_multi` call (see #31683). Per-layer differences go in a `get_rope_params` override: the KV-cache K-shift reads the same params, so a direct call lets the graph and the shift drift apart. The remaining direct calls (deepseek4/41, dflash, sensenova-u1) are not a pattern to copy.
- Test the quantized-KV path (`-ctk`/`-ctv q8_0`), not just default f16 - new speculative/attention features silently break there.
- Preserve existing explanatory comments about model-specific quirks when copying code; note the provenance ("copied from X, with Y added").
- Remove dead code/branches left over from adapting a reference implementation.
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index a1068aae3..ea20062a6 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -2910,7 +2910,6 @@ llm_graph_params llama_context::graph_params(
const llama_memory_context_i * mctx,
llm_graph_type gtype) const {
return {
- /*.arch =*/ model.arch,
/*.model =*/ &model,
/*.hparams =*/ model.hparams,
/*.cparams =*/ cparams,
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index 47007714a..b99370e2d 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -1546,7 +1546,6 @@ void llm_graph_result::set_params(const llm_graph_params & params) {
llm_graph_context::llm_graph_context(const llm_graph_params & params) :
model (*params.model),
- arch (params.arch),
hparams (params.hparams),
cparams (params.cparams),
ubatch (params.ubatch),
@@ -1640,6 +1639,37 @@ ggml_tensor * llm_graph_context::build_hadamard_input(
return x;
}
+static ggml_tensor * build_rope_impl(ggml_context * ctx, ggml_tensor * cur, ggml_tensor * pos, const llama_hparams & hparams, const llama_rope_params & rp) {
+ if (rp.n_rot == 0) {
+ return cur;
+ }
+
+ if (rp.type & GGML_ROPE_TYPE_MROPE) {
+ int sections[GGML_MROPE_SECTIONS];
+ std::copy(hparams.rope_sections.begin(), hparams.rope_sections.end(), sections);
+
+ return ggml_rope_multi(ctx, cur, pos, rp.factors,
+ rp.n_rot, sections, rp.type, rp.n_ctx_orig, rp.freq_base, rp.freq_scale,
+ rp.ext_factor, rp.attn_factor, rp.beta_fast, rp.beta_slow);
+ }
+
+ return ggml_rope_ext(ctx, cur, pos, rp.factors,
+ rp.n_rot, rp.type, rp.n_ctx_orig, rp.freq_base, rp.freq_scale,
+ rp.ext_factor, rp.attn_factor, rp.beta_fast, rp.beta_slow);
+}
+
+ggml_tensor * llm_graph_context::build_rope(ggml_tensor * cur, ggml_tensor * pos, int il) const {
+ return build_rope_impl(ctx0, cur, pos, hparams, model.get_rope_params(cparams, il));
+}
+
+ggml_tensor * llm_graph_context::build_rope_indexer(ggml_tensor * cur, ggml_tensor * pos, int il) const {
+ const llama_rope_params rp = model.get_rope_params_indexer(cparams, il);
+
+ ggml_tensor * res = build_rope_impl(ctx0, cur, pos, hparams, rp);
+
+ return rp.n_rot > 0 && rp.n_offs > 0 ? ggml_rope_set_offset(res, rp.n_offs) : res;
+}
+
ggml_tensor * llm_graph_context::build_lora_mm(
ggml_tensor * w,
ggml_tensor * cur,
@@ -1950,6 +1980,8 @@ ggml_tensor * llm_graph_context::build_ffn(
llm_ffn_op_type type_op,
llm_ffn_gate_type type_gate,
int il) const {
+ const llm_arch arch = model.arch;
+
// NVFP4 support is currently restricted to
// 1) LORA absence (*_s would be applied after LORA residual, which is incorrect)
// 2) bias absense (*_s would be applied after bias addition, which is incorrect)
@@ -2233,6 +2265,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
ggml_tensor * selected_experts_in,
ggml_tensor * post_exp_norm,
float post_exp_norm_eps) const {
+ const llm_arch arch = model.arch;
const int64_t n_embd = cur->ne[0];
const int64_t n_tokens = cur->ne[1];
const bool weight_before_ffn = arch == LLM_ARCH_LLAMA4; // for llama4, we apply the sigmoid-ed weights before the FFN
@@ -2658,7 +2691,7 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to
// mixed path (ubatch.is_mixed()): set_rows the token rows into a copy of the embd rows, with its own inputs as select branches must not share tensors
// TODO: use inp->tokens and inp->embd once ggml_build_forward_select allows it
- const bool has_mixed = llm_arch_supports_mixed_batch(arch) && cparams.ctx_type == LLAMA_CONTEXT_TYPE_DEFAULT;
+ const bool has_mixed = llm_arch_supports_mixed_batch(model.arch) && cparams.ctx_type == LLAMA_CONTEXT_TYPE_DEFAULT;
const int64_t n_tok_rows = has_mixed ? llm_graph_n_tok_rows(ubatch) : 0;
@@ -2884,7 +2917,7 @@ ggml_tensor * llm_graph_context::build_inp_mean() const {
}
ggml_tensor * llm_graph_context::build_inp_cls() const {
- auto inp = std::make_unique<llm_graph_input_cls>(cparams, arch);
+ auto inp = std::make_unique<llm_graph_input_cls>(cparams, model.arch);
auto & cur = inp->cls;
@@ -3055,7 +3088,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(
// while for some models F16 is enough, for others it is not, so we default to F32 here
ggml_prec_set_acc(kq, GGML_PREC_F32);
- if (arch == LLM_ARCH_GROK) {
+ if (model.arch == LLM_ARCH_GROK) {
// need to do the following:
// multiply by attn_output_multiplier
// and then :
@@ -3248,6 +3281,8 @@ ggml_tensor * llm_graph_context::build_attn(
ggml_tensor * v_mla, // TODO: remove
float kq_scale,
int il) const {
+ const llm_arch arch = model.arch;
+
GGML_ASSERT(v_mla == nullptr);
if (inp->self_k_rot[il]) {
@@ -3361,6 +3396,8 @@ ggml_tensor * llm_graph_context::build_attn(
ggml_tensor * v_mla,
float kq_scale,
int il) const {
+ const llm_arch arch = model.arch;
+
// these nodes are added to the graph together so that they are not reordered
// by doing so, the number of splits in the graph is reduced
// expand k later to enable rope fusion which directly writes into k-v cache
@@ -4185,6 +4222,8 @@ void llm_graph_context::build_pooling(
ggml_tensor * cls_out,
ggml_tensor * cls_out_b,
ggml_tensor * cls_norm) const {
+ const llm_arch arch = model.arch;
+
if (!cparams.embeddings) {
// a graph without logits still computes t_embd, so that it is never empty
if (res->t_logits == nullptr && res->t_embd != nullptr) {
diff --git a/src/llama-graph.h b/src/llama-graph.h
index d20343163..0f8ad8ac9 100644
--- a/src/llama-graph.h
+++ b/src/llama-graph.h
@@ -788,8 +788,6 @@ using llm_graph_cb = std::function<void(const llama_ubatch & ubatch, ggml_tensor
class llm_graph_result;
struct llm_graph_params {
- llm_arch arch = LLM_ARCH_UNKNOWN;
-
const llama_model * model = nullptr;
llama_hparams hparams;
@@ -909,7 +907,7 @@ struct llm_graph_params {
cparams.causal_attn == other.cparams.causal_attn &&
layer_inp_dev == other.layer_inp_dev &&
layer_inp_src == other.layer_inp_src &&
- arch == other.arch &&
+ model == other.model &&
gtype == other.gtype &&
cvec == other.cvec &&
loras == other.loras &&
@@ -1028,8 +1026,6 @@ struct llm_graph_amax_src1 {
struct llm_graph_context {
const llama_model & model;
- const llm_arch arch;
-
const llama_hparams & hparams;
const llama_cparams & cparams;
const llama_ubatch & ubatch;
@@ -1253,6 +1249,12 @@ struct llm_graph_context {
// tok_scale: applied to token rows only
ggml_tensor * build_inp_embd(ggml_tensor * tok_embd, float tok_scale = 1.0f) const;
ggml_tensor * build_inp_pos() const;
+
+ // applies the RoPE of layer il, cur is returned as is if the layer has none
+ ggml_tensor * build_rope(ggml_tensor * cur, ggml_tensor * pos, int il) const;
+
+ // same, for the lightning indexer
+ ggml_tensor * build_rope_indexer(ggml_tensor * cur, ggml_tensor * pos, int il) const;
ggml_tensor * build_inp_attn_scale() const;
ggml_tensor * build_inp_out_ids() const;
ggml_tensor * build_inp_mean() const;
diff --git a/src/llama-kv-cache-dsa.cpp b/src/llama-kv-cache-dsa.cpp
index 405eee36b..1221f7f2b 100644
--- a/src/llama-kv-cache-dsa.cpp
+++ b/src/llama-kv-cache-dsa.cpp
@@ -44,7 +44,6 @@ llama_kv_cache_dsa::llama_kv_cache_dsa(
// DSA lightning indexer uses MQA with single key head
std::fill(hparams_lid.n_head_kv_arr.begin(), hparams_lid.n_head_kv_arr.end(), 1);
hparams_lid.n_embd_head_k_full = model.hparams.indexer_head_size;
- hparams_lid.rope_type = LLAMA_ROPE_TYPE_NEOX;
// KV sharing applies to the MLA cache only, each indexer owns its keys
std::iota(hparams_lid.kv_source_impl.begin(), hparams_lid.kv_source_impl.end(), 0);
@@ -54,7 +53,7 @@ llama_kv_cache_dsa::llama_kv_cache_dsa(
kv_lid = std::make_unique<llama_kv_cache>(
model, no_alloc, hparams_lid, type_k, type_v,
v_trans, offload, unified, kv_size, n_seq_max,
- n_swa, swa_type, nullptr, filter_lid, nullptr, nullptr);
+ n_swa, swa_type, nullptr, filter_lid, nullptr, nullptr, "", true);
}
void llama_kv_cache_dsa::clear(bool data) {
diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp
index 06e218ba6..a1ec67e26 100644
--- a/src/llama-kv-cache.cpp
+++ b/src/llama-kv-cache.cpp
@@ -79,9 +79,10 @@ llama_kv_cache::llama_kv_cache(
const layer_filter_cb & filter,
const layer_reuse_cb & reuse,
const layer_share_cb & share,
- const char * name_tag) :
+ const char * name_tag,
+ bool is_indexer) :
model(model), hparams(hparams), v_trans(v_trans),
- n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_swa(n_swa),
+ n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_swa(n_swa), is_indexer(is_indexer),
attn_rot_bufs(no_alloc), swa_type(swa_type), ctxs_bufs(no_alloc),
other(static_cast<llama_kv_cache *>(mem_other)),
v_cells_impl(other ? other->v_cells_impl : std::make_shared<llama_kv_cells_vec>()),
@@ -2089,53 +2090,69 @@ size_t llama_kv_cache::size_v_bytes() const {
ggml_tensor * llama_kv_cache::build_rope_shift(
const llama_cparams & cparams,
ggml_context * ctx,
- ggml_tensor * cur,
+ ggml_tensor * k,
ggml_tensor * shift,
ggml_tensor * rot,
- ggml_tensor * factors,
- float freq_base,
- float freq_scale,
uint32_t il) const {
- const auto & n_ctx_orig = cparams.n_ctx_orig_yarn;
+ const llama_rope_params rp = is_indexer ? model.get_rope_params_indexer(cparams, il) : model.get_rope_params(cparams, il);
- const auto & yarn_ext_factor = cparams.yarn_ext_factor;
- const auto & yarn_beta_fast = cparams.yarn_beta_fast;
- const auto & yarn_beta_slow = cparams.yarn_beta_slow;
- const auto & yarn_attn_factor = cparams.yarn_attn_factor;
-
- const auto & n_rot = hparams.n_rot(il);
- const auto & rope_type = hparams.rope_type == LLAMA_ROPE_TYPE_MROPE || hparams.rope_type == LLAMA_ROPE_TYPE_IMROPE
+ const auto rope_type = rp.type == LLAMA_ROPE_TYPE_MROPE || rp.type == LLAMA_ROPE_TYPE_IMROPE
// @ngxson : this is a workaround
// for M-RoPE, we want to rotate the whole vector when doing KV shift
// a normal RoPE should work, we just need to use the correct ordering
// ref: https://github.com/ggml-org/llama.cpp/pull/13870
? LLAMA_ROPE_TYPE_NEOX
- : hparams.rope_type;
- ggml_tensor * tmp;
+ : rp.type;
- if (ggml_is_quantized(cur->type)) {
- // dequantize to f32 -> RoPE -> quantize back
- tmp = ggml_cast(ctx, cur, GGML_TYPE_F32);
+ const int64_t n_head_kv = hparams.n_head_kv(il);
+ const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);
+ const int64_t n_embd_head_k = hparams.n_embd_head_k(il);
+ const int64_t n_cells = ggml_nelements(k)/n_embd_k_gqa;
- // rotate back
- tmp = llama_mul_mat_hadamard(ctx, tmp, rot);
+ if (!ggml_is_quantized(k->type) && !rot && !rp.k_norm) {
+ // we rotate only the n_rot dimensions
+ ggml_tensor * k_rope =
+ ggml_view_3d(ctx, k,
+ rp.n_rot, n_head_kv, n_cells,
+ ggml_row_size(k->type, n_embd_head_k),
+ ggml_row_size(k->type, n_embd_k_gqa),
+ ggml_row_size(k->type, rp.n_offs));
- tmp = ggml_rope_ext(ctx, tmp,
- shift, factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- yarn_ext_factor, yarn_attn_factor, yarn_beta_fast, yarn_beta_slow);
+ return ggml_rope_ext_inplace(ctx, k_rope,
+ shift, rp.factors, rp.n_rot, rope_type, rp.n_ctx_orig, rp.freq_base, rp.freq_scale,
+ rp.ext_factor, rp.attn_factor, rp.beta_fast, rp.beta_slow);
+ }
- // rotate fwd
+ // convert to f32 -> RoPE -> convert back, on whole rows as a head is not always a multiple of the block size
+ ggml_tensor * tmp = ggml_reshape_3d(ctx, ggml_cast(ctx, k, GGML_TYPE_F32), n_embd_head_k, n_head_kv, n_cells);
+
+ // rotate back
+ if (rot) {
tmp = llama_mul_mat_hadamard(ctx, tmp, rot);
+ }
- tmp = ggml_cpy(ctx, tmp, cur);
- } else {
- // we rotate only the first n_rot dimensions
- tmp = ggml_rope_ext_inplace(ctx, cur,
- shift, factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- yarn_ext_factor, yarn_attn_factor, yarn_beta_fast, yarn_beta_slow);
+ // the RMS norm commutes with RoPE, but the weight does not
+ if (rp.k_norm) {
+ tmp = ggml_div(ctx, tmp, rp.k_norm);
+ }
+
+ tmp = ggml_rope_ext(ctx, tmp,
+ shift, rp.factors, rp.n_rot, rope_type, rp.n_ctx_orig, rp.freq_base, rp.freq_scale,
+ rp.ext_factor, rp.attn_factor, rp.beta_fast, rp.beta_slow);
+ if (rp.n_offs > 0) {
+ tmp = ggml_rope_set_offset(tmp, rp.n_offs);
+ }
+
+ if (rp.k_norm) {
+ tmp = ggml_mul(ctx, tmp, rp.k_norm);
}
- return tmp;
+ // rotate fwd
+ if (rot) {
+ tmp = llama_mul_mat_hadamard(ctx, tmp, rot);
+ }
+
+ return ggml_cpy(ctx, tmp, k);
}
class llm_graph_input_k_shift : public llm_graph_input_i {
@@ -2179,26 +2196,7 @@ ggml_cgraph * llama_kv_cache::build_graph_shift(llm_graph_result * res, llama_co
continue;
}
- const int64_t n_head_kv = hparams.n_head_kv(il);
- const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);
-
- const auto n_rot = hparams.n_rot(il);
- const auto n_embd_head_k = hparams.n_embd_head_k(il);
- const auto n_embd_nope = hparams.n_lora_kv > 0 ? n_embd_head_k - n_rot : 0;
-
- const float freq_base_l = model.get_rope_freq_base (cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
-
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
- ggml_tensor * k =
- ggml_view_3d(ctx, layer.k,
- n_rot, n_head_kv, get_size()*n_stream,
- ggml_row_size(layer.k->type, n_embd_head_k),
- ggml_row_size(layer.k->type, n_embd_k_gqa),
- ggml_row_size(layer.k->type, n_embd_nope));
-
- ggml_tensor * cur = build_rope_shift(cparams, ctx, k, inp->k_shift, layer.k_rot, rope_factors, freq_base_l, freq_scale_l, il);
+ ggml_tensor * cur = build_rope_shift(cparams, ctx, layer.k, inp->k_shift, layer.k_rot, il);
ggml_build_forward_expand(gf, cur);
}
diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h
index 9ed36c562..5aafbebb3 100644
--- a/src/llama-kv-cache.h
+++ b/src/llama-kv-cache.h
@@ -114,7 +114,9 @@ public:
const layer_reuse_cb & reuse,
const layer_share_cb & share,
// a model can hold more than one cache, so the tensor names have to stay unique
- const char * name_tag = "");
+ const char * name_tag = "",
+ // lightning indexer keys are rotated with the indexer RoPE parameters
+ bool is_indexer = false);
~llama_kv_cache() = default;
@@ -283,6 +285,8 @@ private:
// SWA
const uint32_t n_swa = 0;
+ const bool is_indexer = false;
+
// env: LLAMA_ATTN_ROT_DISABLE
uint32_t n_rot_k = 0;
uint32_t n_rot_v = 0;
@@ -339,12 +343,9 @@ private:
ggml_tensor * build_rope_shift(
const llama_cparams & cparams,
ggml_context * ctx,
- ggml_tensor * cur,
+ ggml_tensor * k,
ggml_tensor * shift,
ggml_tensor * rot,
- ggml_tensor * factors,
- float freq_base,
- float freq_scale,
uint32_t il) const;
ggml_cgraph * build_graph_shift(
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 8c3ef97f7..0b063e845 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -2856,6 +2856,36 @@ const ggml_tensor * llama_model::get_tensor(const char * name) const {
return it->second;
}
+llama_rope_params llama_model::get_rope_params(const llama_cparams & cparams, int il) const {
+ llama_rope_params res;
+
+ res.n_rot = hparams.has_rope(il) ? hparams.n_rot(il) : 0;
+ res.n_offs = hparams.n_lora_kv > 0 ? hparams.n_embd_head_k(il) - hparams.n_rot(il) : 0;
+ res.type = hparams.rope_type;
+ res.n_ctx_orig = cparams.n_ctx_orig_yarn;
+ res.freq_base = get_rope_freq_base (cparams, il);
+ res.freq_scale = get_rope_freq_scale(cparams, il);
+ res.ext_factor = cparams.yarn_ext_factor;
+ res.attn_factor = cparams.yarn_attn_factor;
+ res.beta_fast = cparams.yarn_beta_fast;
+ res.beta_slow = cparams.yarn_beta_slow;
+ res.factors = get_rope_factors(cparams, il);
+
+ return res;
+}
+
+llama_rope_params llama_model::get_rope_params_indexer(const llama_cparams & cparams, int il) const {
+ llama_rope_params res = get_rope_params(cparams, il);
+
+ res.n_offs = 0;
+ res.type = LLAMA_ROPE_TYPE_NEOX;
+ res.freq_base = cparams.rope_freq_base;
+ res.freq_scale = cparams.rope_freq_scale;
+ res.factors = nullptr;
+
+ return res;
+}
+
float llama_model::get_rope_freq_base (const llama_cparams & cparams, int il) const {
return hparams.is_swa(il) ? hparams.rope_freq_base_train_swa : cparams.rope_freq_base;
}
@@ -3631,6 +3661,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_KIMI_LINEAR:
case LLM_ARCH_KIMI_K3:
case LLM_ARCH_GLM5_NEXT:
+ case LLM_ARCH_STARCODER:
return LLAMA_ROPE_TYPE_NONE;
// use what we call a normal RoPE, operating on pairs of consecutive head values
@@ -3639,7 +3670,6 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_LLAMA4:
case LLM_ARCH_DECI:
case LLM_ARCH_BAICHUAN:
- case LLM_ARCH_STARCODER:
case LLM_ARCH_INTERNLM2:
case LLM_ARCH_MINICPM:
case LLM_ARCH_XVERSE:
diff --git a/src/llama-model.h b/src/llama-model.h
index f8cffb346..9c0a51f3d 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -746,6 +746,23 @@ struct llama_hadamard {
llama_hadamard_rotations inv; // latent lookup table -> inverse transform
};
+struct llama_rope_params {
+ int32_t n_rot = 0; // 0 if the layer has no RoPE
+ int32_t n_offs = 0; // first rotated dim in a cached K head
+ int32_t type = LLAMA_ROPE_TYPE_NONE;
+ int32_t n_ctx_orig = 0;
+
+ float freq_base = 0.0f;
+ float freq_scale = 0.0f;
+ float ext_factor = 0.0f;
+ float attn_factor = 0.0f;
+ float beta_fast = 0.0f;
+ float beta_slow = 0.0f;
+
+ ggml_tensor * factors = nullptr;
+ ggml_tensor * k_norm = nullptr; // K norm weight applied after RoPE, a K-shift has to undo it
+};
+
struct llama_model {
llm_type type = LLM_TYPE_UNKNOWN;
llm_arch arch = LLM_ARCH_UNKNOWN;
@@ -971,6 +988,12 @@ struct llama_model {
const struct ggml_tensor * get_tensor(const char * name) const;
+ // the RoPE of a layer, used by both the model graph and the K-shift
+ virtual llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const;
+
+ // same, for the keys of the lightning indexer
+ virtual llama_rope_params get_rope_params_indexer(const llama_cparams & cparams, int il) const;
+
float get_rope_freq_base (const llama_cparams & cparams, int il) const;
float get_rope_freq_scale(const llama_cparams & cparams, int il) const;
diff --git a/src/models/afmoe.cpp b/src/models/afmoe.cpp
index 75da5e2eb..0eb208f63 100644
--- a/src/models/afmoe.cpp
+++ b/src/models/afmoe.cpp
@@ -23,6 +23,10 @@ void llama_model_afmoe::load_arch_hparams(llama_model_loader & ml) {
hparams.swa_type = LLAMA_SWA_TYPE_NONE;
}
+ for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
+ hparams.rope_pattern[il] = (il + 1) % hparams.n_no_rope_layer_step != 0;
+ }
+
// Default to sigmoid if not set
if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {
hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;
@@ -126,15 +130,9 @@ llama_model_afmoe::graph::graph(const llm_graph_params & params) : llm_graph_con
const float kq_scale = 1.0f/sqrtf(float(n_embd_head));
for (int il = 0; il < n_layer; ++il) {
- const float freq_base_l = model.get_rope_freq_base (cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
ggml_tensor * inpSA = inpL;
- // This overlaps with SWA layers in current models, so get_rope_freq_base/scale may be superfluous
- const bool use_rope = hparams.n_no_rope_layer_step > 0 &&
- (il + 1) % hparams.n_no_rope_layer_step != 0;
-
// dual attention normalization (pre)
cur = build_norm(inpL,
model.layers[il].attn_norm, NULL,
@@ -158,19 +156,11 @@ llama_model_afmoe::graph::graph(const llm_graph_params & params) : llm_graph_con
cb(Qcur, "Qcur_normed", il);
cb(Kcur, "Kcur_normed", il);
- if (use_rope) {
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
- cb(Qcur, "Qcur_rope", il);
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
- cb(Kcur, "Kcur_rope", il);
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ cb(Qcur, "Qcur_rope", il);
+
+ Kcur = build_rope(Kcur, inp_pos, il);
+ cb(Kcur, "Kcur_rope", il);
cur = build_attn(inp_attn,
NULL, NULL, NULL, // wo will be applied after gating
diff --git a/src/models/apertus.cpp b/src/models/apertus.cpp
index 2b1ebc965..c0bb23659 100644
--- a/src/models/apertus.cpp
+++ b/src/models/apertus.cpp
@@ -84,8 +84,6 @@ llama_model_apertus::graph::graph(const llm_graph_params & params) : llm_graph_c
// self-attention
{
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
// compute Q and K and RoPE them
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
@@ -96,11 +94,8 @@ llama_model_apertus::graph::graph(const llm_graph_params & params) : llm_graph_c
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur_pos", il);
cb(Kcur, "Kcur_pos", il);
diff --git a/src/models/arcee.cpp b/src/models/arcee.cpp
index c58025ccd..58607d82e 100644
--- a/src/models/arcee.cpp
+++ b/src/models/arcee.cpp
@@ -76,24 +76,12 @@ llama_model_arcee::graph::graph(const llm_graph_params & params) : llm_graph_con
// self-attention
{
- // rope freq factors for llama3; may return nullptr for llama2 and other models
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
// compute Q and K and RoPE them
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/arctic.cpp b/src/models/arctic.cpp
index e9db89d99..a72f2043b 100644
--- a/src/models/arctic.cpp
+++ b/src/models/arctic.cpp
@@ -86,17 +86,8 @@ llama_model_arctic::graph::graph(const llm_graph_params & params) : llm_graph_co
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/baichuan.cpp b/src/models/baichuan.cpp
index e7315b304..3bccc0bff 100644
--- a/src/models/baichuan.cpp
+++ b/src/models/baichuan.cpp
@@ -12,6 +12,11 @@ void llama_model_baichuan::load_arch_hparams(llama_model_loader & ml) {
// TODO: become GGUF KV parameter
hparams.f_max_alibi_bias = 8.0f;
}
+
+ // only the 7B uses RoPE
+ if (type != LLM_TYPE_7B) {
+ std::fill(hparams.rope_pattern.begin(), hparams.rope_pattern.end(), 0);
+ }
}
void llama_model_baichuan::load_arch_tensors(llama_model_loader &) {
@@ -74,25 +79,8 @@ llama_model_baichuan::graph::graph(const llm_graph_params & params) : llm_graph_
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- switch (model.type) {
- case LLM_TYPE_7B:
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
- break;
- case LLM_TYPE_13B:
- case LLM_TYPE_UNKNOWN:
- break;
- default:
- GGML_ABORT("fatal error");
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/bailingmoe.cpp b/src/models/bailingmoe.cpp
index b8e1791fc..890dcaa7b 100644
--- a/src/models/bailingmoe.cpp
+++ b/src/models/bailingmoe.cpp
@@ -83,24 +83,12 @@ llama_model_bailingmoe::graph::graph(const llm_graph_params & params) : llm_grap
// self-attention
{
- // rope freq factors for llama3; may return nullptr for llama2 and other models
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
// compute Q and K and RoPE them
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head_k, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/bailingmoe2.cpp b/src/models/bailingmoe2.cpp
index 05f08c84f..651623e0e 100644
--- a/src/models/bailingmoe2.cpp
+++ b/src/models/bailingmoe2.cpp
@@ -120,14 +120,12 @@ llama_model_bailingmoe2::graph::graph(const llm_graph_params & params) :
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/bailingmoe3.cpp b/src/models/bailingmoe3.cpp
index a904dca8a..8e47e2a0a 100644
--- a/src/models/bailingmoe3.cpp
+++ b/src/models/bailingmoe3.cpp
@@ -205,9 +205,6 @@ llama_model_bailingmoe3::graph::graph(const llm_graph_params & params) :
const int64_t n_seqs = ubatch.n_seqs;
const int64_t n_seq_tokens = ubatch.n_seq_tokens;
- const bool use_mrope = hparams.use_mrope();
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
const int64_t qk_head_dim = hparams.n_embd_head_k_mla();
const int64_t v_head_dim = hparams.n_embd_head_v_mla();
const int64_t qk_rope_head_dim = hparams.n_rot();
@@ -299,17 +296,8 @@ llama_model_bailingmoe3::graph::graph(const llm_graph_params & params) :
ggml_row_size(kv_all->type, kv_lora_rank + qk_rope_head_dim),
ggml_row_size(kv_all->type, kv_lora_rank));
- if (use_mrope) {
- q_pe = ggml_rope_multi(ctx0, q_pe, inp_pos, nullptr, n_rot, sections, rope_type,
- n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
- k_pe = ggml_rope_multi(ctx0, k_pe, inp_pos, nullptr, n_rot, sections, rope_type,
- n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
- } else {
- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- }
+ q_pe = build_rope(q_pe, inp_pos, il);
+ k_pe = build_rope(k_pe, inp_pos, il);
kv = build_norm(kv, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
@@ -463,21 +451,8 @@ llama_model_bailingmoe3::graph_mtp::graph_mtp(const llm_graph_params & params) :
ggml_row_size(kv_all->type, kv_lora_rank + qk_rope_head_dim),
ggml_row_size(kv_all->type, kv_lora_rank));
- const bool use_mrope = hparams.use_mrope();
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
-
- if (use_mrope) {
- q_pe = ggml_rope_multi(ctx0, q_pe, inp_pos, nullptr, n_rot, sections, rope_type,
- n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
- k_pe = ggml_rope_multi(ctx0, k_pe, inp_pos, nullptr, n_rot, sections, rope_type,
- n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
- } else {
- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- }
+ q_pe = build_rope(q_pe, inp_pos, il);
+ k_pe = build_rope(k_pe, inp_pos, il);
kv = build_norm(kv, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
diff --git a/src/models/beam.cpp b/src/models/beam.cpp
index ec4fc1436..8654e0c95 100644
--- a/src/models/beam.cpp
+++ b/src/models/beam.cpp
@@ -156,15 +156,9 @@ llama_model_beam::graph::graph(const llm_graph_params & params) : llm_graph_cont
cb(Kcur, "Kcur_normed", il);
if (is_swa) {
- const float freq_base_l = model.get_rope_freq_base (cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
-
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
+
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur_rope", il);
cb(Kcur, "Kcur_rope", il);
} else {
diff --git a/src/models/bert.cpp b/src/models/bert.cpp
index 2b2108dc4..b98faffc9 100644
--- a/src/models/bert.cpp
+++ b/src/models/bert.cpp
@@ -131,11 +131,8 @@ llama_model_bert::graph::graph(const llm_graph_params & params) : llm_graph_cont
// RoPE
if (model.arch == LLM_ARCH_NOMIC_BERT || model.arch == LLM_ARCH_NOMIC_BERT_MOE ||
model.arch == LLM_ARCH_JINA_BERT_V3) {
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
}
cb(Qcur, "Qcur", il);
diff --git a/src/models/bitnet.cpp b/src/models/bitnet.cpp
index 3c366e13f..caee27a3d 100644
--- a/src/models/bitnet.cpp
+++ b/src/models/bitnet.cpp
@@ -78,17 +78,8 @@ llama_model_bitnet::graph::graph(const llm_graph_params & params) : llm_graph_co
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/chameleon.cpp b/src/models/chameleon.cpp
index 496538506..ab16927c7 100644
--- a/src/models/chameleon.cpp
+++ b/src/models/chameleon.cpp
@@ -103,17 +103,8 @@ llama_model_chameleon::graph::graph(const llm_graph_params & params) : llm_graph
cb(Kcur, "Kcur", il);
}
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/chatglm.cpp b/src/models/chatglm.cpp
index 33c6386b6..e7442b75f 100644
--- a/src/models/chatglm.cpp
+++ b/src/models/chatglm.cpp
@@ -87,17 +87,8 @@ llama_model_chatglm::graph::graph(const llm_graph_params & params) : llm_graph_c
n_embd_head, n_head, n_head_kv, il);
//printf("freq_base: %f freq_scale: %f ext_factor: %f attn_factor: %f\n", freq_base, freq_scale, ext_factor, attn_factor);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/clef.cpp b/src/models/clef.cpp
index 03d3e595d..3ac5782a8 100644
--- a/src/models/clef.cpp
+++ b/src/models/clef.cpp
@@ -249,9 +249,6 @@ llama_model_clef::graph::graph(const llm_graph_params & params) :
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
-
ggml_tensor * cur;
ggml_tensor * inpL;
@@ -274,7 +271,7 @@ llama_model_clef::graph::graph(const llm_graph_params & params) :
if (hparams.is_recr(il)) {
cur = build_layer_attn_linear(cur, il);
} else {
- cur = build_layer_attn(inp_attn, cur, inp_pos, sections, il);
+ cur = build_layer_attn(inp_attn, cur, inp_pos, il);
}
cur = ggml_add(ctx0, cur, inpSA);
@@ -359,7 +356,6 @@ ggml_tensor * llama_model_clef::graph::build_layer_attn(
llm_graph_input_attn_no_cache * inp,
ggml_tensor * cur,
ggml_tensor * inp_pos,
- int * sections,
int il) {
const int64_t n_embd_head = hparams.n_embd_head_v();
@@ -389,17 +385,8 @@ ggml_tensor * llama_model_clef::graph::build_layer_attn(
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
- Qcur = ggml_rope_multi(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_multi(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
diff --git a/src/models/clover-moe.cpp b/src/models/clover-moe.cpp
index 498ca275b..9caaa5630 100644
--- a/src/models/clover-moe.cpp
+++ b/src/models/clover-moe.cpp
@@ -252,13 +252,8 @@ ggml_tensor * llama_model_clover_moe::graph::build_layer_attn(
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/codeshell.cpp b/src/models/codeshell.cpp
index 1d499684c..e316602f5 100644
--- a/src/models/codeshell.cpp
+++ b/src/models/codeshell.cpp
@@ -80,17 +80,8 @@ llama_model_codeshell::graph::graph(const llm_graph_params & params) : llm_graph
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/cogvlm.cpp b/src/models/cogvlm.cpp
index c878a3bf0..657b7d89d 100644
--- a/src/models/cogvlm.cpp
+++ b/src/models/cogvlm.cpp
@@ -46,6 +46,22 @@ void llama_model_cogvlm::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_cogvlm::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ // plain ggml_rope
+ res.n_ctx_orig = 0;
+ res.freq_base = 10000.0f;
+ res.freq_scale = 1.0f;
+ res.ext_factor = 0.0f;
+ res.attn_factor = 1.0f;
+ res.beta_fast = 0.0f;
+ res.beta_slow = 0.0f;
+ res.factors = nullptr;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_cogvlm::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -113,8 +129,8 @@ llama_model_cogvlm::graph::graph(const llm_graph_params & params) :
ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, n_embd_head, n_head_kv, n_tokens, n_embd_head * sizeof(float),
qkv->nb[1], 2 * n_embd * ggml_element_size(qkv));
- Qcur = ggml_rope(ctx0, Qcur, inp_pos, n_embd_head, rope_type);
- Kcur = ggml_rope(ctx0, Kcur, inp_pos, n_embd_head, rope_type);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cur = build_attn(inp_attn,
wo, nullptr, wo_s,
diff --git a/src/models/cohere2.cpp b/src/models/cohere2.cpp
index acde0fea6..251b2da62 100644
--- a/src/models/cohere2.cpp
+++ b/src/models/cohere2.cpp
@@ -3,6 +3,7 @@
void llama_model_cohere2::load_arch_hparams(llama_model_loader & ml) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
load_swa_pattern(ml, 4);
+ hparams.rope_pattern = hparams.is_swa_impl;
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
@@ -43,6 +44,16 @@ void llama_model_cohere2::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_cohere2::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ // the SWA layers use the global base and scale
+ res.freq_base = cparams.rope_freq_base;
+ res.freq_scale = cparams.rope_freq_scale;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_cohere2::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -67,10 +78,7 @@ llama_model_cohere2::graph::graph(const llm_graph_params & params) : llm_graph_c
ggml_tensor * inp_out_ids = build_inp_out_ids();
for (int il = 0; il < n_layer; ++il) {
- const bool is_swa = hparams.is_swa(il);
// UNUSED:
- // const float freq_base_l = model.get_rope_freq_base (cparams, il);
- // const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
// norm
cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM, il);
@@ -79,26 +87,12 @@ llama_model_cohere2::graph::graph(const llm_graph_params & params) : llm_graph_c
// self-attention
{
- // rope freq factors for 128k context
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
// compute Q and K and RoPE them
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- if (is_swa) {
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cpp
index f673c99bc..5bb2db15a 100644
--- a/src/models/cohere2moe.cpp
+++ b/src/models/cohere2moe.cpp
@@ -27,6 +27,11 @@ void llama_model_cohere2moe::load_arch_hparams(llama_model_loader & ml) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
load_swa_pattern(ml, 4, true);
+ // the dense prefix and the MTP layers use RoPE even if they are not SWA
+ for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
+ hparams.rope_pattern[il] = hparams.is_swa(il) || il < hparams.n_layer_dense_lead || il >= hparams.n_layer();
+ }
+
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
@@ -133,6 +138,16 @@ void llama_model_cohere2moe::load_arch_tensors(llama_model_loader & ml) {
}
}
+llama_rope_params llama_model_cohere2moe::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ // the SWA layers use the global base and scale
+ res.freq_base = cparams.rope_freq_base;
+ res.freq_scale = cparams.rope_freq_scale;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_cohere2moe::build_arch_graph(const llm_graph_params & params) const {
if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
return std::make_unique<graph_mtp>(params);
@@ -157,9 +172,6 @@ llama_model_cohere2moe::graph::graph(const llm_graph_params & params) : llm_grap
// MTP/NextN layers are loaded as extra decoder blocks but not executed in the main pass.
for (int il = 0; il < n_layer; ++il) {
- const bool is_swa = hparams.is_swa(il);
- // Dense-prefix full-attention layers use RoPE; later layers follow the SWA pattern.
- const bool force_rope = static_cast<uint32_t>(il) < hparams.n_layer_dense_lead;
cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, cohere2moe_norm_type, il);
cb(cur, "attn_norm", il);
@@ -172,19 +184,8 @@ llama_model_cohere2moe::graph::graph(const llm_graph_params & params) : llm_grap
auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur,
n_embd_head, n_head, n_head_kv, il);
- if (is_swa || force_rope) {
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
@@ -346,15 +347,8 @@ llama_model_cohere2moe::graph_mtp::graph_mtp(const llm_graph_params & params) :
ggml_tensor * ffn_inp = cur;
auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il);
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "mtp_Qcur", il);
cb(Kcur, "mtp_Kcur", il);
diff --git a/src/models/command-r.cpp b/src/models/command-r.cpp
index dd1505186..bea97e6d8 100644
--- a/src/models/command-r.cpp
+++ b/src/models/command-r.cpp
@@ -80,15 +80,13 @@ llama_model_command_r::graph::graph(const llm_graph_params & params) :
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM, il);
cb(Qcur, "Qcur", il);
}
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
if (model.layers[il].attn_k_norm) {
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM, il);
cb(Kcur, "Kcur", il);
}
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/dbrx.cpp b/src/models/dbrx.cpp
index 5b4c87d0c..343040e0b 100644
--- a/src/models/dbrx.cpp
+++ b/src/models/dbrx.cpp
@@ -76,17 +76,8 @@ llama_model_dbrx::graph::graph(const llm_graph_params & params) : llm_graph_cont
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/deci.cpp b/src/models/deci.cpp
index 9a5beecc1..114b12640 100644
--- a/src/models/deci.cpp
+++ b/src/models/deci.cpp
@@ -118,18 +118,13 @@ llama_model_deci::graph::graph(const llm_graph_params & params) : llm_graph_cont
cb(cur, "wo", il);
} else if (n_head > 0) {
// self-attention
- // rope freq factors for llama3; may return nullptr for llama2 and other models
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
// compute Q and K and RoPE them
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/deepseek.cpp b/src/models/deepseek.cpp
index 76760a560..c6a6d6a97 100644
--- a/src/models/deepseek.cpp
+++ b/src/models/deepseek.cpp
@@ -102,18 +102,12 @@ llama_model_deepseek::graph::graph(const llm_graph_params & params) :
// self-attention
{
- // rope freq factors for llama3; may return nullptr for llama2 and other models
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
// compute Q and K and RoPE them
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/deepseek2.cpp b/src/models/deepseek2.cpp
index 36c8ea93c..7ed35ee9a 100644
--- a/src/models/deepseek2.cpp
+++ b/src/models/deepseek2.cpp
@@ -308,14 +308,10 @@ llama_model_deepseek2::graph_mtp::graph_mtp(const llm_graph_params & params) :
const float kq_scale =
1.0f * mscale * mscale / sqrtf(float(n_embd_head_k_mla));
- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ q_pe = build_rope(q_pe, inp_pos, il);
cb(q_pe, "mtp_q_pe_rope", il);
- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ k_pe = build_rope(k_pe, inp_pos, il);
cb(k_pe, "mtp_k_pe_rope", il);
q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
@@ -466,7 +462,6 @@ llama_model_deepseek2::graph::graph(const llm_graph_params & params) :
// self_attention
if (is_ocr) {
const int n_embed_head = hparams.n_embd / hparams.n_head();
- const int ocr_rope_type = GGML_ROPE_TYPE_NEOX;
GGML_ASSERT(n_embed_head == n_embd_head_k && n_embed_head == n_embd_head_v);
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
@@ -476,8 +471,8 @@ llama_model_deepseek2::graph::graph(const llm_graph_params & params) :
cb(Vcur, "v", il);
GGML_ASSERT(fabs(freq_base - 10000.0) < 1e-4);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_embed_head, ocr_rope_type, 0, freq_base, 1, 0, 1, 0, 0);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_embed_head, ocr_rope_type, 0, freq_base, 1, 0, 1, 0, 0);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "q_pe", il);
cb(Kcur, "k_pe", il);
@@ -524,8 +519,7 @@ llama_model_deepseek2::graph::graph(const llm_graph_params & params) :
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
cb(k_pe, "k_pe", il);
- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ k_pe = build_rope(k_pe, inp_pos, il);
cb(k_pe, "k_pe", il);
kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
@@ -542,8 +536,7 @@ llama_model_deepseek2::graph::graph(const llm_graph_params & params) :
q->nb[1], q->nb[2], ggml_row_size(q->type, n_embd_head_qk_nope));
cb(q_pe, "q_pe", il);
- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ q_pe = build_rope(q_pe, inp_pos, il);
cb(q_pe, "q_pe", il);
// {n_embd_head_qk_nope, n_tokens, n_head}
@@ -606,8 +599,7 @@ llama_model_deepseek2::graph::graph(const llm_graph_params & params) :
cb(Vcur, "Vcur_cont", il);
// RoPE is applied to the trailing dims only
- ggml_tensor * Qcur = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
- freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
+ ggml_tensor * Qcur = build_rope(q, inp_pos, il);
Qcur = ggml_rope_set_offset(Qcur, n_embd_head_qk_nope);
cb(Qcur, "Qcur", il);
diff --git a/src/models/deepseek2ocr.cpp b/src/models/deepseek2ocr.cpp
index 88026c3f8..3357f1b13 100644
--- a/src/models/deepseek2ocr.cpp
+++ b/src/models/deepseek2ocr.cpp
@@ -74,6 +74,23 @@ void llama_model_deepseek2ocr::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_deepseek2ocr::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ res.n_rot = hparams.n_embd_head_k(il);
+ res.n_offs = 0;
+ res.type = LLAMA_ROPE_TYPE_NEOX;
+ res.n_ctx_orig = 0;
+ res.freq_scale = 1.0f;
+ res.ext_factor = 0.0f;
+ res.attn_factor = 1.0f;
+ res.beta_fast = 0.0f;
+ res.beta_slow = 0.0f;
+ res.factors = nullptr;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_deepseek2ocr::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cpp
index d208c524d..6a7a0a7a9 100644
--- a/src/models/deepseek32.cpp
+++ b/src/models/deepseek32.cpp
@@ -224,9 +224,7 @@ llama_model_deepseek32::graph::graph(const llm_graph_params & params) :
// {n_embd_indexer_head, n_indexer_head, n_tokens}
indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, n_tokens);
- indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_rot,
- LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ indexer_q = build_rope_indexer(indexer_q, inp_pos, il);
cb(indexer_q, "indexer_q", il);
ggml_tensor * indexer_k = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_k, cur);
@@ -237,9 +235,7 @@ llama_model_deepseek32::graph::graph(const llm_graph_params & params) :
// {n_embd_indexer_head, 1, n_tokens}
indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens);
- indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, nullptr, n_rot,
- LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ indexer_k = build_rope_indexer(indexer_k, inp_pos, il);
cb(indexer_k, "indexer_k", il);
// perform Hadamard transform on indexer q and k
@@ -347,12 +343,10 @@ llama_model_deepseek32::graph::graph(const llm_graph_params & params) :
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
cb(k_pe, "k_pe", il);
- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ q_pe = build_rope(q_pe, inp_pos, il);
cb(q_pe, "q_pe", il);
- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ k_pe = build_rope(k_pe, inp_pos, il);
cb(k_pe, "k_pe", il);
kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
@@ -607,12 +601,10 @@ llama_model_deepseek32::graph_mtp::graph_mtp(const llm_graph_params & params)
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
cb(k_pe, "mtp_k_pe", il);
- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ q_pe = build_rope(q_pe, inp_pos, il);
cb(q_pe, "mtp_q_pe", il);
- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ k_pe = build_rope(k_pe, inp_pos, il);
cb(k_pe, "mtp_k_pe", il);
kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
diff --git a/src/models/dots1.cpp b/src/models/dots1.cpp
index 41ffd7101..c2fa96560 100644
--- a/src/models/dots1.cpp
+++ b/src/models/dots1.cpp
@@ -106,14 +106,12 @@ llama_model_dots1::graph::graph(const llm_graph_params & params) :
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/dots3note.cpp b/src/models/dots3note.cpp
index c05337494..a6fccd58f 100644
--- a/src/models/dots3note.cpp
+++ b/src/models/dots3note.cpp
@@ -145,6 +145,14 @@ void llama_model_dots3note::load_arch_tensors(llama_model_loader & ml) {
}
}
+llama_rope_params llama_model_dots3note::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ res.freq_scale = cparams.rope_freq_scale;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_dots3note::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -186,7 +194,6 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope;
const float kq_scale = 1.0f/sqrtf(float(n_embd_head_k_mla));
- const float freq_base_l = model.get_rope_freq_base(cparams, il);
// norm
cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
@@ -211,9 +218,7 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
// {n_embd_indexer_head, n_indexer_head, n_tokens}
indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, n_tokens);
- indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_rot,
- LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ indexer_q = build_rope_indexer(indexer_q, inp_pos, il);
cb(indexer_q, "indexer_q", il);
ggml_tensor * indexer_k = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_k, cur);
@@ -224,9 +229,7 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
// {n_embd_indexer_head, 1, n_tokens}
indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens);
- indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, nullptr, n_rot,
- LLAMA_ROPE_TYPE_NEOX, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ indexer_k = build_rope_indexer(indexer_k, inp_pos, il);
cb(indexer_k, "indexer_k", il);
// perform Hadamard transform on indexer q and k
@@ -332,12 +335,10 @@ llama_model_dots3note::graph::graph(const llm_graph_params & params) :
k_pe = build_norm(k_pe, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);
cb(k_pe, "k_pe", il);
- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ q_pe = build_rope(q_pe, inp_pos, il);
cb(q_pe, "q_pe", il);
- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ k_pe = build_rope(k_pe, inp_pos, il);
cb(k_pe, "k_pe", il);
kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
diff --git a/src/models/dream.cpp b/src/models/dream.cpp
index 173794f57..76c0b03c8 100644
--- a/src/models/dream.cpp
+++ b/src/models/dream.cpp
@@ -81,11 +81,8 @@ llama_model_dream::graph::graph(const llm_graph_params & params) :
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/eagle3.cpp b/src/models/eagle3.cpp
index 580f44e59..990b9fe5d 100644
--- a/src/models/eagle3.cpp
+++ b/src/models/eagle3.cpp
@@ -234,20 +234,9 @@ llama_model_eagle3::graph<false>::graph(const llm_graph_params & params) : llm_g
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
- // rope freq factors, returns nullptr if not available
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
// RoPE
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur_rope", il);
cb(Kcur, "Kcur_rope", il);
diff --git a/src/models/ernie4-5-moe.cpp b/src/models/ernie4-5-moe.cpp
index e5ebe3955..7edf9f508 100644
--- a/src/models/ernie4-5-moe.cpp
+++ b/src/models/ernie4-5-moe.cpp
@@ -37,11 +37,8 @@ llama_model_ernie4_5_moe::graph::graph(const llm_graph_params & params) :
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/ernie4-5.cpp b/src/models/ernie4-5.cpp
index 28197fb0a..932cc74ce 100644
--- a/src/models/ernie4-5.cpp
+++ b/src/models/ernie4-5.cpp
@@ -105,11 +105,8 @@ llama_model_ernie4_5::graph::graph(const llm_graph_params & params) :
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/eurobert.cpp b/src/models/eurobert.cpp
index 3e0d0902d..7a42f0dbf 100644
--- a/src/models/eurobert.cpp
+++ b/src/models/eurobert.cpp
@@ -62,17 +62,8 @@ llama_model_eurobert::graph::graph(const llm_graph_params & params) : llm_graph_
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/exaone-moe.cpp b/src/models/exaone-moe.cpp
index c7cd77da2..15bbd5a3f 100644
--- a/src/models/exaone-moe.cpp
+++ b/src/models/exaone-moe.cpp
@@ -4,6 +4,7 @@ void llama_model_exaone_moe::load_arch_hparams(llama_model_loader & ml) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
hparams.n_swa = 128;
load_swa_pattern(ml, 4);
+ hparams.rope_pattern = hparams.is_swa_impl;
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
@@ -101,6 +102,16 @@ void llama_model_exaone_moe::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_exaone_moe::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ // the SWA layers use the global base and scale
+ res.freq_base = cparams.rope_freq_base;
+ res.freq_scale = cparams.rope_freq_scale;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_exaone_moe::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -127,17 +138,12 @@ llama_model_exaone_moe::graph::graph(const llm_graph_params & params) :
for (int il = 0; il < n_layer; ++il) {
ggml_tensor * inpSA = inpL;
- // use RoPE for SWA layers
- const bool is_local_layer = hparams.is_swa(il);
-
// norm
cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
cb(cur, "attn_norm", il);
// self-attention
{
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
// compute Q and K and RoPE them
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
@@ -147,13 +153,8 @@ llama_model_exaone_moe::graph::graph(const llm_graph_params & params) :
cb(Qcur, "Qcur_normed", il);
cb(Kcur, "Kcur_normed", il);
- if (is_local_layer) {
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base,
- freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base,
- freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
diff --git a/src/models/exaone.cpp b/src/models/exaone.cpp
index cd6611aee..780f34f04 100644
--- a/src/models/exaone.cpp
+++ b/src/models/exaone.cpp
@@ -71,18 +71,12 @@ llama_model_exaone::graph::graph(const llm_graph_params & params) :
// self-attention
{
- // rope freq factors for llama3; may return nullptr for llama2 and other models
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
// compute Q and K and RoPE them
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/exaone4.cpp b/src/models/exaone4.cpp
index de141b173..bca08e061 100644
--- a/src/models/exaone4.cpp
+++ b/src/models/exaone4.cpp
@@ -5,6 +5,7 @@ void llama_model_exaone4::load_arch_hparams(llama_model_loader & ml) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
hparams.n_swa = 4096;
load_swa_pattern(ml, 4);
+ hparams.rope_pattern = hparams.is_swa_impl;
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
@@ -70,6 +71,16 @@ void llama_model_exaone4::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_exaone4::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ // the SWA layers use the global base and scale
+ res.freq_base = cparams.rope_freq_base;
+ res.freq_scale = cparams.rope_freq_scale;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_exaone4::build_arch_graph(const llm_graph_params & params) const {
if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) {
return std::make_unique<graph<true>>(params);
@@ -108,13 +119,11 @@ llama_model_exaone4::graph<iswa>::graph(const llm_graph_params & params) :
ggml_tensor * inpSA = inpL;
// use RoPE for SWA layers or non-SWA models
- const bool use_rope = hparams.is_swa(il) || hparams.swa_type == LLAMA_SWA_TYPE_NONE;
cur = inpL;
// self-attention
{
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
@@ -124,13 +133,8 @@ llama_model_exaone4::graph<iswa>::graph(const llm_graph_params & params) :
cb(Qcur, "Qcur_normed", il);
cb(Kcur, "Kcur_normed", il);
- if (use_rope) {
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base,
- freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base,
- freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
diff --git a/src/models/falcon-h1.cpp b/src/models/falcon-h1.cpp
index 08bd20240..22c677b47 100644
--- a/src/models/falcon-h1.cpp
+++ b/src/models/falcon-h1.cpp
@@ -105,6 +105,14 @@ void llama_model_falcon_h1::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_falcon_h1::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ res.factors = nullptr;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_falcon_h1::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -139,11 +147,8 @@ llama_model_falcon_h1::graph::graph(const llm_graph_params & params) :
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, hparams.rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, hparams.rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur-post-rope", il);
cb(Kcur, "Kcur-post-rope", il);
diff --git a/src/models/falcon.cpp b/src/models/falcon.cpp
index 5eaecfd8f..6269e633c 100644
--- a/src/models/falcon.cpp
+++ b/src/models/falcon.cpp
@@ -91,17 +91,8 @@ llama_model_falcon::graph::graph(const llm_graph_params & params) : llm_graph_co
n_embd_head, n_head, n_head_kv, il);
// using mode = 2 for neox mode
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/flint.cpp b/src/models/flint.cpp
index a2995ff4d..520334ce6 100644
--- a/src/models/flint.cpp
+++ b/src/models/flint.cpp
@@ -70,6 +70,16 @@ void llama_model_flint::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_flint::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ // the SWA layers use the global base and scale
+ res.freq_base = cparams.rope_freq_base;
+ res.freq_scale = cparams.rope_freq_scale;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_flint::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -109,12 +119,8 @@ llama_model_flint::graph::graph(const llm_graph_params & params) :
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);
- if (hparams.has_rope(il)) {
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
- freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
- freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
diff --git a/src/models/gemma-embedding.cpp b/src/models/gemma-embedding.cpp
index 9c5dc9825..aa43910be 100644
--- a/src/models/gemma-embedding.cpp
+++ b/src/models/gemma-embedding.cpp
@@ -89,8 +89,6 @@ llama_model_gemma_embedding::graph::graph(const llm_graph_params & params) :
ggml_tensor * inp_out_ids = build_inp_out_ids();
for (int il = 0; il < n_layer; ++il) {
- const float freq_base_l = model.get_rope_freq_base(cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
// norm
cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
@@ -105,14 +103,12 @@ llama_model_gemma_embedding::graph::graph(const llm_graph_params & params) :
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/gemma-embedding2.cpp b/src/models/gemma-embedding2.cpp
index d81903b63..920827dc5 100644
--- a/src/models/gemma-embedding2.cpp
+++ b/src/models/gemma-embedding2.cpp
@@ -100,10 +100,6 @@ llama_model_gemma_embedding2::graph::graph(const llm_graph_params & params) :
const int64_t n_head = hparams.n_head(il);
const int64_t n_head_kv = hparams.n_head_kv(il);
- const float freq_base_l = model.get_rope_freq_base(cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
- const int n_rot_l = hparams.n_rot(il);
-
// norm
cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "attn_norm", il);
@@ -128,10 +124,8 @@ llama_model_gemma_embedding2::graph::graph(const llm_graph_params & params) :
cb(Kcur, "Kcur_normed", il);
cb(Vcur, "Vcur_normed", il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur_pos", il);
cb(Kcur, "Kcur_pos", il);
diff --git a/src/models/gemma.cpp b/src/models/gemma.cpp
index 03f917923..1d42acd01 100644
--- a/src/models/gemma.cpp
+++ b/src/models/gemma.cpp
@@ -69,15 +69,8 @@ llama_model_gemma::graph::graph(const llm_graph_params & params) : llm_graph_con
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/gemma2.cpp b/src/models/gemma2.cpp
index 0a6a6f42a..826396ce1 100644
--- a/src/models/gemma2.cpp
+++ b/src/models/gemma2.cpp
@@ -76,8 +76,6 @@ llama_model_gemma2::graph::graph(const llm_graph_params & params) : llm_graph_co
ggml_tensor * inp_out_ids = build_inp_out_ids();
for (int il = 0; il < n_layer; ++il) {
- const float freq_base_l = model.get_rope_freq_base (cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
// norm
cur = build_norm(inpL,
@@ -91,15 +89,8 @@ llama_model_gemma2::graph::graph(const llm_graph_params & params) : llm_graph_co
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/gemma3.cpp b/src/models/gemma3.cpp
index 907d35fb1..9520c3fcf 100644
--- a/src/models/gemma3.cpp
+++ b/src/models/gemma3.cpp
@@ -105,17 +105,6 @@ llama_model_gemma3::graph<iswa>::graph(const llm_graph_params & params) : llm_gr
ggml_tensor * inp_out_ids = build_inp_out_ids();
for (int il = 0; il < n_layer; ++il) {
- float freq_base_l = 0.0f;
- float freq_scale_l = 0.0f;
-
- if constexpr (iswa) {
- freq_base_l = model.get_rope_freq_base (cparams, il);
- freq_scale_l = model.get_rope_freq_scale(cparams, il);
- } else {
- freq_base_l = freq_base;
- freq_scale_l = freq_scale;
- }
-
// norm
cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
cb(cur, "attn_norm", il);
@@ -129,18 +118,12 @@ llama_model_gemma3::graph<iswa>::graph(const llm_graph_params & params) : llm_gr
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/gemma3n.cpp b/src/models/gemma3n.cpp
index 90c062aff..64809def8 100644
--- a/src/models/gemma3n.cpp
+++ b/src/models/gemma3n.cpp
@@ -132,8 +132,6 @@ llama_model_gemma3n::graph::graph(const llm_graph_params & params) :
for (int il = 0; il < n_layer; ++il) {
// this block is made to be closely resemble Gemma3p5DecoderLayer on python code
- const float freq_base_l = model.get_rope_freq_base(cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
ggml_tensor * cur = inpL; // [n_embd, n_tokens, n_altup]
ggml_tensor * predictions = altup_predict(cur, il); // [n_embd, n_tokens, n_altup]
@@ -162,11 +160,8 @@ llama_model_gemma3n::graph::graph(const llm_graph_params & params) :
cb(Kcur, "Kcur_normed", il);
cb(Vcur, "Vcur_normed", il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur_pos", il);
cb(Kcur, "Kcur_pos", il);
@@ -190,8 +185,7 @@ llama_model_gemma3n::graph::graph(const llm_graph_params & params) :
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
cb(Qcur, "Qcur_pos", il);
cur = build_attn(inp_attn,
diff --git a/src/models/gemma4-assistant.cpp b/src/models/gemma4-assistant.cpp
index b6e2d5af1..77968cbb3 100644
--- a/src/models/gemma4-assistant.cpp
+++ b/src/models/gemma4-assistant.cpp
@@ -131,15 +131,10 @@ llama_model_gemma4_assistant::graph::graph(const llm_graph_params & params) :
ggml_tensor * inpL = cur;
for (int il = 0; il < n_layer_nextn; ++il) {
- const bool is_swa = hparams.is_swa(il);
const int64_t n_embd_head = hparams.n_embd_head_k(il);
const int64_t n_head = hparams.n_head(il);
- const float freq_base_l = model.get_rope_freq_base(cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
- const int n_rot_l = hparams.n_rot(il);
-
ggml_tensor * cur_norm = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur_norm, "attn_norm", il);
@@ -148,9 +143,7 @@ llama_model_gemma4_assistant::graph::graph(const llm_graph_params & params) :
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
- ggml_tensor * freq_factors = is_swa ? nullptr : model.layers[il].rope_freqs;
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig,
- freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
cb(Qcur, "Qcur_pos", il);
cur = build_attn(inp_attn, model.layers[il].wo, nullptr, nullptr,
diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cpp
index b937fc1fd..9c061310d 100644
--- a/src/models/gemma4.cpp
+++ b/src/models/gemma4.cpp
@@ -233,22 +233,12 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) :
const int64_t n_head = hparams.n_head(il);
const int64_t n_head_kv = hparams.n_head_kv(il);
- const float freq_base_l = model.get_rope_freq_base(cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
- const int n_rot_l = hparams.n_rot(il);
-
res->t_layer_inp[il] = inpL;
// norm
cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "attn_norm", il);
- ggml_tensor * freq_factors = nullptr;
- if (!hparams.is_swa(il)) {
- // full_attention layers use rope_freqs for proportional rope
- freq_factors = model.layers[il].rope_freqs;
- }
-
// Q projection (shared for both non-KV and KV layers)
// this is to mirror Gemma4Attention in pytorch code
ggml_tensor * qkv_fused = nullptr;
@@ -269,8 +259,7 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) :
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
cb(Qcur, "Qcur_pos", il);
}
@@ -303,8 +292,7 @@ llama_model_gemma4::graph::graph(const llm_graph_params & params) :
cb(Kcur, "Kcur_normed", il);
cb(Vcur, "Vcur_normed", il);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, freq_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Kcur, "Kcur_pos", il);
diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cpp
index f8d6956b3..358defe83 100644
--- a/src/models/glm-dsa.cpp
+++ b/src/models/glm-dsa.cpp
@@ -176,6 +176,14 @@ void llama_model_glm_dsa::load_arch_tensors(llama_model_loader & ml) {
}
}
+llama_rope_params llama_model_glm_dsa::get_rope_params_indexer(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params_indexer(cparams, il);
+
+ res.type = LLAMA_ROPE_TYPE_NORM;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_glm_dsa::build_arch_graph(const llm_graph_params & params) const {
if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
return std::make_unique<graph_mtp>(params);
@@ -258,9 +266,7 @@ llama_model_glm_dsa::graph::graph(const llm_graph_params & params) :
// {n_embd_indexer_head, n_indexer_head, n_tokens}
indexer_q = ggml_reshape_3d(ctx0, indexer_q, n_embd_indexer_head, n_indexer_head, n_tokens);
- indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, nullptr, n_rot,
- LLAMA_ROPE_TYPE_NORM, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ indexer_q = build_rope_indexer(indexer_q, inp_pos, il);
cb(indexer_q, "indexer_q", il);
ggml_tensor * indexer_k = ggml_mul_mat(ctx0, model.layers[il].indexer_attn_k, cur);
@@ -271,9 +277,7 @@ llama_model_glm_dsa::graph::graph(const llm_graph_params & params) :
// {n_embd_indexer_head, 1, n_tokens}
indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens);
- indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, nullptr, n_rot,
- LLAMA_ROPE_TYPE_NORM, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ indexer_k = build_rope_indexer(indexer_k, inp_pos, il);
cb(indexer_k, "indexer_k", il);
// perform Hadamard transform on indexer q and k
@@ -387,12 +391,10 @@ llama_model_glm_dsa::graph::graph(const llm_graph_params & params) :
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
cb(k_pe, "k_pe", il);
- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ q_pe = build_rope(q_pe, inp_pos, il);
cb(q_pe, "q_pe", il);
- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ k_pe = build_rope(k_pe, inp_pos, il);
cb(k_pe, "k_pe", il);
kv_cmpr = build_norm(kv_cmpr, model.layers[il].attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
@@ -647,12 +649,10 @@ llama_model_glm_dsa::graph_mtp::graph_mtp(const llm_graph_params & params)
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
cb(k_pe, "mtp_k_pe", il);
- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ q_pe = build_rope(q_pe, inp_pos, il);
cb(q_pe, "mtp_q_pe", il);
- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ k_pe = build_rope(k_pe, inp_pos, il);
cb(k_pe, "mtp_k_pe", il);
kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
diff --git a/src/models/glm4-moe.cpp b/src/models/glm4-moe.cpp
index 7bf186c5d..2d958aa95 100644
--- a/src/models/glm4-moe.cpp
+++ b/src/models/glm4-moe.cpp
@@ -203,13 +203,8 @@ llama_model_glm4_moe::graph_mtp::graph_mtp(const llm_graph_params & params)
cb(Kcur, "mtp_Kcur_normed", il);
}
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot,
- rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot,
- rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "mtp_Qcur", il);
cb(Kcur, "mtp_Kcur", il);
@@ -287,9 +282,6 @@ llama_model_glm4_moe::graph::graph(const llm_graph_params & params) : llm_graph_
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
-
ggml_tensor * cur;
ggml_tensor * inpL;
@@ -331,24 +323,9 @@ llama_model_glm4_moe::graph::graph(const llm_graph_params & params) : llm_graph_
cb(Kcur, "Kcur_normed", il);
}
- if (use_mrope) {
- Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- } else {
- // Normal RoPE
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot,
- rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot,
- rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- }
+ // Normal RoPE
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/glm4.cpp b/src/models/glm4.cpp
index b7cd2b434..e57c51cbe 100644
--- a/src/models/glm4.cpp
+++ b/src/models/glm4.cpp
@@ -70,9 +70,6 @@ llama_model_glm4::graph::graph(const llm_graph_params & params) : llm_graph_cont
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
-
ggml_tensor * cur;
ggml_tensor * inpL;
@@ -105,24 +102,9 @@ llama_model_glm4::graph::graph(const llm_graph_params & params) : llm_graph_cont
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- if (use_mrope) {
- Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- } else {
- // Normal RoPE
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot,
- rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot,
- rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- }
+ // Normal RoPE
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/gptneox.cpp b/src/models/gptneox.cpp
index ce211b18f..8217e5a8a 100644
--- a/src/models/gptneox.cpp
+++ b/src/models/gptneox.cpp
@@ -113,17 +113,8 @@ llama_model_gptneox::graph::graph(const llm_graph_params & params) : llm_graph_c
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/granite-hybrid.cpp b/src/models/granite-hybrid.cpp
index 6d363f4cf..9e77e2efd 100644
--- a/src/models/granite-hybrid.cpp
+++ b/src/models/granite-hybrid.cpp
@@ -206,14 +206,8 @@ ggml_tensor * llama_model_granite_hybrid::graph::build_attention_layer(ggml_tens
const int il) {
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il);
- if (hparams.has_rope(il)) {
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/granite-swa.cpp b/src/models/granite-swa.cpp
index 4ca4efed8..0276a8400 100644
--- a/src/models/granite-swa.cpp
+++ b/src/models/granite-swa.cpp
@@ -114,6 +114,16 @@ void llama_model_granite_swa::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_granite_swa::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ // the SWA layers use the global base and scale
+ res.freq_base = cparams.rope_freq_base;
+ res.freq_scale = cparams.rope_freq_scale;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_granite_swa::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -204,21 +214,8 @@ ggml_tensor * llama_model_granite_swa::graph::build_attention_layer(
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il);
- const bool use_rope = hparams.has_rope(il);
- if (use_rope) {
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/granite-switch.cpp b/src/models/granite-switch.cpp
index a972bc0e6..53195c97c 100644
--- a/src/models/granite-switch.cpp
+++ b/src/models/granite-switch.cpp
@@ -360,15 +360,8 @@ ggml_tensor * llama_model_granite_switch::graph::build_attention_layer(
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
- if (hparams.has_rope(il)) {
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
diff --git a/src/models/granite.cpp b/src/models/granite.cpp
index 0c89c6949..d46aa7f2e 100644
--- a/src/models/granite.cpp
+++ b/src/models/granite.cpp
@@ -209,20 +209,8 @@ ggml_tensor * llama_model_granite::graph::build_attention_layer(
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il);
- if (hparams.has_rope(il)) {
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/grok.cpp b/src/models/grok.cpp
index e82c23f75..42486e60b 100644
--- a/src/models/grok.cpp
+++ b/src/models/grok.cpp
@@ -116,17 +116,8 @@ llama_model_grok::graph::graph(const llm_graph_params & params) : llm_graph_cont
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/grovemoe.cpp b/src/models/grovemoe.cpp
index 44f44cecb..c86ddbbf5 100644
--- a/src/models/grovemoe.cpp
+++ b/src/models/grovemoe.cpp
@@ -100,14 +100,12 @@ llama_model_grovemoe::graph::graph(const llm_graph_params & params) :
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/hrm-text.cpp b/src/models/hrm-text.cpp
index 5fe7e1542..8e3b34c57 100644
--- a/src/models/hrm-text.cpp
+++ b/src/models/hrm-text.cpp
@@ -114,12 +114,10 @@ ggml_tensor * llama_model_hrm_text::graph::build_stack(llm_graph_input_attn_kv *
auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head_k, n_head, n_head_kv, s);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
cb(Qcur, "Qcur", s);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Kcur, "Kcur", s);
cur = build_attn(inp_attn,
diff --git a/src/models/hunyuan-dense.cpp b/src/models/hunyuan-dense.cpp
index 3f9a75ba4..574a6412f 100644
--- a/src/models/hunyuan-dense.cpp
+++ b/src/models/hunyuan-dense.cpp
@@ -1,5 +1,13 @@
#include "models.h"
+llama_rope_params llama_model_hunyuan_dense::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ res.k_norm = layers[il].attn_k_norm;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_hunyuan_dense::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
diff --git a/src/models/hunyuan-moe.cpp b/src/models/hunyuan-moe.cpp
index 2cfc17f8f..00659c88e 100644
--- a/src/models/hunyuan-moe.cpp
+++ b/src/models/hunyuan-moe.cpp
@@ -49,6 +49,14 @@ void llama_model_hunyuan_moe::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_hunyuan_moe::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ res.k_norm = layers[il].attn_k_norm;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_hunyuan_moe::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -84,28 +92,17 @@ llama_model_hunyuan_moe::graph::graph(const llm_graph_params & params) : llm_gra
// self-attention
{
- // rope freq factors for llama3; may return nullptr for llama2 and other models
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
// compute Q and K and RoPE them
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Kcur = build_rope(Kcur, inp_pos, il);
Kcur = build_norm(Kcur,
model.layers[il].attn_k_norm, nullptr,
diff --git a/src/models/hunyuan-vl.cpp b/src/models/hunyuan-vl.cpp
index 52fc75ddf..219ba39bd 100644
--- a/src/models/hunyuan-vl.cpp
+++ b/src/models/hunyuan-vl.cpp
@@ -63,11 +63,6 @@ llama_model_hunyuan_vl::graph::graph(const llm_graph_params & params) : llm_grap
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
GGML_ASSERT(n_embd_head == n_rot);
- const bool use_mrope = hparams.use_mrope();
-
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
-
ggml_tensor * cur;
ggml_tensor * inpL;
@@ -94,38 +89,12 @@ llama_model_hunyuan_vl::graph::graph(const llm_graph_params & params) : llm_grap
cb(cur, "attn_norm", il);
// self-attention
{
- // rope freq factors for llama3; may return nullptr for llama2 and other models
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
// compute Q and K and RoPE them
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- if (use_mrope) {
- Qcur = ggml_rope_multi(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_multi(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
- } else {
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/hy-v3.cpp b/src/models/hy-v3.cpp
index 37150ef12..3fce6a246 100644
--- a/src/models/hy-v3.cpp
+++ b/src/models/hy-v3.cpp
@@ -119,19 +119,14 @@ llama_model_hy_v3::graph::graph(const llm_graph_params & params) : llm_graph_con
// self-attention
{
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cur = build_attn(inp_attn,
model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,
@@ -286,19 +281,14 @@ llama_model_hy_v3::graph_mtp::graph_mtp(const llm_graph_params & params)
cb(cur, "mtp_attn_norm", il);
{
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il);
Qcur = build_norm(Qcur, layer.attn_q_norm, nullptr, LLM_NORM_RMS, il);
Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
const float kq_scale = 1.0f / sqrtf(float(n_embd_head));
diff --git a/src/models/hy-v4.cpp b/src/models/hy-v4.cpp
index 4e226bafd..386d504bc 100644
--- a/src/models/hy-v4.cpp
+++ b/src/models/hy-v4.cpp
@@ -154,6 +154,16 @@ void llama_model_hy_v4::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_hy_v4::get_rope_params_indexer(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params_indexer(cparams, il);
+
+ // [nope | rope], the same as the MLA keys
+ res.n_offs = hparams.indexer_head_size - res.n_rot;
+ res.type = hparams.rope_type;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_hy_v4::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -297,11 +307,9 @@ ggml_tensor * llama_model_hy_v4::graph::build_attention(
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ q_pe = build_rope(q_pe, inp_pos, il);
cb(q_pe, "q_pe", il);
- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ k_pe = build_rope(k_pe, inp_pos, il);
cb(k_pe, "k_pe", il);
kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
@@ -347,17 +355,13 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k(
const int64_t n_indexer_head = hparams.indexer_n_head;
const int64_t n_embd_indexer = hparams.indexer_head_size;
- const int64_t n_embd_indexer_rope = hparams.n_rot();
- const int64_t n_embd_indexer_nope = n_embd_indexer - n_embd_indexer_rope;
// nope rows come first, so rope only the last n_embd_indexer_rope rows, same as the MLA path
ggml_tensor * iq = ggml_mul_mat(ctx0, layer.indexer_attn_q_b, qr);
iq = ggml_reshape_3d(ctx0, iq, n_embd_indexer, n_indexer_head, n_tokens);
- iq = ggml_rope_ext(ctx0, iq, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base,
- freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
- iq = ggml_rope_set_offset(iq, n_embd_indexer_nope);
+ iq = build_rope_indexer(iq, inp_pos, il);
cb(iq, "indexer_q", il);
ggml_tensor * ik = ggml_mul_mat(ctx0, layer.indexer_attn_k, cur);
@@ -366,9 +370,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k(
ik = ggml_reshape_3d(ctx0, ik, n_embd_indexer, 1, n_tokens);
- ik = ggml_rope_ext(ctx0, ik, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base,
- freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
- ik = ggml_rope_set_offset(ik, n_embd_indexer_nope);
+ ik = build_rope_indexer(ik, inp_pos, il);
cb(ik, "indexer_k", il);
// the reference applies a Hadamard rotation here, but it only helps its FP8 kernels.
@@ -456,11 +458,9 @@ llama_model_hy_v4::graph::build_attention_dsa(
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ q_pe = build_rope(q_pe, inp_pos, il);
cb(q_pe, "q_pe", il);
- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ k_pe = build_rope(k_pe, inp_pos, il);
cb(k_pe, "k_pe", il);
kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
diff --git a/src/models/internlm2.cpp b/src/models/internlm2.cpp
index 4f7fb4346..39ce67af6 100644
--- a/src/models/internlm2.cpp
+++ b/src/models/internlm2.cpp
@@ -71,17 +71,8 @@ llama_model_internlm2::graph::graph(const llm_graph_params & params) : llm_graph
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/jais2.cpp b/src/models/jais2.cpp
index 8cbdcaff4..8376423ea 100644
--- a/src/models/jais2.cpp
+++ b/src/models/jais2.cpp
@@ -84,17 +84,8 @@ llama_model_jais2::graph::graph(const llm_graph_params & params) : llm_graph_con
n_embd_head, n_head, n_head_kv, il);
// Apply RoPE
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur_rope", il);
cb(Kcur, "Kcur_rope", il);
diff --git a/src/models/k2-horizon.cpp b/src/models/k2-horizon.cpp
index 487e7fc3a..534efa523 100644
--- a/src/models/k2-horizon.cpp
+++ b/src/models/k2-horizon.cpp
@@ -261,12 +261,8 @@ llama_model_k2_horizon::graph::graph(const llm_graph_params & params) : llm_grap
Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il);
}
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/kolibri1.cpp b/src/models/kolibri1.cpp
index bf11fba3c..c4d8eca1b 100644
--- a/src/models/kolibri1.cpp
+++ b/src/models/kolibri1.cpp
@@ -15,6 +15,7 @@ void llama_model_kolibri1::load_arch_hparams(llama_model_loader & ml) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+ hparams.rope_pattern = hparams.is_swa_impl;
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
@@ -95,22 +96,11 @@ llama_model_kolibri1::graph::graph(const llm_graph_params & params)
cb(Qcur, "Qcur_normed", il);
cb(Kcur, "Kcur_normed", il);
- if (hparams.is_swa(il)) {
- const float freq_base_l = model.get_rope_freq_base (cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
-
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
- cb(Qcur, "Qcur_rope", il);
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
- cb(Kcur, "Kcur_rope", il);
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ cb(Qcur, "Qcur_rope", il);
+
+ Kcur = build_rope(Kcur, inp_pos, il);
+ cb(Kcur, "Kcur_rope", il);
cur = build_attn(inp_attn,
layer.wo, NULL, layer.wo_s,
diff --git a/src/models/laguna.cpp b/src/models/laguna.cpp
index bc399670b..1bf25eacf 100644
--- a/src/models/laguna.cpp
+++ b/src/models/laguna.cpp
@@ -148,6 +148,21 @@ void llama_model_laguna::load_arch_tensors(llama_model_loader & ml) {
}
}
+llama_rope_params llama_model_laguna::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ // plain RoPE on the SWA layers
+ if (hparams.is_swa(il)) {
+ res.n_ctx_orig = hparams.n_ctx_train;
+ res.ext_factor = 0.0f;
+ res.attn_factor = 1.0f;
+ res.beta_fast = 0.0f;
+ res.beta_slow = 0.0f;
+ }
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_laguna::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -173,26 +188,11 @@ llama_model_laguna::graph::graph(const llm_graph_params & params) : llm_graph_co
const float kq_scale = 1.0f / sqrtf(float(n_embd_head));
for (int il = 0; il < n_layer; ++il) {
- const bool is_swa_il = hparams.is_swa(il);
const int64_t n_head_il = hparams.n_head(il);
const int64_t n_head_kv_il = hparams.n_head_kv(il);
// Per-layer-type RoPE config. SWA layers run plain rope (no YaRN),
// achieved by zeroing the YaRN ext/beta params for those layers.
- const int n_rot_l = is_swa_il ? hparams.n_rot_swa : n_rot;
- const float freq_base_l = is_swa_il ? hparams.rope_freq_base_train_swa : freq_base;
- const float freq_scale_l = is_swa_il ? hparams.rope_freq_scale_train_swa : freq_scale;
- const float ext_factor_l = is_swa_il ? 0.0f : ext_factor;
- // YaRN magnitude scaling (mscale) is already handled by the framework:
- // llama_context pre-divides cparams.yarn_attn_factor by (1 + 0.1*ln(factor))
- // to cancel ggml rope_yarn's internal mscale *= 1 + 0.1*ln(1/freq_scale).
- // Pass attn_factor straight through (like every other arch); SWA layers run
- // plain RoPE (ext_factor 0, no mscale) so force 1.0 there.
- const float attn_factor_l = is_swa_il ? 1.0f : attn_factor;
- const float beta_fast_l = is_swa_il ? 0.0f : beta_fast;
- const float beta_slow_l = is_swa_il ? 0.0f : beta_slow;
- const int n_ctx_orig_l = is_swa_il ? hparams.n_ctx_train : n_ctx_orig;
-
ggml_tensor * inpSA = inpL;
// Pre-norm
@@ -218,12 +218,8 @@ llama_model_laguna::graph::graph(const llm_graph_params & params) : llm_graph_co
cb(Qcur, "Qcur_normed", il);
cb(Kcur, "Kcur_normed", il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,
- n_rot_l, rope_type, n_ctx_orig_l, freq_base_l, freq_scale_l,
- ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
- n_rot_l, rope_type, n_ctx_orig_l, freq_base_l, freq_scale_l,
- ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur_rope", il);
cb(Kcur, "Kcur_rope", il);
diff --git a/src/models/lfm2.cpp b/src/models/lfm2.cpp
index 9d83b3dc1..86b948cf3 100644
--- a/src/models/lfm2.cpp
+++ b/src/models/lfm2.cpp
@@ -134,6 +134,16 @@ void llama_model_lfm2::load_arch_tensors(llama_model_loader &) {
dense_2_out_layers_b = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "bias"), {hparams.n_embd_out() }, TENSOR_NOT_REQUIRED);
}
+llama_rope_params llama_model_lfm2::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ // the SWA layers use the global base and scale
+ res.freq_base = cparams.rope_freq_base;
+ res.freq_scale = cparams.rope_freq_scale;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_lfm2::build_arch_graph(const llm_graph_params & params) const {
if (hparams.n_layer_decision > 0) {
return std::make_unique<graph_decision>(params);
@@ -170,10 +180,8 @@ llama_model_lfm2::graph<iswa>::graph(const llm_graph_params & params) :
cb(k, "model.layers.{}.self_attn.k_layernorm", il);
// RoPE
- q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor,
- attn_factor, beta_fast, beta_slow);
- k = ggml_rope_ext(ctx0, k, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor,
- attn_factor, beta_fast, beta_slow);
+ q = build_rope(q, inp_pos, il);
+ k = build_rope(k, inp_pos, il);
cur = build_attn(inp_attn,
model.layers[il].wo, NULL, model.layers[il].wo_s,
@@ -384,10 +392,8 @@ llama_model_lfm2::graph_decision::graph_decision(const llm_graph_params & params
q = build_norm(q, layer.attn_q_norm, NULL, LLM_NORM_RMS, il);
k = build_norm(k, layer.attn_k_norm, NULL, LLM_NORM_RMS, il);
- q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor,
- attn_factor, beta_fast, beta_slow);
- k = ggml_rope_ext(ctx0, k, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor,
- attn_factor, beta_fast, beta_slow);
+ q = build_rope(q, inp_pos, il);
+ k = build_rope(k, inp_pos, il);
cur = build_attn(inp_attn[0],
layer.wo, NULL, layer.wo_s,
diff --git a/src/models/lfm3.cpp b/src/models/lfm3.cpp
index 9c5399c2f..0746394ce 100644
--- a/src/models/lfm3.cpp
+++ b/src/models/lfm3.cpp
@@ -106,12 +106,6 @@ llama_model_lfm3::graph<decision>::graph(const llm_graph_params & params) :
const int64_t n_head_il = hparams.n_head(il);
const int64_t n_head_kv = hparams.n_head_kv(il);
- const int n_rot_l = hparams.n_rot(il);
- const float freq_base_l = model.get_rope_freq_base (cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
-
- ggml_tensor * rope_factors = hparams.is_swa(il) ? nullptr : model.layers[il].rope_freqs;
-
auto [q, k, v] = build_qkv(model.layers[il], cur_in, n_embd_head, n_head_il, n_head_kv, il);
q = build_norm(q, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
@@ -119,10 +113,8 @@ llama_model_lfm3::graph<decision>::graph(const llm_graph_params & params) :
k = build_norm(k, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(k, "model.layers.{}.self_attn.k_layernorm", il);
- q = ggml_rope_ext(ctx0, q, inp_pos, rope_factors, n_rot_l, rope_type, n_ctx_orig,
- freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow);
- k = ggml_rope_ext(ctx0, k, inp_pos, rope_factors, n_rot_l, rope_type, n_ctx_orig,
- freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow);
+ q = build_rope(q, inp_pos, il);
+ k = build_rope(k, inp_pos, il);
ggml_tensor * attn_out = build_attn(inp_attn,
NULL, NULL, NULL,
diff --git a/src/models/limite.cpp b/src/models/limite.cpp
index 67b368cfd..2bd45561a 100644
--- a/src/models/limite.cpp
+++ b/src/models/limite.cpp
@@ -13,6 +13,7 @@ void llama_model_limite::load_arch_hparams(llama_model_loader & ml) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
load_swa_pattern(ml, 4);
+ hparams.rope_pattern = hparams.is_swa_impl;
std::fill(hparams.mudd_tap_idx.begin(), hparams.mudd_tap_idx.end(), -1);
ml.get_key(LLM_KV_MUDD_TAP_COUNT, hparams.n_mudd_taps, false);
@@ -68,6 +69,16 @@ void llama_model_limite::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_limite::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ // the SWA layers use the global base and scale
+ res.freq_base = cparams.rope_freq_base;
+ res.freq_scale = cparams.rope_freq_scale;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_limite::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -160,14 +171,10 @@ llama_model_limite::graph::graph(const llm_graph_params & params) : llm_graph_co
cb(Kcur, "Kcur_normed", il);
// the full attention layers use no positional encoding
- if (hparams.is_swa(il)) {
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- cb(Qcur, "Qcur", il);
- cb(Kcur, "Kcur", il);
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
cur = build_attn(inp_attn,
nullptr, nullptr, nullptr,
diff --git a/src/models/llada-moe.cpp b/src/models/llada-moe.cpp
index d7baf7a4b..0086d52af 100644
--- a/src/models/llada-moe.cpp
+++ b/src/models/llada-moe.cpp
@@ -90,17 +90,8 @@ llama_model_llada_moe::graph::graph(const llm_graph_params & params) : llm_graph
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/llada.cpp b/src/models/llada.cpp
index 019966a37..8777cff64 100644
--- a/src/models/llada.cpp
+++ b/src/models/llada.cpp
@@ -59,6 +59,14 @@ void llama_model_llada::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_llada::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ res.factors = nullptr;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_llada::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -96,11 +104,8 @@ llama_model_llada::graph::graph(const llm_graph_params & params) : llm_graph_con
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/llama.cpp b/src/models/llama.cpp
index 87174c6bc..26d54968c 100644
--- a/src/models/llama.cpp
+++ b/src/models/llama.cpp
@@ -136,24 +136,12 @@ llama_model_llama::graph<embed>::graph(const llm_graph_params & params) : llm_gr
// self-attention
{
- // rope freq factors for llama3; may return nullptr for llama2 and other models
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
// compute Q and K and RoPE them
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/llama4.cpp b/src/models/llama4.cpp
index 433d1ef58..5fa2a623f 100644
--- a/src/models/llama4.cpp
+++ b/src/models/llama4.cpp
@@ -23,6 +23,10 @@ void llama_model_llama4::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
}
+ for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
+ hparams.rope_pattern[il] = (il + 1) % hparams.n_no_rope_layer_step != 0;
+ }
+
switch (hparams.n_expert) {
case 0: {
// MobileLLM (no MoE)
@@ -134,8 +138,6 @@ llama_model_llama4::graph<iswa>::graph(const llm_graph_params & params) : llm_gr
ggml_tensor * inp_out_ids = build_inp_out_ids();
for (int il = 0; il < n_layer; ++il) {
- const float freq_base_l = model.get_rope_freq_base (cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
ggml_tensor * inpSA = inpL;
@@ -151,25 +153,13 @@ llama_model_llama4::graph<iswa>::graph(const llm_graph_params & params) : llm_gr
// self-attention
{
- // rope freq factors for llama3; may return nullptr for llama2 and other models
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
// compute Q and K and RoPE them
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
if (use_rope) {
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
} else if (inp_attn_scale) {
Qcur = ggml_mul(ctx0, Qcur, inp_attn_scale);
}
diff --git a/src/models/maincoder.cpp b/src/models/maincoder.cpp
index c9bf837c1..1a2d2c695 100644
--- a/src/models/maincoder.cpp
+++ b/src/models/maincoder.cpp
@@ -40,6 +40,14 @@ void llama_model_maincoder::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_maincoder::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ res.k_norm = layers[il].attn_k_norm;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_maincoder::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -77,17 +85,8 @@ llama_model_maincoder::graph::graph(const llm_graph_params & params) : llm_graph
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
diff --git a/src/models/maion-coder.cpp b/src/models/maion-coder.cpp
index 1c778e06a..5c1e376c5 100644
--- a/src/models/maion-coder.cpp
+++ b/src/models/maion-coder.cpp
@@ -94,12 +94,8 @@ llama_model_maion_coder::graph::graph(const llm_graph_params & params) : llm_gra
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
// Full-attention layers do not use RoPE.
- if (hparams.is_swa(il)) {
- const float freq_base_l = model.get_rope_freq_base(cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow);
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cur = build_attn(inp_attn, model.layers[il].wo, NULL, model.layers[il].wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
diff --git a/src/models/maple.cpp b/src/models/maple.cpp
index d853d601c..eb0b3b4ac 100644
--- a/src/models/maple.cpp
+++ b/src/models/maple.cpp
@@ -8,6 +8,7 @@ void llama_model_maple::load_arch_hparams(llama_model_loader & ml) {
ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+ hparams.rope_pattern = hparams.is_swa_impl;
hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
@@ -87,16 +88,8 @@ llama_model_maple::graph::graph(const llm_graph_params & params) :
cb(Qcur, "Qcur_normed", il);
cb(Kcur, "Kcur_normed", il);
- if (hparams.is_swa(il)) {
- const int64_t n_rot_l = hparams.n_rot(il);
- const float freq_base_l = model.get_rope_freq_base(cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
-
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot_l, rope_type, n_ctx_orig, freq_base_l,
- freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot_l, rope_type, n_ctx_orig, freq_base_l,
- freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow);
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
diff --git a/src/models/mellum.cpp b/src/models/mellum.cpp
index e25711b2c..83f70019d 100644
--- a/src/models/mellum.cpp
+++ b/src/models/mellum.cpp
@@ -88,6 +88,19 @@ void llama_model_mellum::load_arch_tensors(llama_model_loader & ml) {
}
}
+llama_rope_params llama_model_mellum::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ if (hparams.is_swa(il)) {
+ res.ext_factor = 0.0f;
+ res.attn_factor = 1.0f;
+ res.beta_fast = 32.0f;
+ res.beta_slow = 1.0f;
+ }
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_mellum::build_arch_graph(const llm_graph_params & params) const {
const bool iswa = hparams.swa_type == LLAMA_SWA_TYPE_STANDARD;
@@ -151,26 +164,8 @@ llama_model_mellum::graph<iswa>::graph(const llm_graph_params & params) : llm_gr
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- const bool is_swa = hparams.is_swa(il);
-
- const float freq_base_l = model.get_rope_freq_base(cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
- const float ext_factor_l = is_swa ? 0.0f : ext_factor;
- const float attn_factor_l = is_swa ? 1.0f : attn_factor;
- const float beta_fast_l = is_swa ? 32.0f : beta_fast;
- const float beta_slow_l = is_swa ? 1.0f : beta_slow;
-
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
@@ -329,26 +324,8 @@ llama_model_mellum::graph_mtp<iswa>::graph_mtp(const llm_graph_params & params)
Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il);
cb(Kcur, "mtp_Kcur_normed", il);
- const bool is_swa = hparams.is_swa(il);
-
- const float freq_base_l = model.get_rope_freq_base(cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
- const float ext_factor_l = is_swa ? 0.0f : ext_factor;
- const float attn_factor_l = is_swa ? 1.0f : attn_factor;
- const float beta_fast_l = is_swa ? 32.0f : beta_fast;
- const float beta_slow_l = is_swa ? 1.0f : beta_slow;
-
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "mtp_Qcur", il);
cb(Kcur, "mtp_Kcur", il);
diff --git a/src/models/mimo2.cpp b/src/models/mimo2.cpp
index a86b0f96d..b318b05ed 100644
--- a/src/models/mimo2.cpp
+++ b/src/models/mimo2.cpp
@@ -104,8 +104,6 @@ llama_model_mimo2::graph::graph(const llm_graph_params & params) : llm_graph_con
uint32_t n_head_l = hparams.n_head(il);
uint32_t n_head_kv_l = hparams.n_head_kv(il);
- const float freq_base_l = model.get_rope_freq_base(cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
cur = inpL;
@@ -148,17 +146,8 @@ llama_model_mimo2::graph::graph(const llm_graph_params & params) : llm_graph_con
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head_v, n_head_kv_l, n_tokens);
}
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
@@ -276,8 +265,6 @@ llama_model_mimo2::graph_mtp::graph_mtp(const llm_graph_params & params)
const uint32_t n_head_l = hparams.n_head(il);
const uint32_t n_head_kv_l = hparams.n_head_kv(il);
- const float freq_base_l = model.get_rope_freq_base(cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
const float v_scale = hparams.f_attn_value_scale;
auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
@@ -333,15 +320,8 @@ llama_model_mimo2::graph_mtp::graph_mtp(const llm_graph_params & params)
ggml_tensor * Kcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_kv_l, n_tokens, row_k, row_full, k_off);
ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, n_embd_head_v, n_head_kv_l, n_tokens, row_v, row_full, v_off);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "mtp_Qcur", il);
cb(Kcur, "mtp_Kcur", il);
diff --git a/src/models/minicpm3.cpp b/src/models/minicpm3.cpp
index e816c66ee..67c8d6708 100644
--- a/src/models/minicpm3.cpp
+++ b/src/models/minicpm3.cpp
@@ -91,8 +91,6 @@ llama_model_minicpm3::graph::graph(const llm_graph_params & params) : llm_graph_
for (int il = 0; il < n_layer; ++il) {
ggml_tensor * inpSA = inpL;
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
// norm
cur = build_norm(inpL,
model.layers[il].attn_norm, NULL,
@@ -162,20 +160,12 @@ llama_model_minicpm3::graph::graph(const llm_graph_params & params) : llm_graph_
v_states = ggml_cont(ctx0, v_states);
cb(v_states, "v_states", il);
- q = ggml_rope_ext(
- ctx0, q, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ q = build_rope(q, inp_pos, il);
q = ggml_rope_set_offset(q, n_embd_head_qk_nope);
cb(q, "q_rope", il);
// shared RoPE key
- k_pe = ggml_rope_ext(
- ctx0, k_pe, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ k_pe = build_rope(k_pe, inp_pos, il);
cb(k_pe, "k_pe", il);
ggml_tensor * q_states = q;
diff --git a/src/models/minimax-01.cpp b/src/models/minimax-01.cpp
index 7a2e55431..08268e0f3 100644
--- a/src/models/minimax-01.cpp
+++ b/src/models/minimax-01.cpp
@@ -255,17 +255,8 @@ llama_model_minimax_01::graph::graph(const llm_graph_params & params) : llm_grap
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/minimax-m2.cpp b/src/models/minimax-m2.cpp
index 46fffb101..5b8258255 100644
--- a/src/models/minimax-m2.cpp
+++ b/src/models/minimax-m2.cpp
@@ -92,17 +92,8 @@ llama_model_minimax_m2::graph::graph(const llm_graph_params & params) : llm_grap
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp
index 4b326896f..2002d08be 100644
--- a/src/models/minimax-m3.cpp
+++ b/src/models/minimax-m3.cpp
@@ -318,14 +318,8 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap
cb(Kcur, "Kcur_normed", il);
// partial rotary: only the first n_rot dims are rotated
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
@@ -347,10 +341,8 @@ llama_model_minimax_m3::graph::graph(const llm_graph_params & params) : llm_grap
ik = ggml_reshape_3d(ctx0, ik, n_idx_dim, 1, n_tokens);
iq = build_norm(iq, model.layers[il].index_q_norm, NULL, LLM_NORM_RMS, il); // +1 baked
ik = build_norm(ik, model.layers[il].index_k_norm, NULL, LLM_NORM_RMS, il);
- iq = ggml_rope_ext(ctx0, iq, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
- freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
- ik = ggml_rope_ext(ctx0, ik, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
- freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
+ iq = build_rope(iq, inp_pos, il);
+ ik = build_rope(ik, inp_pos, il);
const auto * mctx_msa_l = static_cast<const llama_kv_cache_msa_context *>(mctx);
const auto * mctx_cur = mctx_msa_l->get_base();
diff --git a/src/models/mistral-large4.cpp b/src/models/mistral-large4.cpp
index b7a6c4279..a4315a991 100644
--- a/src/models/mistral-large4.cpp
+++ b/src/models/mistral-large4.cpp
@@ -148,6 +148,15 @@ void llama_model_mistral_large4::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_mistral_large4::get_rope_params_indexer(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params_indexer(cparams, il);
+
+ res.type = hparams.rope_type;
+ res.factors = get_rope_factors(cparams, il);
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_mistral_large4::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -216,8 +225,6 @@ llama_model_mistral_large4::graph::graph(const llm_graph_params & params) :
const int64_t n_rot_l = use_rope ? n_rot : 0;
const int64_t n_embd_head_qk_nope = n_embd_head_k - n_rot_l;
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
ggml_tensor * inpSA = inpL;
cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il);
@@ -250,14 +257,10 @@ llama_model_mistral_large4::graph::graph(const llm_graph_params & params) :
indexer_k = ggml_reshape_3d(ctx0, indexer_k, n_embd_indexer_head, 1, n_tokens);
if (use_rope) {
- indexer_q = ggml_rope_ext(ctx0, indexer_q, inp_pos, rope_factors, n_rot,
- rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ indexer_q = build_rope_indexer(indexer_q, inp_pos, il);
cb(indexer_q, "indexer_q", il);
- indexer_k = ggml_rope_ext(ctx0, indexer_k, inp_pos, rope_factors, n_rot,
- rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ indexer_k = build_rope_indexer(indexer_k, inp_pos, il);
cb(indexer_k, "indexer_k", il);
}
@@ -368,8 +371,7 @@ llama_model_mistral_large4::graph::graph(const llm_graph_params & params) :
ggml_row_size(q->type, n_embd_head_k),
ggml_row_size(q->type, n_embd_head_k) * n_head,
ggml_row_size(q->type, n_embd_head_qk_nope));
- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ q_pe = build_rope(q_pe, inp_pos, il);
cb(q_pe, "q_pe", il);
Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0);
@@ -406,8 +408,7 @@ llama_model_mistral_large4::graph::graph(const llm_graph_params & params) :
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_rot),
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_rot),
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ k_pe = build_rope(k_pe, inp_pos, il);
cb(k_pe, "k_pe", il);
Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0);
diff --git a/src/models/mistral3.cpp b/src/models/mistral3.cpp
index 3686d9c21..ab8121863 100644
--- a/src/models/mistral3.cpp
+++ b/src/models/mistral3.cpp
@@ -175,24 +175,12 @@ llama_model_mistral3::graph::graph(const llm_graph_params & params) : llm_graph_
// self-attention
{
- // rope freq factors for llama3; may return nullptr for llama2 and other models
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
-
// compute Q and K and RoPE them
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/models.h b/src/models/models.h
index 4c8f85918..3e338c26b 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -174,7 +174,6 @@ struct llama_model_llama : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_llama4 : public llama_model_base {
llama_model_llama4(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -188,7 +187,6 @@ struct llama_model_llama4 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_llama_embed : public llama_model_llama {
llama_model_llama_embed(const struct llama_model_params & params) : llama_model_llama(params) {}
// reuse load_arch_hparams and load_arch_tensors from llama_model_llama
@@ -199,7 +197,6 @@ struct llama_model_llama_embed : public llama_model_llama {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_maincoder : public llama_model_base {
llama_model_maincoder(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -209,10 +206,10 @@ struct llama_model_maincoder : public llama_model_base {
graph(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_maion_coder : public llama_model_base {
llama_model_maion_coder(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -225,7 +222,6 @@ struct llama_model_maion_coder : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_talkie : public llama_model_base {
llama_model_talkie(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -238,7 +234,6 @@ struct llama_model_talkie : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_deci : public llama_model_base {
llama_model_deci(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -251,7 +246,6 @@ struct llama_model_deci : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_baichuan : public llama_model_base {
llama_model_baichuan(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -264,7 +258,6 @@ struct llama_model_baichuan : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_falcon : public llama_model_base {
llama_model_falcon(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -277,7 +270,6 @@ struct llama_model_falcon : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_grok : public llama_model_base {
llama_model_grok(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -290,7 +282,6 @@ struct llama_model_grok : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_starcoder : public llama_model_base {
llama_model_starcoder(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -303,7 +294,6 @@ struct llama_model_starcoder : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_refact : public llama_model_base {
llama_model_refact(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -316,7 +306,6 @@ struct llama_model_refact : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_bert : public llama_model_base {
llama_model_bert(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -329,7 +318,6 @@ struct llama_model_bert : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_jina_bert_v2 : public llama_model_base {
llama_model_jina_bert_v2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -340,7 +328,6 @@ struct llama_model_jina_bert_v2 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_jina_bert_v3 : public llama_model_base {
llama_model_jina_bert_v3(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -351,7 +338,6 @@ struct llama_model_jina_bert_v3 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_nomic_bert : public llama_model_base {
llama_model_nomic_bert(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -362,7 +348,6 @@ struct llama_model_nomic_bert : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_nomic_bert_moe : public llama_model_base {
llama_model_nomic_bert_moe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -373,7 +358,6 @@ struct llama_model_nomic_bert_moe : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_modern_bert : public llama_model_base {
llama_model_modern_bert(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -391,7 +375,6 @@ struct llama_model_modern_bert : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_neo_bert : public llama_model_base {
llama_model_neo_bert(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -404,7 +387,6 @@ struct llama_model_neo_bert : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_eurobert : public llama_model_base {
llama_model_eurobert(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -417,7 +399,6 @@ struct llama_model_eurobert : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_bloom : public llama_model_base {
llama_model_bloom(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -430,7 +411,6 @@ struct llama_model_bloom : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
// Quant-only stub for mmproj GGUFs
// none of these are ever called, they only exist to satisfy the llama_model_base interface
struct llama_model_clip : public llama_model_base {
@@ -446,7 +426,6 @@ struct llama_model_clip : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_mpt : public llama_model_base {
llama_model_mpt(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -459,7 +438,6 @@ struct llama_model_mpt : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_stablelm : public llama_model_base {
llama_model_stablelm(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -487,6 +465,7 @@ struct llama_model_mellum : public llama_model_base {
graph_mtp(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
@@ -518,7 +497,6 @@ struct llama_model_qwen : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_qwen2 : public llama_model_base {
llama_model_qwen2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -531,7 +509,6 @@ struct llama_model_qwen2 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_dream : public llama_model_base {
llama_model_dream(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -544,7 +521,6 @@ struct llama_model_dream : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_llada : public llama_model_base {
llama_model_llada(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -554,10 +530,10 @@ struct llama_model_llada : public llama_model_base {
graph(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_llada_moe : public llama_model_base {
llama_model_llada_moe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -570,7 +546,6 @@ struct llama_model_llada_moe : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_rnd1 : public llama_model_base {
llama_model_rnd1(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -583,7 +558,6 @@ struct llama_model_rnd1 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_qwen2vl : public llama_model_base {
llama_model_qwen2vl(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -596,7 +570,6 @@ struct llama_model_qwen2vl : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_qwen2moe : public llama_model_base {
llama_model_qwen2moe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -609,7 +582,6 @@ struct llama_model_qwen2moe : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_sensenova_u1 : public llama_model_base {
llama_model_sensenova_u1(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -624,7 +596,6 @@ struct llama_model_sensenova_u1 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_qwen3 : public llama_model_base {
llama_model_qwen3(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -675,7 +646,6 @@ struct llama_model_vllm_decision2 : public llama_model_qwen3 {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_qwen3moe : public llama_model_base {
llama_model_qwen3moe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -688,7 +658,6 @@ struct llama_model_qwen3moe : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_flint : public llama_model_base {
llama_model_flint(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -698,10 +667,10 @@ struct llama_model_flint : public llama_model_base {
graph(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_qwen3vl : public llama_model_base {
llama_model_qwen3vl(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -714,7 +683,6 @@ struct llama_model_qwen3vl : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_qwen3vlmoe : public llama_model_base {
llama_model_qwen3vlmoe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -727,12 +695,10 @@ struct llama_model_qwen3vlmoe : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_qwen3tts : public llama_model_qwen3vl {
llama_model_qwen3tts(const struct llama_model_params & params) : llama_model_qwen3vl(params) {}
};
-
struct llama_model_phi2 : public llama_model_base {
llama_model_phi2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -745,7 +711,6 @@ struct llama_model_phi2 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_phi3 : public llama_model_base {
llama_model_phi3(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -759,7 +724,6 @@ struct llama_model_phi3 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_phimoe : public llama_model_base {
llama_model_phimoe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -771,7 +735,6 @@ struct llama_model_phimoe : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_plamo : public llama_model_base {
llama_model_plamo(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -784,7 +747,6 @@ struct llama_model_plamo : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_plamo2 : public llama_model_base {
llama_model_plamo2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -801,7 +763,6 @@ struct llama_model_plamo2 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_plamo3 : public llama_model_base {
llama_model_plamo3(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -815,7 +776,6 @@ struct llama_model_plamo3 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_gpt2 : public llama_model_base {
llama_model_gpt2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -828,7 +788,6 @@ struct llama_model_gpt2 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_pockettts : public llama_model_base {
llama_model_pockettts(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -841,7 +800,6 @@ struct llama_model_pockettts : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_codeshell : public llama_model_base {
llama_model_codeshell(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -854,7 +812,6 @@ struct llama_model_codeshell : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_orion : public llama_model_base {
llama_model_orion(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -867,7 +824,6 @@ struct llama_model_orion : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_internlm2 : public llama_model_base {
llama_model_internlm2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -880,7 +836,6 @@ struct llama_model_internlm2 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_minicpm3 : public llama_model_base {
llama_model_minicpm3(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -893,7 +848,6 @@ struct llama_model_minicpm3 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_gemma : public llama_model_base {
llama_model_gemma(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -906,7 +860,6 @@ struct llama_model_gemma : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_gemma2 : public llama_model_base {
llama_model_gemma2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -919,7 +872,6 @@ struct llama_model_gemma2 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_gemma3 : public llama_model_base {
llama_model_gemma3(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -933,7 +885,6 @@ struct llama_model_gemma3 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_gemma3n : public llama_model_base {
llama_model_gemma3n(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -964,7 +915,6 @@ struct llama_model_gemma3n : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_gemma4 : public llama_model_base {
llama_model_gemma4(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -983,7 +933,6 @@ struct llama_model_gemma4 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_gemma4_assistant : public llama_model_base {
llama_model_gemma4_assistant(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -996,7 +945,6 @@ struct llama_model_gemma4_assistant : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_gemma_embedding : public llama_model_base {
llama_model_gemma_embedding(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1009,7 +957,6 @@ struct llama_model_gemma_embedding : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_gemma_embedding2 : public llama_model_base {
llama_model_gemma_embedding2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1024,7 +971,6 @@ struct llama_model_gemma_embedding2 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_starcoder2 : public llama_model_base {
llama_model_starcoder2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1037,7 +983,6 @@ struct llama_model_starcoder2 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_mamba : public llama_model_base {
llama_model_mamba(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1050,7 +995,6 @@ struct llama_model_mamba : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_mamba2 : public llama_model_base {
llama_model_mamba2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1061,7 +1005,6 @@ struct llama_model_mamba2 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_maple : public llama_model_base {
llama_model_maple(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1074,7 +1017,6 @@ struct llama_model_maple : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_jamba : public llama_model_base {
llama_model_jamba(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1087,7 +1029,6 @@ struct llama_model_jamba : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_xverse : public llama_model_base {
llama_model_xverse(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1100,7 +1041,6 @@ struct llama_model_xverse : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_command_r : public llama_model_base {
llama_model_command_r(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1113,7 +1053,6 @@ struct llama_model_command_r : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_cohere2 : public llama_model_base {
llama_model_cohere2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1123,10 +1062,10 @@ struct llama_model_cohere2 : public llama_model_base {
graph(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_cohere2moe : public llama_model_base {
llama_model_cohere2moe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1140,10 +1079,10 @@ struct llama_model_cohere2moe : public llama_model_base {
graph_mtp(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_dbrx : public llama_model_base {
llama_model_dbrx(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1156,7 +1095,6 @@ struct llama_model_dbrx : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_olmo : public llama_model_base {
llama_model_olmo(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1169,7 +1107,6 @@ struct llama_model_olmo : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_olmo2 : public llama_model_base {
llama_model_olmo2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1180,10 +1117,10 @@ struct llama_model_olmo2 : public llama_model_base {
graph(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_olmoe : public llama_model_base {
llama_model_olmoe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1196,7 +1133,6 @@ struct llama_model_olmoe : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_muse_spark : public llama_model_base {
llama_model_muse_spark(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1209,7 +1145,6 @@ struct llama_model_muse_spark : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_muse_glimmer : public llama_model_base {
llama_model_muse_glimmer(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1222,7 +1157,6 @@ struct llama_model_muse_glimmer : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_openelm : public llama_model_base {
llama_model_openelm(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1235,7 +1169,6 @@ struct llama_model_openelm : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_gptneox : public llama_model_base {
llama_model_gptneox(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1248,7 +1181,6 @@ struct llama_model_gptneox : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_arctic : public llama_model_base {
llama_model_arctic(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1261,7 +1193,6 @@ struct llama_model_arctic : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_deepseek : public llama_model_base {
llama_model_deepseek(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1274,7 +1205,6 @@ struct llama_model_deepseek : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_deepseek2 : public llama_model_base {
llama_model_deepseek2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1291,7 +1221,6 @@ struct llama_model_deepseek2 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_deepseek32 : public llama_model_base {
llama_model_deepseek32(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1308,7 +1237,6 @@ struct llama_model_deepseek32 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_dots3note : public llama_model_base {
llama_model_dots3note(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1318,6 +1246,7 @@ struct llama_model_dots3note : public llama_model_base {
graph(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
@@ -1583,10 +1512,10 @@ struct llama_model_deepseek2ocr : public llama_model_base {
using graph = llama_model_deepseek2::graph;
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_glm_dsa : public llama_model_base {
llama_model_glm_dsa(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1600,6 +1529,7 @@ struct llama_model_glm_dsa : public llama_model_base {
graph_mtp(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params_indexer(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
@@ -1618,7 +1548,6 @@ struct llama_model_eagle3 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_dflash : public llama_model_base {
llama_model_dflash(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1648,7 +1577,6 @@ struct llama_model_dflash : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_mistral_large4 : public llama_model_base {
llama_model_mistral_large4(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1658,10 +1586,10 @@ struct llama_model_mistral_large4 : public llama_model_base {
graph(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params_indexer(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_mistral4 : public llama_model_deepseek2 {
llama_model_mistral4(const struct llama_model_params & params) : llama_model_deepseek2(params) {}
// reuse load_arch_hparams and load_arch_tensors from llama_model_deepseek2
@@ -1671,7 +1599,6 @@ struct llama_model_mistral4 : public llama_model_deepseek2 {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_chatglm : public llama_model_base {
llama_model_chatglm(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1684,7 +1611,6 @@ struct llama_model_chatglm : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_glm4 : public llama_model_base {
llama_model_glm4(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1697,7 +1623,6 @@ struct llama_model_glm4 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_glm4_moe : public llama_model_base {
llama_model_glm4_moe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1714,7 +1639,6 @@ struct llama_model_glm4_moe : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_bitnet : public llama_model_base {
llama_model_bitnet(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1727,7 +1651,6 @@ struct llama_model_bitnet : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_t5 : public llama_model_base {
llama_model_t5(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1741,7 +1664,6 @@ struct llama_model_t5 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_t5encoder : public llama_model_base {
llama_model_t5encoder(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1752,7 +1674,6 @@ struct llama_model_t5encoder : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_jais : public llama_model_base {
llama_model_jais(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1765,7 +1686,6 @@ struct llama_model_jais : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_jais2 : public llama_model_base {
llama_model_jais2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1778,7 +1698,6 @@ struct llama_model_jais2 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_nemotron : public llama_model_base {
llama_model_nemotron(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1791,7 +1710,6 @@ struct llama_model_nemotron : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_nemotron_h : public llama_model_base {
llama_model_nemotron_h(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1807,7 +1725,6 @@ struct llama_model_nemotron_h : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_nemotron_h_moe : public llama_model_nemotron_h {
llama_model_nemotron_h_moe(const struct llama_model_params & params) : llama_model_nemotron_h(params) {}
// reuse load_arch_hparams and load_arch_tensors from llama_model_nemotron_h
@@ -1821,7 +1738,6 @@ struct llama_model_nemotron_h_moe : public llama_model_nemotron_h {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_metis16 : public llama_model_base {
llama_model_metis16(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1854,7 +1770,6 @@ struct llama_model_metis16 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_exaone : public llama_model_base {
llama_model_exaone(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1867,7 +1782,6 @@ struct llama_model_exaone : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_exaone4 : public llama_model_base {
llama_model_exaone4(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1878,10 +1792,10 @@ struct llama_model_exaone4 : public llama_model_base {
graph(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_exaone_moe : public llama_model_base {
llama_model_exaone_moe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1891,10 +1805,10 @@ struct llama_model_exaone_moe : public llama_model_base {
graph(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_rwkv6 : public llama_model_base {
llama_model_rwkv6(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1907,7 +1821,6 @@ struct llama_model_rwkv6 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_rwkv6qwen2 : public llama_model_base {
llama_model_rwkv6qwen2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1920,7 +1833,6 @@ struct llama_model_rwkv6qwen2 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_rwkv7 : public llama_model_base {
llama_model_rwkv7(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1933,7 +1845,6 @@ struct llama_model_rwkv7 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_arwkv7 : public llama_model_base {
llama_model_arwkv7(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1946,7 +1857,6 @@ struct llama_model_arwkv7 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_granite : public llama_model_base {
llama_model_granite(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1972,7 +1882,6 @@ struct llama_model_granite : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_granite_moe : public llama_model_base {
llama_model_granite_moe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1983,7 +1892,6 @@ struct llama_model_granite_moe : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_granite_switch : public llama_model_base {
llama_model_granite_switch(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2031,7 +1939,6 @@ struct llama_model_granite_switch : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_minicpm : public llama_model_base {
llama_model_minicpm(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2042,7 +1949,6 @@ struct llama_model_minicpm : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_granite_hybrid : public llama_model_base {
llama_model_granite_hybrid(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2058,7 +1964,6 @@ struct llama_model_granite_hybrid : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_granite_swa : public llama_model_base {
llama_model_granite_swa(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2081,10 +1986,10 @@ struct llama_model_granite_swa : public llama_model_base {
const int il);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_chameleon : public llama_model_base {
llama_model_chameleon(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2097,7 +2002,6 @@ struct llama_model_chameleon : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_wavtokenizer_dec : public llama_model_base {
llama_model_wavtokenizer_dec(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2110,7 +2014,6 @@ struct llama_model_wavtokenizer_dec : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_plm : public llama_model_base {
llama_model_plm(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2123,7 +2026,6 @@ struct llama_model_plm : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_limite : public llama_model_base {
llama_model_limite(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2133,10 +2035,10 @@ struct llama_model_limite : public llama_model_base {
graph(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_hrm_text : public llama_model_base {
llama_model_hrm_text(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2155,7 +2057,6 @@ struct llama_model_hrm_text : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_bailingmoe : public llama_model_base {
llama_model_bailingmoe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2168,7 +2069,6 @@ struct llama_model_bailingmoe : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_bailingmoe2 : public llama_model_base {
llama_model_bailingmoe2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2181,7 +2081,6 @@ struct llama_model_bailingmoe2 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_bailingmoe3 : public llama_model_base {
llama_model_bailingmoe3(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2203,7 +2102,6 @@ struct llama_model_bailingmoe3 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_seed_oss : public llama_model_base {
llama_model_seed_oss(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2216,7 +2114,6 @@ struct llama_model_seed_oss : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_dots1 : public llama_model_base {
llama_model_dots1(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2229,7 +2126,6 @@ struct llama_model_dots1 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_arcee : public llama_model_base {
llama_model_arcee(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2242,7 +2138,6 @@ struct llama_model_arcee : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_afmoe : public llama_model_base {
llama_model_afmoe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2255,7 +2150,6 @@ struct llama_model_afmoe : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_beam : public llama_model_base {
llama_model_beam(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2290,10 +2184,10 @@ struct llama_model_laguna : public llama_model_base {
graph(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_ernie4_5 : public llama_model_base {
llama_model_ernie4_5(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2306,7 +2200,6 @@ struct llama_model_ernie4_5 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_ernie4_5_moe : public llama_model_ernie4_5 {
llama_model_ernie4_5_moe(const struct llama_model_params & params) : llama_model_ernie4_5(params) {}
// reuse load_arch_hparams and load_arch_tensors from llama_model_ernie4_5
@@ -2318,7 +2211,6 @@ struct llama_model_ernie4_5_moe : public llama_model_ernie4_5 {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_paddleocr : public llama_model_ernie4_5 {
llama_model_paddleocr(const struct llama_model_params & params) : llama_model_ernie4_5(params) {}
// reuse load_arch_hparams and load_arch_tensors from llama_model_ernie4_5
@@ -2330,7 +2222,6 @@ struct llama_model_paddleocr : public llama_model_ernie4_5 {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_hunyuan_moe : public llama_model_base {
llama_model_hunyuan_moe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2340,6 +2231,7 @@ struct llama_model_hunyuan_moe : public llama_model_base {
graph(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
@@ -2359,7 +2251,6 @@ struct llama_model_hy_v3 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_hy_v4 : public llama_model_base {
llama_model_hy_v4(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2426,10 +2317,10 @@ struct llama_model_hy_v4 : public llama_model_base {
int il) const;
};
+ llama_rope_params get_rope_params_indexer(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_hunyuan_vl : public llama_model_base {
llama_model_hunyuan_vl(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2442,17 +2333,16 @@ struct llama_model_hunyuan_vl : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_hunyuan_dense : public llama_model_hunyuan_vl {
llama_model_hunyuan_dense(const struct llama_model_params & params) : llama_model_hunyuan_vl(params) {}
// reuse load_arch_hparams and load_arch_tensors from llama_model_hunyuan_vl
using graph = llama_model_hunyuan_vl::graph;
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_smollm3 : public llama_model_base {
llama_model_smollm3(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2465,7 +2355,6 @@ struct llama_model_smollm3 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_openai_moe : public llama_model_base {
llama_model_openai_moe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2478,7 +2367,6 @@ struct llama_model_openai_moe : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_falcon_h1 : public llama_model_base {
llama_model_falcon_h1(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2488,10 +2376,10 @@ struct llama_model_falcon_h1 : public llama_model_base {
graph(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_lfm2 : public llama_model_base {
llama_model_lfm2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2512,10 +2400,10 @@ struct llama_model_lfm2 : public llama_model_base {
ggml_tensor * inp_out_ids);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_lfm2moe : public llama_model_base {
llama_model_lfm2moe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2527,7 +2415,6 @@ struct llama_model_lfm2moe : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_lfm3 : public llama_model_base {
llama_model_lfm3(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2542,7 +2429,6 @@ struct llama_model_lfm3 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_smallthinker : public llama_model_base {
llama_model_smallthinker(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2556,7 +2442,6 @@ struct llama_model_smallthinker : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_grovemoe : public llama_model_base {
llama_model_grovemoe(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2569,7 +2454,6 @@ struct llama_model_grovemoe : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_apertus : public llama_model_base {
llama_model_apertus(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2582,7 +2466,6 @@ struct llama_model_apertus : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_minimax_01 : public llama_model_base {
llama_model_minimax_01(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2595,7 +2478,6 @@ struct llama_model_minimax_01 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_minimax_m2 : public llama_model_base {
llama_model_minimax_m2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2641,10 +2523,10 @@ struct llama_model_cogvlm : public llama_model_base {
graph(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_pangu_embed : public llama_model_base {
llama_model_pangu_embed(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2654,10 +2536,10 @@ struct llama_model_pangu_embed : public llama_model_base {
graph(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_qwen3next : public llama_model_base {
llama_model_qwen3next(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2700,7 +2582,6 @@ struct llama_model_qwen3next : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_qwen35 : public llama_model_base {
llama_model_qwen35(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -2713,7 +2594,6 @@ struct llama_model_qwen35 : public llama_model_base {
llm_graph_input_attn_kv * inp_attn,
ggml_tensor * cur,
ggml_tensor * inp_pos,
- int * sections,
int il);
ggml_tensor * build_layer_attn_linear(
@@ -2744,7 +2624,6 @@ struct llama_model_qwen35 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
// Qwen3.5 backbone evaluated without memory, with a joint decision head on top of its hidden states
struct llama_model_clef : public llama_model_qwen35 {
llama_model_clef(const struct llama_model_params & params) : llama_model_qwen35(params) {}
@@ -2815,7 +2694,6 @@ struct llama_model_clef : public llama_model_qwen35 {
llm_graph_input_attn_no_cache * inp_attn,
ggml_tensor * cur,
ggml_tensor * inp_pos,
- int * sections,
int il);
ggml_tensor * build_layer_attn_linear(
@@ -2837,7 +2715,6 @@ struct llama_model_clef : public llama_model_qwen35 {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_qwen4exp : public llama_model_base {
llama_model_qwen4exp(const struct llama_model_params & params) : llama_model_base(params) {}
@@ -2881,7 +2758,6 @@ struct llama_model_qwen4exp : public llama_model_base {
llm_graph_input_kpool * inp_kpool,
ggml_tensor * cur,
ggml_tensor * inp_pos,
- int * sections,
int il);
// dense self-attention over the cells the QSA mask keeps
@@ -2904,7 +2780,6 @@ struct llama_model_qwen4exp : public llama_model_base {
llm_graph_input_kpool * inp_kpool,
ggml_tensor * cur,
ggml_tensor * inp_pos,
- int * sections,
int il);
ggml_tensor * build_layer_attn_linear(
@@ -2969,7 +2844,6 @@ struct llama_model_qwen35moe : public llama_model_base {
llm_graph_input_attn_kv * inp_attn,
ggml_tensor * cur,
ggml_tensor * inp_pos,
- int * sections,
int il);
ggml_tensor * build_layer_attn_linear(
@@ -3000,7 +2874,6 @@ struct llama_model_qwen35moe : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_clover_moe : public llama_model_base {
llama_model_clover_moe(const llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -3033,7 +2906,6 @@ struct llama_model_mistral3 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_mimo2 : public llama_model_base {
llama_model_mimo2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -3050,7 +2922,6 @@ struct llama_model_mimo2 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_kimi_k3 : public llama_model_base {
llama_model_kimi_k3(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -3207,7 +3078,6 @@ struct llama_model_kimi_linear : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_step35 : public llama_model_base {
llama_model_step35(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -3221,10 +3091,10 @@ struct llama_model_step35 : public llama_model_base {
graph_mtp(const llm_graph_params & params);
};
+ llama_rope_params get_rope_params(const llama_cparams & cparams, int il) const override;
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_spark2_5 : public llama_model_base {
llama_model_spark2_5(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -3251,7 +3121,6 @@ struct llama_model_k2_horizon : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
-
struct llama_model_kolibri1 : public llama_model_base {
llama_model_kolibri1(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/src/models/modern-bert.cpp b/src/models/modern-bert.cpp
index 5c6718897..17c9c5ab6 100644
--- a/src/models/modern-bert.cpp
+++ b/src/models/modern-bert.cpp
@@ -160,8 +160,6 @@ llama_model_modern_bert::graph::graph(const llm_graph_params & params) : llm_gra
const int n_layer_enc = n_layer - n_layer_dec;
for (int il = 0; il < n_layer_enc; ++il) {
- const float freq_base_l = model.get_rope_freq_base(cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
cur = inpL;
@@ -178,17 +176,8 @@ llama_model_modern_bert::graph::graph(const llm_graph_params & params) : llm_gra
n_embd_head, n_head, n_head_kv, il);
// RoPE
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/muse-glimmer.cpp b/src/models/muse-glimmer.cpp
index 768010c87..e3f478fe3 100644
--- a/src/models/muse-glimmer.cpp
+++ b/src/models/muse-glimmer.cpp
@@ -11,6 +11,7 @@ void llama_model_muse_glimmer::load_arch_hparams(llama_model_loader & ml) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
load_swa_pattern(ml, 4);
+ hparams.rope_pattern = hparams.is_swa_impl;
switch (hparams.n_layer()) {
case 52: type = LLM_TYPE_30B; break;
@@ -79,14 +80,8 @@ llama_model_muse_glimmer::graph::graph(const llm_graph_params & params)
// expose per-layer residual for speculative drafts (see LLM_KV_TARGET_LAYERS).
res->t_layer_inp[il] = inpL;
- const float freq_base_l = model.get_rope_freq_base (cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
-
ggml_tensor * inpSA = inpL;
- // RoPE runs on the SWA layers, NoPE on full ones.
- const bool use_rope = hparams.is_swa(il);
-
// pre-attention norm (weight+1 folded at conversion time)
cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
cb(cur, "attn_norm", il);
@@ -109,19 +104,11 @@ llama_model_muse_glimmer::graph::graph(const llm_graph_params & params)
cb(Qcur, "Qcur_normed", il);
cb(Kcur, "Kcur_normed", il);
- if (use_rope) {
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
- cb(Qcur, "Qcur_rope", il);
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
- cb(Kcur, "Kcur_rope", il);
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ cb(Qcur, "Qcur_rope", il);
+
+ Kcur = build_rope(Kcur, inp_pos, il);
+ cb(Kcur, "Kcur_rope", il);
// SDPA. wo is deferred; the gate goes between attn_out and o_proj.
cur = build_attn(inp_attn,
diff --git a/src/models/muse-spark.cpp b/src/models/muse-spark.cpp
index 30dd2d1ca..ea6ab84ab 100644
--- a/src/models/muse-spark.cpp
+++ b/src/models/muse-spark.cpp
@@ -15,6 +15,7 @@ void llama_model_muse_spark::load_arch_hparams(llama_model_loader & ml) {
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+ hparams.rope_pattern = hparams.is_swa_impl;
switch (hparams.n_layer()) {
case 62: type = LLM_TYPE_816B_A42B; break;
@@ -96,14 +97,8 @@ llama_model_muse_spark::graph::graph(const llm_graph_params & params)
for (int il = 0; il < n_layer; ++il) {
res->t_layer_inp[il] = inpL;
- const float freq_base_l = model.get_rope_freq_base (cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
-
ggml_tensor * inpSA = inpL;
- // RoPE runs on the SWA layers, NoPE on the global ones.
- const bool use_rope = hparams.is_swa(il);
-
cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
cb(cur, "attn_norm", il);
@@ -120,19 +115,11 @@ llama_model_muse_spark::graph::graph(const llm_graph_params & params)
cb(Qcur, "Qcur_normed", il);
cb(Kcur, "Kcur_normed", il);
- if (use_rope) {
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
- cb(Qcur, "Qcur_rope", il);
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
- cb(Kcur, "Kcur_rope", il);
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ cb(Qcur, "Qcur_rope", il);
+
+ Kcur = build_rope(Kcur, inp_pos, il);
+ cb(Kcur, "Kcur_rope", il);
cur = build_attn(inp_attn,
NULL, NULL, NULL,
diff --git a/src/models/nanbeige.cpp b/src/models/nanbeige.cpp
index 43235b86f..2f3e0ff1c 100644
--- a/src/models/nanbeige.cpp
+++ b/src/models/nanbeige.cpp
@@ -110,20 +110,12 @@ llama_model_nanbeige::graph::graph(const llm_graph_params & params) :
cb(cur, "attn_norm", il);
{
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/nemotron.cpp b/src/models/nemotron.cpp
index a6126c652..f769b2537 100644
--- a/src/models/nemotron.cpp
+++ b/src/models/nemotron.cpp
@@ -84,17 +84,8 @@ llama_model_nemotron::graph::graph(const llm_graph_params & params) : llm_graph_
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/neo-bert.cpp b/src/models/neo-bert.cpp
index 21bd380ee..58a012a68 100644
--- a/src/models/neo-bert.cpp
+++ b/src/models/neo-bert.cpp
@@ -70,17 +70,8 @@ llama_model_neo_bert::graph::graph(const llm_graph_params & params) : llm_graph_
n_embd_head, n_head, n_head_kv, il);
// RoPE
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/olmo.cpp b/src/models/olmo.cpp
index 710cef98e..2afead631 100644
--- a/src/models/olmo.cpp
+++ b/src/models/olmo.cpp
@@ -73,17 +73,8 @@ llama_model_olmo::graph::graph(const llm_graph_params & params) : llm_graph_cont
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/olmo2.cpp b/src/models/olmo2.cpp
index ff09b07dd..c0a3e8295 100644
--- a/src/models/olmo2.cpp
+++ b/src/models/olmo2.cpp
@@ -51,6 +51,21 @@ void llama_model_olmo2::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_olmo2::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ res.freq_base = cparams.rope_freq_base;
+
+ // regular RoPE on the SWA layers, without YaRN
+ if (hparams.is_swa(il)) {
+ res.freq_scale = 1.0f;
+ res.ext_factor = 0.0f;
+ res.attn_factor = 1.0f;
+ }
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_olmo2::build_arch_graph(const llm_graph_params & params) const {
if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) {
return std::make_unique<graph<true>>(params);
@@ -112,36 +127,8 @@ llama_model_olmo2::graph<iswa>::graph(const llm_graph_params & params) : llm_gra
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
- const bool is_swa = hparams.is_swa(il);
-
- if (is_swa) {
- // For sliding window layers, Olmo3 use regular rope with no yarn rope scaling.
- // This is achieved here by setting freq_scale and attn_factor to 1.
- // We also set ext_factor to 0 to avoid a few unnecessary computations.
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, 1.0,
- 0.0, 1.0, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, 1.0,
- 0.0, 1.0, beta_fast, beta_slow
- );
- } else {
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
diff --git a/src/models/olmoe.cpp b/src/models/olmoe.cpp
index 88c5d2115..9ebdd9c94 100644
--- a/src/models/olmoe.cpp
+++ b/src/models/olmoe.cpp
@@ -100,17 +100,8 @@ llama_model_olmoe::graph::graph(const llm_graph_params & params) : llm_graph_con
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/openai-moe.cpp b/src/models/openai-moe.cpp
index a05ad1cb5..920acbb64 100644
--- a/src/models/openai-moe.cpp
+++ b/src/models/openai-moe.cpp
@@ -75,9 +75,6 @@ llama_model_openai_moe::graph::graph(const llm_graph_params & params) : llm_grap
for (int il = 0; il < n_layer; ++il) {
res->t_layer_inp[il] = inpL;
- const float freq_base_l = model.get_rope_freq_base (cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
-
ggml_tensor * inpSA = inpL;
// norm
@@ -92,17 +89,8 @@ llama_model_openai_moe::graph::graph(const llm_graph_params & params) : llm_grap
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_rot, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/openelm.cpp b/src/models/openelm.cpp
index e9b43c749..3328bd54c 100644
--- a/src/models/openelm.cpp
+++ b/src/models/openelm.cpp
@@ -103,17 +103,8 @@ llama_model_openelm::graph::graph(const llm_graph_params & params) : llm_graph_c
LLM_NORM_RMS, il);
cb(Kcur, "Kcur", il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, NULL,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, NULL,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/orion.cpp b/src/models/orion.cpp
index 2c6e361ff..4fe921189 100644
--- a/src/models/orion.cpp
+++ b/src/models/orion.cpp
@@ -73,17 +73,8 @@ llama_model_orion::graph::graph(const llm_graph_params & params) : llm_graph_con
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/paddleocr.cpp b/src/models/paddleocr.cpp
index 084536784..4dbd9e215 100644
--- a/src/models/paddleocr.cpp
+++ b/src/models/paddleocr.cpp
@@ -19,9 +19,6 @@ llama_model_paddleocr::graph::graph(const llm_graph_params & params) :
inpL = build_inp_embd(model.tok_embd);
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
-
// inp_pos - contains the positions
ggml_tensor * inp_pos = build_inp_pos();
@@ -42,17 +39,8 @@ llama_model_paddleocr::graph::graph(const llm_graph_params & params) :
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_multi(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_multi(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/pangu-embed.cpp b/src/models/pangu-embed.cpp
index d8d0a146d..336c71a41 100644
--- a/src/models/pangu-embed.cpp
+++ b/src/models/pangu-embed.cpp
@@ -51,6 +51,14 @@ void llama_model_pangu_embed::load_arch_tensors(llama_model_loader &) {
}
}
+llama_rope_params llama_model_pangu_embed::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ res.factors = nullptr;
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_pangu_embed::build_arch_graph(const llm_graph_params & params) const {
return std::make_unique<graph>(params);
}
@@ -88,16 +96,8 @@ llama_model_pangu_embed::graph::graph(const llm_graph_params & params) : llm_gra
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/phi2.cpp b/src/models/phi2.cpp
index d92dfc6e0..90c502cfd 100644
--- a/src/models/phi2.cpp
+++ b/src/models/phi2.cpp
@@ -74,17 +74,8 @@ llama_model_phi2::graph::graph(const llm_graph_params & params) : llm_graph_cont
{
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], attn_norm_output,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/phi3.cpp b/src/models/phi3.cpp
index c7a1d2131..815ab0465 100644
--- a/src/models/phi3.cpp
+++ b/src/models/phi3.cpp
@@ -93,8 +93,6 @@ llama_model_phi3::graph<iswa>::graph(const llm_graph_params & params) : llm_grap
// self-attention
{
- // rope freq factors for 128k context
- ggml_tensor * rope_factors = model.get_rope_factors(cparams, il);
ggml_tensor* attn_norm_output = build_norm(inpL,
model.layers[il].attn_norm,
@@ -104,17 +102,8 @@ llama_model_phi3::graph<iswa>::graph(const llm_graph_params & params) : llm_grap
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], attn_norm_output,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/plamo.cpp b/src/models/plamo.cpp
index e48ff2bb0..10348f291 100644
--- a/src/models/plamo.cpp
+++ b/src/models/plamo.cpp
@@ -69,17 +69,8 @@ llama_model_plamo::graph::graph(const llm_graph_params & params) : llm_graph_con
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/plamo2.cpp b/src/models/plamo2.cpp
index d392c3fb4..2d9edf56e 100644
--- a/src/models/plamo2.cpp
+++ b/src/models/plamo2.cpp
@@ -233,14 +233,12 @@ ggml_tensor * llama_model_plamo2::graph::build_plamo2_attn_layer(llm_graph_input
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Kcur = build_rope(Kcur, inp_pos, il);
cur = build_attn(inp,
model.layers[il].wo, NULL, model.layers[il].wo_s,
diff --git a/src/models/plamo3.cpp b/src/models/plamo3.cpp
index ab07450a3..d6de1a356 100644
--- a/src/models/plamo3.cpp
+++ b/src/models/plamo3.cpp
@@ -89,16 +89,6 @@ llama_model_plamo3::graph<iswa>::graph(const llm_graph_params & params) :
for (int il = 0; il < n_layer; ++il) {
ggml_tensor * residual = inpL;
- float freq_base_l = 0.0f;
- float freq_scale_l = 0.0f;
- if constexpr (iswa) {
- freq_base_l = model.get_rope_freq_base (cparams, il);
- freq_scale_l = model.get_rope_freq_scale(cparams, il);
- } else {
- freq_base_l = freq_base;
- freq_scale_l = freq_scale;
- }
-
cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
cb(cur, "attn_norm", il);
@@ -128,12 +118,8 @@ llama_model_plamo3::graph<iswa>::graph(const llm_graph_params & params) :
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "attn_k_norm", il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
const float attn_scale = 1.0f / sqrtf(float(head_dim_q));
diff --git a/src/models/plm.cpp b/src/models/plm.cpp
index 61b530e8b..b0069e49b 100644
--- a/src/models/plm.cpp
+++ b/src/models/plm.cpp
@@ -133,20 +133,12 @@ llama_model_plm::graph::graph(const llm_graph_params & params) : llm_graph_conte
0);
cb(v_states, "v_states", il);
- q = ggml_rope_ext(
- ctx0, q, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ q = build_rope(q, inp_pos, il);
q = ggml_rope_set_offset(q, n_embd_head_qk_nope);
cb(q, "q_rope", il);
// shared RoPE key
- k_pe = ggml_rope_ext(
- ctx0, k_pe, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ k_pe = build_rope(k_pe, inp_pos, il);
cb(k_pe, "k_pe", il);
ggml_tensor * q_states = q;
diff --git a/src/models/pockettts.cpp b/src/models/pockettts.cpp
index a55fe138f..a7dcb5366 100644
--- a/src/models/pockettts.cpp
+++ b/src/models/pockettts.cpp
@@ -74,17 +74,8 @@ llama_model_pockettts::graph::graph(const llm_graph_params & params) : llm_graph
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/qwen.cpp b/src/models/qwen.cpp
index e009069db..4e0b1986d 100644
--- a/src/models/qwen.cpp
+++ b/src/models/qwen.cpp
@@ -71,17 +71,8 @@ llama_model_qwen::graph::graph(const llm_graph_params & params) : llm_graph_cont
n_embd_head, n_head, n_head_kv, il);
// using mode = 2 for neox mode
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/qwen2.cpp b/src/models/qwen2.cpp
index f38e7a0a3..bb93ca9b1 100644
--- a/src/models/qwen2.cpp
+++ b/src/models/qwen2.cpp
@@ -83,17 +83,8 @@ llama_model_qwen2::graph::graph(const llm_graph_params & params) : llm_graph_con
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/qwen2moe.cpp b/src/models/qwen2moe.cpp
index 6d898eefd..e24919bc6 100644
--- a/src/models/qwen2moe.cpp
+++ b/src/models/qwen2moe.cpp
@@ -95,17 +95,8 @@ llama_model_qwen2moe::graph::graph(const llm_graph_params & params) : llm_graph_
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/qwen2vl.cpp b/src/models/qwen2vl.cpp
index d87c14ba4..5831fc435 100644
--- a/src/models/qwen2vl.cpp
+++ b/src/models/qwen2vl.cpp
@@ -55,9 +55,6 @@ llama_model_qwen2vl::graph::graph(const llm_graph_params & params) : llm_graph_c
auto * inp_attn = build_attn_inp_kv();
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
-
ggml_tensor * inp_out_ids = build_inp_out_ids();
for (int il = 0; il < n_layer; ++il) {
@@ -75,17 +72,8 @@ llama_model_qwen2vl::graph::graph(const llm_graph_params & params) : llm_graph_c
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_multi(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_multi(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/qwen3.cpp b/src/models/qwen3.cpp
index 28d01db54..7a6052970 100644
--- a/src/models/qwen3.cpp
+++ b/src/models/qwen3.cpp
@@ -102,20 +102,12 @@ ggml_tensor * llama_model_qwen3::graph::build_backbone() {
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp
index 5601d2052..d0b4f4c2c 100644
--- a/src/models/qwen35.cpp
+++ b/src/models/qwen35.cpp
@@ -140,9 +140,6 @@ llama_model_qwen35::graph::graph(const llm_graph_params & params) :
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
-
ggml_tensor * cur;
ggml_tensor * inpL;
@@ -172,7 +169,7 @@ llama_model_qwen35::graph::graph(const llm_graph_params & params) :
cur = build_layer_attn_linear(inp->get_recr(), cur, il);
} else {
// Full attention layer
- cur = build_layer_attn(inp->get_attn(), cur, inp_pos, sections, il);
+ cur = build_layer_attn(inp->get_attn(), cur, inp_pos, il);
}
if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {
@@ -267,7 +264,6 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn(
llm_graph_input_attn_kv * inp,
ggml_tensor * cur,
ggml_tensor * inp_pos,
- int * sections,
int il) {
const int64_t n_embd_head = hparams.n_embd_head_v();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
@@ -308,17 +304,8 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn(
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
// Apply MRoPE
- Qcur = ggml_rope_multi(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_multi(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
@@ -509,9 +496,6 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llm_graph_params & params)
GGML_ASSERT(layer.nextn.enorm && "MTP block missing nextn.enorm");
GGML_ASSERT(layer.nextn.hnorm && "MTP block missing nextn.hnorm");
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
-
// TODO: extract in a common llm_graph_context::build_inp_embd_h()
auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
@@ -604,12 +588,8 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llm_graph_params & params)
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
cb(Vcur, "mtp_Vcur", il);
- Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
const float kq_scale = hparams.f_attention_scale == 0.0f
? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp
index eb23efdd3..0b06b1f5d 100644
--- a/src/models/qwen35moe.cpp
+++ b/src/models/qwen35moe.cpp
@@ -159,9 +159,6 @@ llama_model_qwen35moe::graph::graph(const llm_graph_params & params) :
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
-
ggml_tensor * cur;
ggml_tensor * inpL;
@@ -191,7 +188,7 @@ llama_model_qwen35moe::graph::graph(const llm_graph_params & params) :
cur = build_layer_attn_linear(inp->get_recr(), cur, il);
} else {
// Full attention layer
- cur = build_layer_attn(inp->get_attn(), cur, inp_pos, sections, il);
+ cur = build_layer_attn(inp->get_attn(), cur, inp_pos, il);
}
if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {
@@ -277,7 +274,6 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn(
llm_graph_input_attn_kv * inp,
ggml_tensor * cur,
ggml_tensor * inp_pos,
- int * sections,
int il) {
const int64_t n_embd_head = hparams.n_embd_head_v();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
@@ -318,17 +314,8 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn(
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
// Apply IMRoPE
- Qcur = ggml_rope_multi(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_multi(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
@@ -558,9 +545,6 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llm_graph_params & params)
GGML_ASSERT(layer.nextn.hnorm && "MTP block missing nextn.hnorm");
GGML_ASSERT(layer.ffn_gate_inp && "MTP block missing ffn_gate_inp");
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
-
// TODO: extract in a common llm_graph_context::build_inp_embd_h()
auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
@@ -643,12 +627,8 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llm_graph_params & params)
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
cb(Vcur, "mtp_Vcur", il);
- Qcur = ggml_rope_multi(ctx0, Qcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
const float kq_scale = hparams.f_attention_scale == 0.0f
? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
diff --git a/src/models/qwen3moe.cpp b/src/models/qwen3moe.cpp
index 99263096b..ce7144a22 100644
--- a/src/models/qwen3moe.cpp
+++ b/src/models/qwen3moe.cpp
@@ -97,20 +97,12 @@ llama_model_qwen3moe::graph::graph(const llm_graph_params & params) : llm_graph_
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/qwen3next.cpp b/src/models/qwen3next.cpp
index 864cb0343..0731838f8 100644
--- a/src/models/qwen3next.cpp
+++ b/src/models/qwen3next.cpp
@@ -273,15 +273,8 @@ ggml_tensor * llama_model_qwen3next::graph::build_layer_attn(
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base,
- freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
@@ -705,12 +698,8 @@ llama_model_qwen3next::graph_mtp::graph_mtp(const llm_graph_params & params)
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "mtp_Qcur", il);
cb(Kcur, "mtp_Kcur", il);
diff --git a/src/models/qwen3vl.cpp b/src/models/qwen3vl.cpp
index 7c07e6c94..480e570dc 100644
--- a/src/models/qwen3vl.cpp
+++ b/src/models/qwen3vl.cpp
@@ -71,9 +71,6 @@ llama_model_qwen3vl::graph::graph(const llm_graph_params & params) : llm_graph_c
inpL = build_inp_embd(model.tok_embd);
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
-
// inp_pos - contains the positions
ggml_tensor * inp_pos = build_inp_pos();
@@ -99,20 +96,12 @@ llama_model_qwen3vl::graph::graph(const llm_graph_params & params) : llm_graph_c
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
- Qcur = ggml_rope_multi(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- Kcur = ggml_rope_multi(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/qwen3vlmoe.cpp b/src/models/qwen3vlmoe.cpp
index 20835cb84..c13f61990 100644
--- a/src/models/qwen3vlmoe.cpp
+++ b/src/models/qwen3vlmoe.cpp
@@ -75,9 +75,6 @@ llama_model_qwen3vlmoe::graph::graph(const llm_graph_params & params) : llm_grap
inpL = build_inp_embd(model.tok_embd);
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
-
// inp_pos - contains the positions
ggml_tensor * inp_pos = build_inp_pos();
@@ -103,20 +100,12 @@ llama_model_qwen3vlmoe::graph::graph(const llm_graph_params & params) : llm_grap
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
- Qcur = ggml_rope_multi(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- Kcur = ggml_rope_multi(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp
index a81d62662..c543edf0a 100644
--- a/src/models/qwen4exp.cpp
+++ b/src/models/qwen4exp.cpp
@@ -450,9 +450,6 @@ llama_model_qwen4exp::graph::graph(const llm_graph_params & params) :
GGML_ASSERT(hparams.n_embd_head_v() == hparams.n_embd_head_k());
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
-
auto * inp = build_inp_mem_hybrid();
// qwen4exp always builds llama_memory_hybrid_idx, so this downcast is safe
@@ -519,7 +516,7 @@ llama_model_qwen4exp::graph::graph(const llm_graph_params & params) :
if (hparams.is_recr(il)) {
cur = build_layer_attn_linear(inp->get_recr(), cur, il);
} else {
- cur = build_layer_attn(inp->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, sections, il);
+ cur = build_layer_attn(inp->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, il);
}
if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) {
@@ -591,9 +588,6 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llm_graph_params & params) :
GGML_ASSERT(layer.nextn.eh_proj && layer.nextn.enorm && layer.nextn.hnorm && layer.nextn.hc_head_norm &&
"MTP block missing, load the model with MTP enabled");
- int sections[4];
- std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
-
auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd_out());
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
@@ -646,7 +640,7 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llm_graph_params & params) :
hc_mix attn_mix = build_hc_mix(res_hc, layer.hc_attn_norm, layer.hc_attn_down, layer.hc_attn_up, layer.hc_attn_inject, layer.hc_attn_down_inject, il);
ggml_tensor * cur = attn_mix.cur;
- cur = build_layer_attn(inp_hyb->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, sections, il);
+ cur = build_layer_attn(inp_hyb->get_attn(), mctx_hyb, inp_kpool, cur, inp_pos, il);
res_hc = build_hc_combine(res_hc, cur, attn_mix.inject, il);
hc_mix ffn_mix = build_hc_mix(res_hc, layer.hc_ffn_norm, layer.hc_ffn_down, layer.hc_ffn_up, layer.hc_ffn_inject, layer.hc_ffn_down_inject, il);
@@ -818,7 +812,6 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_sel(
llm_graph_input_kpool * inp_kpool,
ggml_tensor * cur,
ggml_tensor * inp_pos,
- int * sections,
int il) {
const llama_kv_cache_context * mctx_idx = mctx_hyb->get_idx();
@@ -855,9 +848,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_sel(
pooled_new = build_norm(pooled_new, model.layers[il].index_k_norm, nullptr, LLM_NORM_RMS, il);
pooled_new = ggml_reshape_3d(ctx0, pooled_new, idx_dim, 1, n_new);
- pooled_new = ggml_rope_multi(ctx0, pooled_new, inp_kpool->new_pool_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ pooled_new = build_rope(pooled_new, inp_kpool->new_pool_pos, il);
pooled_new = ggml_reshape_2d(ctx0, pooled_new, idx_dim, n_new);
cb(pooled_new, "indexer_pool_k_new", il);
@@ -871,9 +862,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_qsa_sel(
ggml_tensor * q = build_lora_mm(model.layers[il].index_q_proj, cur);
q = ggml_reshape_3d(ctx0, q, idx_dim, n_idx_h, n_tokens);
q = build_norm(q, model.layers[il].index_q_norm, nullptr, LLM_NORM_RMS, il);
- q = ggml_rope_multi(ctx0, q, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ q = build_rope(q, inp_pos, il);
cb(q, "indexer_q", il);
// the reference sums the rectified head scores unweighted, scaled by 1/sqrt(head_dim),
@@ -1022,7 +1011,6 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn(
llm_graph_input_kpool * inp_kpool,
ggml_tensor * cur,
ggml_tensor * inp_pos,
- int * sections,
int il) {
const int64_t n_embd_head = hparams.n_embd_head_v();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
@@ -1030,7 +1018,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn(
// indexer reads the same block input as q/k/v; no cache or no ratio means dense
const bool qsa = inp_kpool != nullptr && hparams.dsv4_compress_ratios[il] > 0;
- ggml_tensor * sel = qsa ? build_qsa_sel(mctx_hyb, inp_kpool, cur, inp_pos, sections, il) : nullptr;
+ ggml_tensor * sel = qsa ? build_qsa_sel(mctx_hyb, inp_kpool, cur, inp_pos, il) : nullptr;
// Qwen3Next uses a single Q projection that outputs query + gate
ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); // [ (n_embd_head * 2) * n_head, n_tokens ]
@@ -1064,17 +1052,8 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn(
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
// Apply IMRoPE
- Qcur = ggml_rope_multi(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_multi(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/ricursive.cpp b/src/models/ricursive.cpp
index cd5dc3547..e14caa8b3 100644
--- a/src/models/ricursive.cpp
+++ b/src/models/ricursive.cpp
@@ -162,17 +162,8 @@ llama_model_ricursive::graph::graph(const llm_graph_params & params) : llm_graph
cb(Kcur, "Kcur_normed", il);
// global layers use no RoPE (NoPE): the reference rotates only sliding_attention layers (modeling_ricursive.py:127-129,203-205)
- if (hparams.has_rope(il)) {
- const float freq_base_l = model.get_rope_freq_base (cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
-
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
diff --git a/src/models/rnd1.cpp b/src/models/rnd1.cpp
index f55448433..8ea8d2364 100644
--- a/src/models/rnd1.cpp
+++ b/src/models/rnd1.cpp
@@ -99,20 +99,12 @@ llama_model_rnd1::graph::graph(const llm_graph_params & params) : llm_graph_cont
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
cb(Qcur, "Qcur_normed", il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
cb(Kcur, "Kcur_normed", il);
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/seed-oss.cpp b/src/models/seed-oss.cpp
index fcf66290b..c7940dc80 100644
--- a/src/models/seed-oss.cpp
+++ b/src/models/seed-oss.cpp
@@ -81,17 +81,8 @@ llama_model_seed_oss::graph::graph(const llm_graph_params & params) : llm_graph_
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/smallthinker.cpp b/src/models/smallthinker.cpp
index 0e9e36ee1..2c183e33c 100644
--- a/src/models/smallthinker.cpp
+++ b/src/models/smallthinker.cpp
@@ -16,6 +16,10 @@ void llama_model_smallthinker::load_arch_hparams(llama_model_loader & ml) {
hparams.n_no_rope_layer_step = hparams.n_layer();
}
+ for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
+ hparams.rope_pattern[il] = hparams.n_no_rope_layer_step == hparams.n_layer() || il % hparams.n_no_rope_layer_step != 0;
+ }
+
ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);
@@ -97,15 +101,9 @@ llama_model_smallthinker::graph<iswa>::graph(const llm_graph_params & params) :
ggml_tensor * inp_out_ids = build_inp_out_ids();
for (int il = 0; il < n_layer; ++il) {
- const float freq_base_l = model.get_rope_freq_base (cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
ggml_tensor * inpSA = inpL;
- // This overlaps with SWA layers in current models, so get_rope_freq_base/scale may be superfluous
- const bool use_rope = hparams.n_no_rope_layer_step == n_layer ||
- il % hparams.n_no_rope_layer_step != 0;
-
ggml_tensor * probs = build_lora_mm(model.layers[il].ffn_gate_inp, inpL); // [n_expert, n_tokens]
cb(probs, "ffn_moe_logits", il);
@@ -119,13 +117,8 @@ llama_model_smallthinker::graph<iswa>::graph(const llm_graph_params & params) :
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- if (use_rope) {
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/smollm3.cpp b/src/models/smollm3.cpp
index 438e5d334..fa9e9c539 100644
--- a/src/models/smollm3.cpp
+++ b/src/models/smollm3.cpp
@@ -3,6 +3,9 @@
void llama_model_smollm3::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
hparams.n_no_rope_layer_step = 4;
+ for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
+ hparams.rope_pattern[il] = (il + 1) % hparams.n_no_rope_layer_step != 0;
+ }
switch (hparams.n_layer()) {
case 36: type = LLM_TYPE_3B; break;
@@ -66,8 +69,6 @@ llama_model_smollm3::graph::graph(const llm_graph_params & params) : llm_graph_c
for (int il = 0; il < n_layer; ++il) {
ggml_tensor * inpSA = inpL;
- const bool use_rope = (il + 1) % hparams.n_no_rope_layer_step != 0;
-
// norm
cur = build_norm(inpL,
model.layers[il].attn_norm, NULL,
@@ -80,19 +81,8 @@ llama_model_smollm3::graph::graph(const llm_graph_params & params) : llm_graph_c
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- if (use_rope) {
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
- }
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
cb(Vcur, "Vcur", il);
diff --git a/src/models/spark2-5.cpp b/src/models/spark2-5.cpp
index 1c6543a2c..e90184d8e 100644
--- a/src/models/spark2-5.cpp
+++ b/src/models/spark2-5.cpp
@@ -73,19 +73,12 @@ llama_model_spark2_5::graph::graph(const llm_graph_params & params) : llm_graph_
const int64_t n_head_i = hparams.n_head(il);
const int64_t n_head_kv_i = hparams.n_head_kv(il);
- const int64_t n_rot_i = hparams.n_rot(il);
- const float freq_base_i = model.get_rope_freq_base(cparams, il);
- const float freq_scale_i = model.get_rope_freq_scale(cparams, il);
ggml_tensor * attn_inp = cur;
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head_i, n_head_kv_i, il);
- Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,
- n_rot_i, rope_type, n_ctx_orig, freq_base_i, freq_scale_i,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
- n_rot_i, rope_type, n_ctx_orig, freq_base_i, freq_scale_i,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur_rope", il);
cb(Kcur, "Kcur_rope", il);
diff --git a/src/models/stablelm.cpp b/src/models/stablelm.cpp
index 84e9b8c76..5aec7beee 100644
--- a/src/models/stablelm.cpp
+++ b/src/models/stablelm.cpp
@@ -96,17 +96,8 @@ llama_model_stablelm::graph::graph(const llm_graph_params & params) : llm_graph_
cb(Kcur, "Kcur", il);
}
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/starcoder2.cpp b/src/models/starcoder2.cpp
index 54f9cb8b1..2b670d562 100644
--- a/src/models/starcoder2.cpp
+++ b/src/models/starcoder2.cpp
@@ -89,17 +89,8 @@ llama_model_starcoder2::graph::graph(const llm_graph_params & params) : llm_grap
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/src/models/step35.cpp b/src/models/step35.cpp
index bd06ce0db..ad8a3a1c7 100644
--- a/src/models/step35.cpp
+++ b/src/models/step35.cpp
@@ -172,6 +172,16 @@ void llama_model_step35::load_arch_tensors(llama_model_loader & ml) {
}
}
+llama_rope_params llama_model_step35::get_rope_params(const llama_cparams & cparams, int il) const {
+ auto res = llama_model::get_rope_params(cparams, il);
+
+ if (hparams.is_swa(il)) {
+ res.factors = nullptr;
+ }
+
+ return res;
+}
+
std::unique_ptr<llm_graph_context> llama_model_step35::build_arch_graph(const llm_graph_params & params) const {
if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
return std::make_unique<graph_mtp>(params);
@@ -195,9 +205,6 @@ llama_model_step35::graph::graph(const llm_graph_params & params) : llm_graph_co
const uint32_t n_head_l = hparams.n_head(il);
const uint32_t n_head_kv_l = hparams.n_head_kv(il);
- const float freq_base_l = model.get_rope_freq_base(cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
-
cur = inpL;
// dump pre-attn RMSNorm input to pinpoint layer boundary issues
@@ -232,19 +239,8 @@ llama_model_step35::graph::graph(const llm_graph_params & params) : llm_graph_co
}
// RoPE (partial rotary factors per layer)
- const bool is_swa = hparams.is_swa(il);
- ggml_tensor * rope_factors = is_swa ? nullptr : model.get_rope_factors(cparams, il);
- const int64_t n_rot_l = hparams.n_rot(il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur_pos", il);
cb(Kcur, "Kcur_pos", il);
@@ -375,9 +371,6 @@ llama_model_step35::graph_mtp::graph_mtp(const llm_graph_params & params)
const uint32_t n_head_l = hparams.n_head(il);
const uint32_t n_head_kv_l = hparams.n_head_kv(il);
- const float freq_base_l = model.get_rope_freq_base(cparams, il);
- const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
-
auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
@@ -441,18 +434,8 @@ llama_model_step35::graph_mtp::graph_mtp(const llm_graph_params & params)
cb(Kcur, "mtp_Kcur_normed", il);
}
- const bool is_swa = hparams.is_swa(il);
- ggml_tensor * rope_factors = is_swa ? nullptr : model.get_rope_factors(cparams, il);
- const int64_t n_rot_l = hparams.n_rot(il);
-
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, rope_factors,
- n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, rope_factors,
- n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "mtp_Qcur_pos", il);
cb(Kcur, "mtp_Kcur_pos", il);
diff --git a/src/models/talkie.cpp b/src/models/talkie.cpp
index b1e4b7c87..5bf18f066 100644
--- a/src/models/talkie.cpp
+++ b/src/models/talkie.cpp
@@ -73,15 +73,8 @@ llama_model_talkie::graph::graph(const llm_graph_params & params) : llm_graph_co
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
// reference applies qknorm after rope
Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);
diff --git a/src/models/xing4-0.cpp b/src/models/xing4-0.cpp
index 677c3d90f..c95acd6b8 100644
--- a/src/models/xing4-0.cpp
+++ b/src/models/xing4-0.cpp
@@ -318,14 +318,10 @@ llama_model_xing4_0::graph_mtp::graph_mtp(const llama_model & model, const llm_g
const float kq_scale =
1.0f * mscale * mscale / sqrtf(float(n_embd_head_k_mla));
- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ q_pe = build_rope(q_pe, inp_pos, il);
cb(q_pe, "mtp_q_pe_rope", il);
- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ k_pe = build_rope(k_pe, inp_pos, il);
cb(k_pe, "mtp_k_pe_rope", il);
q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
@@ -686,12 +682,10 @@ llama_model_xing4_0::graph::graph(const llama_model & model, const llm_graph_par
ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
cb(k_pe, "k_pe_in", il);
- q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ q_pe = build_rope(q_pe, inp_pos, il);
cb(q_pe, "q_pe", il);
- k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow);
+ k_pe = build_rope(k_pe, inp_pos, il);
cb(k_pe, "k_pe", il);
kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
diff --git a/src/models/xverse.cpp b/src/models/xverse.cpp
index 14e12518b..e0710bd9c 100644
--- a/src/models/xverse.cpp
+++ b/src/models/xverse.cpp
@@ -69,17 +69,8 @@ llama_model_xverse::graph::graph(const llm_graph_params & params) : llm_graph_co
auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,
n_embd_head, n_head, n_head_kv, il);
- Qcur = ggml_rope_ext(
- ctx0, Qcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
-
- Kcur = ggml_rope_ext(
- ctx0, Kcur, inp_pos, nullptr,
- n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
- ext_factor, attn_factor, beta_fast, beta_slow
- );
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
cb(Qcur, "Qcur", il);
cb(Kcur, "Kcur", il);
diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv
index 46aec1076..6d7599e4d 100644
--- a/tests/fusion/MTL.csv
+++ b/tests/fusion/MTL.csv
@@ -336,4 +336,5 @@ talkie ,0 ,any ,ADD+ADD , 2
xing4_0 ,0 ,any ,ADD+ADD+ADD , 1
xing4_0 ,0 ,any ,MUL+ADD , 2
xing4_0 ,0 ,any ,RMS_NORM+MUL , 9
+xing4_0 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1
xverse ,0 ,any ,RMS_NORM+MUL , 5
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 39255c341..64637ed8d 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -95,6 +95,14 @@ static void fill_ssm_a(std::vector<float> & dst, const char * name, const tensor
}
}
+// RoPE factors divide the frequencies, a value close to 0 gives angles too large for F32 sin/cos to stay accurate
+static void fill_rope_factors(std::vector<float> & dst, const char * name, const tensor_data_params & params) {
+ fill_normal(dst, name, params);
+ for (float & v : dst) {
+ v = 1.0f + fabsf(v);
+ }
+}
+
// a norm preserving residual gate holds alpha^2 + beta^2 == 1, and the conversion resolves both from one raw weight
// the alpha and the beta of a block share that weight, so the key drops the suffix and keeps the block
static void fill_resid_gate(std::vector<float> & dst, const char * name, const tensor_data_params & params) {
@@ -116,9 +124,11 @@ static void fill_exp_probs_b(std::vector<float> & dst, const char *, const tenso
// a tensor whose values hold a property of their own, matched on a substring of its name, first match wins
static const struct { const char * match; tensor_filler fill; } TENSOR_FILLERS[] = {
- { "ssm_a", fill_ssm_a },
- { "_resid_", fill_resid_gate },
- { "exp_probs_b", fill_exp_probs_b },
+ { "ssm_a", fill_ssm_a },
+ { "_resid_", fill_resid_gate },
+ { "exp_probs_b", fill_exp_probs_b },
+ { "rope_freqs", fill_rope_factors },
+ { "rope_factors", fill_rope_factors },
};
static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {
@@ -157,6 +167,7 @@ static void usage(char ** argv) {
LOG(" -o, --out <dir> Save generated test models to <dir> instead of running backend tests\n");
LOG(" -v <N> Set log verbosity level\n");
LOG(" -b, --backend <backend> Run only on the given backend device\n");
+ LOG(" --kv-shift Check K-shift against a decode at the shifted positions (CPU only)\n");
LOG(" -h, --help Show this help message\n\n");
LOG("Examples:\n");
LOG(" %s\n", argv[0]);
@@ -227,7 +238,7 @@ struct kv_writer {
}
};
-static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
+static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const uint32_t n_layer_min = 0) {
gguf_context_ptr ret(gguf_init_empty());
kv_writer ms(arch, ret.get());
const uint32_t n_ctx = 256;
@@ -319,6 +330,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
n_head_kv = 2;
n_layer = 4;
}
+ n_layer = std::max(n_layer, n_layer_min);
const uint32_t n_embd_head = n_embd / n_head;
ms.add_kv(LLM_KV_GENERAL_ARCHITECTURE, llama_arch_name(arch));
@@ -748,8 +760,14 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_KDA_SAFE_GATE, true);
ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
if (arch == LLM_ARCH_BAILINGMOE3) {
- ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector<float>({0.0f, 4.0f}));
- ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector<float>({0.0f, 5.0f}));
+ std::vector<float> clamp_exp;
+ std::vector<float> clamp_shexp;
+ for (uint32_t il = 0; il < n_layer; il++) {
+ clamp_exp .push_back(il % 2 ? 4.0f : 0.0f);
+ clamp_shexp.push_back(il % 2 ? 5.0f : 0.0f);
+ }
+ ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, clamp_exp);
+ ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, clamp_shexp);
}
ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head);
ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3));
@@ -1436,6 +1454,319 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con
return all_ok ? 0 : 1;
}
+struct kv_shift_move {
+ llama_pos p0;
+ llama_pos p1;
+ llama_pos delta;
+};
+
+static void decode_range(llama_context * lctx, const llama_token * tokens, uint32_t n_tokens, llama_pos pos0, llama_seq_id seq_id, bool output) {
+ common_batch batch(lctx);
+ for (uint32_t i = 0; i < n_tokens; i++) {
+ batch.add(tokens[i], pos0 + i, seq_id, output || i == n_tokens - 1);
+ }
+ if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
+ throw std::runtime_error("failed to decode batch");
+ }
+}
+
+// the raw attention scores of each layer are linear in K, unlike the logits they do not depend on how sharp the attention is
+struct kq_capture {
+ bool active = false;
+ std::vector<std::pair<std::string, std::vector<float>>> data;
+};
+
+// the scores are compared cell by cell, so the cells of the probe must match: prefix and probe fit in the SWA window,
+// otherwise the probe can reuse a masked SWA cell in one run and not in the other
+static bool kq_capture_cb(struct ggml_tensor * t, bool ask, void * user_data) {
+ kq_capture * cap = (kq_capture *) user_data;
+ const char * name = ggml_get_name(t);
+ const bool want = cap->active && t->type == GGML_TYPE_F32 && ggml_is_contiguous(t) &&
+ (strncmp(name, "kq-", 3) == 0 || strncmp(name, "indexer_kq-", 11) == 0 || strncmp(name, "indexer_score-", 14) == 0);
+ if (ask) {
+ return want;
+ }
+ if (want) {
+ std::vector<float> buf(ggml_nelements(t));
+ ggml_backend_tensor_get(t, buf.data(), 0, ggml_nbytes(t));
+ auto it = std::find_if(cap->data.begin(), cap->data.end(), [&](const auto & e) { return e.first == name; });
+ if (it == cap->data.end()) {
+ cap->data.emplace_back(name, std::move(buf));
+ } else {
+ it->second = std::move(buf);
+ }
+ }
+ return true;
+}
+
+struct kq_diff {
+ double worst = 0.0;
+ std::string first_bad; // first tensor above the threshold, later layers can inherit its error through Q
+ std::string bad; // all tensors above the threshold, with their nmse
+};
+
+static kq_diff compare_kq(const std::vector<std::pair<std::string, std::vector<float>>> & ref,
+ const std::vector<std::pair<std::string, std::vector<float>>> & cur, double thold) {
+ if (ref.size() != cur.size()) {
+ throw std::runtime_error("kq tensor count mismatch");
+ }
+ kq_diff res;
+ for (size_t i = 0; i < ref.size(); i++) {
+ if (ref[i].first != cur[i].first || ref[i].second.size() != cur[i].second.size()) {
+ throw std::runtime_error("kq tensor mismatch: " + ref[i].first);
+ }
+ double mse_a_b = 0.0;
+ double mse_a_0 = 0.0;
+ for (size_t j = 0; j < ref[i].second.size(); j++) {
+ const double a = ref[i].second[j];
+ const double b = cur[i].second[j];
+ // masked rows may hold -inf in both
+ if (!std::isfinite(a) && a == b) {
+ continue;
+ }
+ mse_a_b += (a - b)*(a - b);
+ mse_a_0 += a*a;
+ }
+ const double val = mse_a_b == 0.0 ? 0.0 : mse_a_0 == 0.0 ? INFINITY : mse_a_b/mse_a_0;
+ res.worst = std::max(res.worst, std::isnan(val) ? INFINITY : val);
+ if (!(val <= thold)) {
+ if (res.first_bad.empty()) {
+ res.first_bad = ref[i].first;
+ }
+ char buf[64];
+ snprintf(buf, sizeof(buf), " %s=%.1e", ref[i].first.c_str(), val);
+ res.bad += buf;
+ }
+ }
+ return res;
+}
+
+// decodes the prefix at pos_prefix, applies each step of moves with its own K-shift and returns the logits of the probe at pos_probe
+static std::vector<float> get_logits_kv_shift(
+ llama_model * model, llama_context * lctx, kq_capture & cap, const std::vector<llama_token> & prefix, const std::vector<llama_token> & probe,
+ llama_pos pos_prefix, const std::vector<std::vector<kv_shift_move>> & steps, llama_pos pos_probe) {
+ const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
+ llama_memory_t mem = llama_get_memory(lctx);
+
+ cap.data.clear();
+ llama_memory_clear(mem, true);
+ decode_range(lctx, prefix.data(), prefix.size(), pos_prefix, 0, false);
+
+ for (size_t i = 0; i < steps.size(); i++) {
+ for (const kv_shift_move & mv : steps[i]) {
+ llama_memory_seq_add(mem, 0, mv.p0, mv.p1, mv.delta);
+ }
+ // there is no public API to apply a pending shift, a decode of a throwaway token in seq 1 does it
+ if (i + 1 < steps.size()) {
+ decode_range(lctx, probe.data(), 1, 0, 1, false);
+ llama_memory_seq_rm(mem, 1, -1, -1);
+ }
+ }
+
+ cap.active = true;
+ decode_range(lctx, probe.data(), probe.size(), pos_probe, 0, true);
+ cap.active = false;
+
+ std::vector<float> ret;
+ ret.reserve(probe.size()*n_vocab);
+ for (uint32_t i = 0; i < probe.size(); i++) {
+ const float * logits_ith = llama_get_logits_ith(lctx, i);
+ ret.insert(ret.end(), logits_ith, logits_ith + n_vocab);
+ }
+ return ret;
+}
+
+// a K-shift is correct if decoding at [d, d+n) and shifting by -d gives the same probe logits as decoding at [0, n)
+static int test_kv_shift(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity) {
+ struct user_data_t {
+ struct {
+ ggml_log_callback callback;
+ void * user_data;
+ } log_old;
+
+ int verbosity;
+
+ user_data_t(int verbosity) : verbosity(verbosity) {
+ llama_log_get(&log_old.callback, &log_old.user_data);
+ }
+ };
+ user_data_t ud(verbosity);
+
+ llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
+ const user_data_t * ud = (const user_data_t *) user_data;
+ int verbosity = common_log_get_verbosity(level);
+ if (verbosity <= ud->verbosity) {
+ ud->log_old.callback(level, text, ud->log_old.user_data);
+ }
+ }, &ud);
+
+ // n_prefix + n_probe stays below the fixture SWA window (n_ctx/8 = 32)
+ const uint32_t n_prefix = 20;
+ const uint32_t n_probe = 8;
+ const llama_pos d = 64;
+ const llama_pos gap = 4;
+
+ const std::vector<llama_token> tokens = get_tokens(n_prefix + n_probe, 128, seed);
+ const std::vector<llama_token> prefix(tokens.begin(), tokens.begin() + n_prefix);
+ const std::vector<llama_token> probe (tokens.begin() + n_prefix, tokens.end());
+
+ const std::vector<std::vector<kv_shift_move>> steps_shift = { { { d, d + (llama_pos) n_prefix, -d } } };
+ const std::vector<std::vector<kv_shift_move>> steps_shift2 = { { { d, d + (llama_pos) n_prefix, -d/2 } }, { { d/2, d/2 + (llama_pos) n_prefix, -d/2 } } };
+ // control: the second half of the prefix ends up gap positions too far, which tells if the probe is sensitive to K positions at all
+ const std::vector<std::vector<kv_shift_move>> steps_ctrl = { {
+ { d, d + (llama_pos) n_prefix/2, -d },
+ { d + (llama_pos) n_prefix/2, d + (llama_pos) n_prefix, -d + gap },
+ } };
+
+ size_t max_arch_name_length = 0;
+ for (const llm_arch & arch : llama_arch_all()) {
+ max_arch_name_length = std::max(max_arch_name_length, strlen(llama_arch_name(arch)));
+ }
+ const std::string template_row = "|%" + std::to_string(max_arch_name_length) + "s|%6s|%10s|%10s|%10s|%10s|%s\n";
+
+ size_t n_tests = 0;
+ size_t n_failed = 0;
+ common_log_flush(common_log_main());
+ LOG(template_row.c_str(), "Model arch.", "Config", "Shift kq", "Shift2 kq", "Logits", "Control kq", "Status");
+
+ for (const llm_arch & arch : llama_arch_all()) {
+ if (arch == LLM_ARCH_UNKNOWN || !arch_matches(arch_filter, arch) || !arch_supported(arch)) {
+ continue;
+ }
+ if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
+ continue;
+ }
+ const bool encode = arch == LLM_ARCH_T5 || arch == LLM_ARCH_DREAM || arch == LLM_ARCH_LLADA || arch == LLM_ARCH_LLADA_MOE || arch == LLM_ARCH_RND1;
+ if (encode) {
+ continue;
+ }
+ for (bool moe : {false, true}) {
+ if (moe && !moe_implemented(arch)) {
+ continue;
+ }
+ if (!moe && moe_mandatory(arch)) {
+ continue;
+ }
+ const char * config_name = moe ? "MoE" : "Dense";
+
+ // the attention scale depends on the absolute position, so the hidden states of the prefix do too
+ const double thold = arch == LLM_ARCH_MISTRAL_LARGE4 ? 1e-4 : 1e-6;
+
+ // at least 4 layers, so that "every 4th layer" patterns (NoPE, full attention) are covered
+ // exaone4 is SWA only with 64 layers, smallthinker always ropes with 4
+ const uint32_t n_layer_min = arch == LLM_ARCH_EXAONE4 ? 64 : arch == LLM_ARCH_SMALLTHINKER ? 8 : 4;
+ gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe, n_layer_min);
+ // different bases, so that a shift using the wrong one is caught
+ gguf_set_val_f32(gguf_ctx.get(), llama_arch_kv_name(arch, LLM_KV_ROPE_FREQ_BASE).c_str(), 20000.0f);
+ gguf_set_val_f32(gguf_ctx.get(), llama_arch_kv_name(arch, LLM_KV_ROPE_FREQ_BASE_SWA).c_str(), 10000.0f);
+ if (arch == LLM_ARCH_BAILINGMOE3) {
+ GGML_ASSERT(gguf_remove_key(gguf_ctx.get(), "bailingmoe3.kda.safe_gate") >= 0);
+ }
+
+ llama_model_ptr model;
+ try {
+ model = get_model(gguf_ctx.get(), nullptr, seed, stdev, {});
+ } catch (const std::exception & err) {
+ LOG(template_row.c_str(), llama_arch_name(arch), config_name, "", "", "", "", "\033[1;31mERROR\033[0m (model load)");
+ n_tests++;
+ n_failed++;
+ continue;
+ }
+ // absolute position embeddings are not translation invariant, the comparison does not hold
+ if (llama_model_rope_type(model.get()) == LLAMA_ROPE_TYPE_NONE) {
+ LOG(template_row.c_str(), llama_arch_name(arch), config_name, "", "", "", "", "\033[1;33mSKIP\033[0m (no rope)");
+ continue;
+ }
+
+ LOG("|%*s|%6s|", (int) max_arch_name_length, llama_arch_name(arch), config_name);
+ fflush(stdout);
+
+ llama_context_params ctx_params = llama_context_default_params();
+ ctx_params.n_ctx = 0;
+ ctx_params.n_threads = 4;
+ ctx_params.n_threads_batch = 4;
+ ctx_params.n_seq_max = 2;
+ ctx_params.kv_unified = true;
+ ctx_params.type_k = GGML_TYPE_F32;
+ ctx_params.type_v = GGML_TYPE_F32;
+ ctx_params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED;
+ // differs from the SWA default, so that a shift using the wrong scale is caught
+ ctx_params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR;
+ ctx_params.rope_freq_scale = 0.5f;
+ kq_capture cap;
+ ctx_params.cb_eval = kq_capture_cb;
+ ctx_params.cb_eval_user_data = ∩
+
+ ud.verbosity = LOG_LEVEL_OUTPUT;
+ llama_context_ptr lctx(llama_init_from_model(model.get(), ctx_params));
+ ud.verbosity = verbosity;
+ if (!lctx) {
+ LOG("%10s|%10s|%10s|%10s|\033[1;33mSKIP\033[0m (context)\n", "", "", "", "");
+ continue;
+ }
+ llama_memory_t mem = llama_get_memory(lctx.get());
+ if (mem == nullptr || !llama_memory_can_shift(mem)) {
+ LOG("%10s|%10s|%10s|%10s|\033[1;33mSKIP\033[0m (no shift)\n", "", "", "", "");
+ continue;
+ }
+
+ std::string status;
+ char shift_str[16] = "";
+ char shift2_str[16] = "";
+ char logits_str[16] = "";
+ char ctrl_str[16] = "";
+ n_tests++;
+ try {
+ const std::vector<float> logits_ref = get_logits_kv_shift(model.get(), lctx.get(), cap, prefix, probe, 0, {}, n_prefix);
+ const auto kq_ref = cap.data;
+ if (verbosity >= LOG_LEVEL_INFO) {
+ for (const auto & e : kq_ref) {
+ LOG("[%s: %zu] ", e.first.c_str(), e.second.size());
+ }
+ }
+ const std::vector<float> logits_shift = get_logits_kv_shift(model.get(), lctx.get(), cap, prefix, probe, d, steps_shift, n_prefix);
+ const kq_diff diff_shift = compare_kq(kq_ref, cap.data, thold);
+ get_logits_kv_shift(model.get(), lctx.get(), cap, prefix, probe, d, steps_shift2, n_prefix);
+ const kq_diff diff_shift2 = compare_kq(kq_ref, cap.data, thold);
+ const std::vector<float> logits_ctrl = get_logits_kv_shift(model.get(), lctx.get(), cap, prefix, probe, d, steps_ctrl, n_prefix + gap);
+ const kq_diff diff_ctrl = compare_kq(kq_ref, cap.data, thold);
+
+ const double nmse_logits = nmse(logits_ref, logits_shift);
+ snprintf(shift_str, sizeof(shift_str), "%.2e", diff_shift.worst);
+ snprintf(shift2_str, sizeof(shift2_str), "%.2e", diff_shift2.worst);
+ snprintf(logits_str, sizeof(logits_str), "%.2e", nmse_logits);
+ snprintf(ctrl_str, sizeof(ctrl_str), "%.2e", diff_ctrl.worst);
+
+ if (kq_ref.empty()) {
+ status = "\033[1;33mSKIP\033[0m (no kq captured)";
+ } else if (!diff_shift.first_bad.empty() || !diff_shift2.first_bad.empty()) {
+ n_failed++;
+ status = "\033[1;31mFAIL\033[0m" + (diff_shift.bad.empty() ? " x2:" + diff_shift2.bad : diff_shift.bad);
+ } else if (!(diff_ctrl.worst > 10*thold)) {
+ status = "\033[1;33mOK\033[0m (low sensitivity)";
+ } else {
+ status = "\033[1;32mOK\033[0m";
+ }
+ } catch (const std::exception & err) {
+ n_failed++;
+ status = std::string("\033[1;31mERROR\033[0m (") + err.what() + ")";
+ }
+ LOG("%10s|%10s|%10s|%10s|%s\n", shift_str, shift2_str, logits_str, ctrl_str, status.c_str());
+ }
+ }
+
+ if (n_tests == 0) {
+ LOG("Summary: no tests executed\n");
+ } else if (n_failed == 0) {
+ LOG("Summary: all %zu test(s) passed\n", n_tests);
+ } else {
+ LOG("Summary: %zu test(s) executed, %zu failed\n", n_tests, n_failed);
+ }
+
+ llama_log_set(ud.log_old.callback, ud.log_old.user_data);
+ return n_failed == 0 ? 0 : 1;
+}
+
int main(int argc, char ** argv) {
// init the logger at max verbosity. filter with a custom callback respecting the user-configure verbosity
common_log_set_verbosity_thold(LOG_LEVEL_DEBUG);
@@ -1449,6 +1780,7 @@ int main(int argc, char ** argv) {
float stdev = 0.1f;
std::string out;
const char * target_backend = nullptr;
+ bool kv_shift = false;
int verbosity = LOG_LEVEL_ERROR;
@@ -1516,6 +1848,8 @@ int main(int argc, char ** argv) {
usage(argv);
return 1;
}
+ } else if (strcmp(argv[i], "--kv-shift") == 0) {
+ kv_shift = true;
} else {
LOG_ERR("%s: unknown argument: %s\n", __func__, argv[i]);
usage(argv);
@@ -1534,6 +1868,9 @@ int main(int argc, char ** argv) {
if (!out.empty()) {
return save_models(arch_filter, seed, stdev, verbosity, out);
}
+ if (kv_shift) {
+ return test_kv_shift(arch_filter, seed, stdev, verbosity);
+ }
return test_backends(arch_filter, seed, stdev, verbosity, target_backend);
} catch (const std::exception & err) {
fprintf(stderr, "encountered runtime error: %s\n", err.what());