Commit 9e2a04a43 for llama.cpp
commit 9e2a04a439edc49f08c5acdafefae19dfa25a658
Author: shaofeiqi <shaoqi@qti.qualcomm.com>
Date: Wed Nov 4 10:45:30 2026 -0800
opencl: improve decode perf with FA (#31633)
* opencl: use more workgroups for q4_K gate up + GLU
Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
* opencl: handle FA for GQA 2 and 6 with head size 256
Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
* opencl: handle FA for GQA 16 with head size of 128, 512
Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
* opencl: handle FA for GQA 2 with head size 512
Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
---------
Co-authored-by: Li He <lih@qti.qualcomm.com>
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 22cf6f67d..a2620135d 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -7835,7 +7835,8 @@ static void ggml_opencl_ensure_fa_f32_f16_mq_decode(ggml_backend_opencl_context
}
const bool cluster = dk == 64 || dk == 128;
- const int head_sub = cluster ? 2 : (gqa == 8 ? (dk == 512 ? 4 : 2) : 1);
+ const int head_sub = gqa == 16 ? (cluster ? 4 : 8) :
+ cluster ? 2 : (gqa == 8 ? (dk == 512 ? 4 : 2) : 1);
const int nsg_max = dk == 64 ? 1 : (dk == 128 || (gqa == 8 && dk == 256) ? 2 : 4);
const char * kernel_name = cluster ? "flash_attn_f32_f16_q1_vec_mq_split_c8" : "flash_attn_f32_f16_q1_vec_mq_split";
const std::string src = ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16);
@@ -7843,7 +7844,7 @@ static void ggml_opencl_ensure_fa_f32_f16_mq_decode(ggml_backend_opencl_context
" -D FA_MQ_ONLY -D MQ_GQA=" + std::to_string(gqa / head_sub) +
" -D FA_HEAD_SUB=" + std::to_string(head_sub) +
(cluster ? " -D MQ_NSG=" + std::to_string(nsg_max) + " -D FA_CL_C=16" : " -D FA_MQ_SPLIT_ONLY") +
- (dk == 64 ? " -D FA_CL_MHRED -D FA_CL_MASK_BCAST" : "") +
+ (dk == 64 ? " -D FA_CL_MHRED -D FA_CL_MASK_BCAST" : (cluster && gqa == 16 ? " -D FA_CL_MHRED" : "")) +
(gqa == 8 && dk == 256 ? " -D FA_Q1_Q_REG" : "");
for (int nsg = nsg_max; nsg >= 1; nsg /= 2) {
const size_t wg = 64 * nsg;
@@ -10839,7 +10840,8 @@ static void ggml_cl_mul_mat_q4_k_glu_fused(ggml_backend_t backend, ggml_tensor *
// byte-identical to the per-op path. Keep the maxwg query as a further floor
// for any driver that reports < 512.
size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel);
- size_t nsg_y = 8;
+ // X2E: 4 is faster than 8 on every q4_K FFN shape measured there
+ size_t nsg_y = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E ? 4 : 8;
while (nsg_y > 1 && 64 * nsg_y > maxwg) { nsg_y >>= 1; }
size_t local_work_size[3] = { 64, nsg_y, 1 };
size_t global_work_size[3] = { (size_t)CEIL_DIV(M / 2, 64) * 64, nsg_y, 1 };
@@ -12148,9 +12150,13 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
}
if (q->ne[1] == 1) {
const char * decode_env = getenv("GGML_OPENCL_FA_DK512_DECODE");
+ const int64_t gqa = k->ne[2] > 0 ? q->ne[2] / k->ne[2] : 0;
+ // GQA 2 and 16 decode on the X2E MQ kernels (see mq_decode_x2e in ggml_cl_flash_attn)
+ const bool gqa_ok = gqa == 4 || gqa == 8 ||
+ (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && (gqa == 2 || gqa == 16));
if ((decode_env && decode_env[0] == '0') ||
backend_ctx->gpu_family != ADRENO || k->ne[2] <= 0 ||
- (q->ne[2] / k->ne[2] != 4 && q->ne[2] / k->ne[2] != 8) || q->ne[2] % k->ne[2] != 0 ||
+ !gqa_ok || q->ne[2] % k->ne[2] != 0 ||
!ggml_opencl_ensure_fa_f32_f16_vec_512(backend_ctx)) {
return false;
}
@@ -22596,11 +22602,18 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co
const std::pair<int, int> dk_dv = {d_head_q, d_head_v};
const int mq_gqa = n_head_kv > 0 ? n_head / n_head_kv : 0;
const std::tuple<int, int, int> mq_decode_key = {d_head_q, d_head_v, mq_gqa};
+ // X2E also takes GQA 2 and 6 at head size 256 (gemma-4 SWA layers, Qwen3.5-27B),
+ // GQA 16 at 512 and 128 (gemma-4-12b global layers, muse-glimmer) and GQA 2 at 512 (gemma-4-E4B MTP drafter)
+ const bool mq_decode_x2e = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E &&
+ ((d_head_q == 256 && (mq_gqa == 2 || mq_gqa == 6)) ||
+ (d_head_q == 512 && (mq_gqa == 2 || mq_gqa == 16)) ||
+ (d_head_q == 128 && mq_gqa == 16 && backend_ctx->has_subgroup_shuffle));
const bool mq_decode_shape = backend_ctx->gpu_family == ADRENO && is_mixed && n_q == 1 &&
d_head_q == d_head_v && n_head_kv > 0 && n_head % n_head_kv == 0 &&
((((d_head_q == 64 && mq_gqa == 8) || (d_head_q == 128 && mq_gqa == 4)) &&
backend_ctx->has_subgroup_shuffle) ||
- ((d_head_q == 256 || d_head_q == 512) && (mq_gqa == 4 || mq_gqa == 8)));
+ ((d_head_q == 256 || d_head_q == 512) && (mq_gqa == 4 || mq_gqa == 8)) ||
+ mq_decode_x2e);
if (mq_decode_shape && n_kv >= 32) {
ggml_opencl_ensure_fa_f32_f16_mq_decode(backend_ctx, d_head_q, d_head_v, mq_gqa);
}