Commit 6900fb5ac for llama.cpp
commit 6900fb5ac8344afd4103b136cfd631ca6707e038
Author: lhez <lih@qti.qualcomm.com>
Date: Wed Nov 4 00:07:14 2026 -0800
opencl: improve q2_0 GEMV for Adreno (#31577)
* opencl: improve q2_0 decoding and small f32 matmuls
Assisted-by: Codex
* opencl: optimize small f32 matmuls on Adreno 8xx
Assisted-by: Codex
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index f0d401d55..22cf6f67d 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -887,6 +887,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_cpy_f32_f32_pack = nullptr;
cl_kernel kernel_cpy_f32_f32_flat = nullptr;
cl_kernel kernel_mul_mat_f32_f32;
+ cl_kernel kernel_mul_mv_f32_f32_r2 = nullptr;
cl_kernel kernel_fwht[2] = {};
size_t fwht_workgroup_size[2] = {};
cl_kernel kernel_mul_mat_f16_f16;
@@ -1027,10 +1028,13 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_mul_mv_q2_0_f32_flat;
cl_kernel kernel_mul_mm_q2_0_f32_l4_lm;
cl_kernel kernel_gemv_noshuffle_q2_0_f32 = nullptr;
+ cl_kernel kernel_gemv_noshuffle_q2_0_f32_wide = nullptr;
+ size_t q2_0_mv_workgroup_size = 512;
+ size_t q2_0_mv_workgroup_size_wide = 512;
cl_kernel kernel_gemm_noshuffle_q2_0_f32 = nullptr;
cl_kernel kernel_gemm_noshuffle_q2_0_q8_1_dp4a = nullptr;
cl_kernel kernel_gemm_noshuffle_q2_0_q8_1_dp4a_narrow = nullptr;
- int q2_0_mv_nsg = 8;
+ int q2_0_mv_nsg = 32;
cl_kernel kernel_mul_mv_q2_0_f32;
cl_kernel kernel_convert_block_iq1_m, kernel_restore_block_iq1_m;
cl_kernel kernel_mul_mv_iq1_m_f32_flat;
@@ -3591,6 +3595,17 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_mul_mat_f32_f32 = clCreateKernel(backend_ctx->program_mul_mv_f32_f32, "kernel_mul_mat_f32_f32", &err), err));
+ if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X) {
+ cl_kernel kernel = clCreateKernel(backend_ctx->program_mul_mv_f32_f32, "kernel_mul_mv_f32_f32_r2", &err);
+ CL_CHECK(err);
+ size_t max_workgroup;
+ CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr));
+ if (max_workgroup >= 64) {
+ backend_ctx->kernel_mul_mv_f32_f32_r2 = kernel;
+ } else {
+ CL_CHECK(clReleaseKernel(kernel));
+ }
+ }
GGML_LOG_CONT(".");
}
@@ -5076,25 +5091,30 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
#endif
if (const char * env = getenv("GGML_OPENCL_Q2_0_MV_NSG")) {
const int nsg = atoi(env);
- if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) {
+ if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8 || nsg == 16 || nsg == 32) {
backend_ctx->q2_0_mv_nsg = nsg;
}
}
- for (;;) {
- const std::string opts = compile_opts + " -DQ2_0_MV_NSG=" + std::to_string(backend_ctx->q2_0_mv_nsg);
+ for (int wide = 0; wide < 2; ++wide) {
+ const std::string opts = compile_opts + " -DQ2_0_MV_WIDE=" + std::to_string(wide);
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts);
cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q2_0_f32", &err);
CL_CHECK(err);
CL_CHECK(clReleaseProgram(prog));
size_t max_workgroup;
CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr));
- if (64*(size_t)backend_ctx->q2_0_mv_nsg <= max_workgroup) {
+ size_t workgroup = 512;
+ while (workgroup > max_workgroup) {
+ workgroup /= 2;
+ }
+ GGML_ASSERT(workgroup >= 64);
+ if (wide) {
+ backend_ctx->kernel_gemv_noshuffle_q2_0_f32_wide = kernel;
+ backend_ctx->q2_0_mv_workgroup_size_wide = workgroup;
+ } else {
backend_ctx->kernel_gemv_noshuffle_q2_0_f32 = kernel;
- break;
+ backend_ctx->q2_0_mv_workgroup_size = workgroup;
}
- CL_CHECK(clReleaseKernel(kernel));
- GGML_ASSERT(backend_ctx->q2_0_mv_nsg > 1);
- backend_ctx->q2_0_mv_nsg /= 2;
}
GGML_LOG_CONT(".");
}
@@ -24248,7 +24268,15 @@ static void ggml_cl_mul_mat_q2_0_f32_adreno(ggml_backend_t backend, const ggml_t
const int k = src0->ne[0];
if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
- cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_q2_0_f32;
+ const bool wide = m >= 16384 && m % 8 == 0;
+ cl_kernel gemv = wide ? backend_ctx->kernel_gemv_noshuffle_q2_0_f32_wide : backend_ctx->kernel_gemv_noshuffle_q2_0_f32;
+ const size_t workgroup = wide ? backend_ctx->q2_0_mv_workgroup_size_wide : backend_ctx->q2_0_mv_workgroup_size;
+ const size_t rows = wide ? 8 : 4;
+ size_t nsg = std::min((size_t)backend_ctx->q2_0_mv_nsg, wide ? size_t(8) : m >= 4096 ? size_t(16) : size_t(32));
+ while (nsg > (size_t)k/64 || nsg*16 > workgroup) {
+ nsg /= 2;
+ }
+ const size_t lanes = workgroup/nsg;
const int stride_d = dst->ne[0];
const int stride_b = src1->nb[1]/sizeof(float);
CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs));
@@ -24261,9 +24289,8 @@ static void ggml_cl_mul_mat_q2_0_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clSetKernelArg(gemv, 7, sizeof(int), &m));
CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &stride_b));
CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &stride_d));
- const size_t nsg = backend_ctx->q2_0_mv_nsg;
- size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 };
- size_t local[] = { 64, nsg, 1 };
+ size_t global[] = { (size_t)CEIL_DIV(m, rows*lanes)*lanes, (size_t)n*nsg, 1 };
+ size_t local[] = { lanes, nsg, 1 };
backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst);
return;
}
@@ -30881,9 +30908,15 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
// Override with GGML_OPENCL_A7X_F32_LM_BYPASS=0.
static const char * a7x_f32lm_env = getenv("GGML_OPENCL_A7X_F32_LM_BYPASS");
static const bool a7x_f32lm_bypass = (a7x_f32lm_env == nullptr || a7x_f32lm_env[0] != '0');
+ // The 64-column tile wastes work on small F32 matrices, including block rotations.
+ const bool f32_use_mv = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E ||
+ (backend_ctx->kernel_mul_mv_f32_f32_r2 != nullptr && ne00 <= 512)) &&
+ src0t == GGML_TYPE_F32 && ne00 >= 256 && ne00 <= 1024 && ne00 % 16 == 0 && ne01 <= 512 && ne11 <= 16 &&
+ ne02 == 1 && ne12 == 1 && ne13 == 1 &&
+ ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && (offset0 | offset1) % 16 == 0;
if (src1t == GGML_TYPE_F32 &&
ne00 % 16 == 0 &&
- ne11 > 1 &&
+ ne11 > 1 && !f32_use_mv &&
!(a7x_f32lm_bypass && src0t == GGML_TYPE_F32 && ne11 > 8 &&
backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X)) {
switch(src0t) {
@@ -32072,6 +32105,9 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
GGML_ASSERT(src1t == GGML_TYPE_F32);
kernel = backend_ctx->kernel_mul_mat_f32_f32;
nrows = 4;
+ if (f32_use_mv && ne11 > 1 && backend_ctx->kernel_mul_mv_f32_f32_r2 != nullptr) {
+ kernel = backend_ctx->kernel_mul_mv_f32_f32_r2;
+ }
if (backend_ctx->gpu_family == INTEL) {
nth0 = 32;
@@ -33833,8 +33869,9 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
} else {
int64_t ny = (ne11 + nrows - 1)/nrows;
+ int64_t nx = kernel == backend_ctx->kernel_mul_mv_f32_f32_r2 ? CEIL_DIV(ne01, 2) : ne01;
- size_t global_work_size[] = {(size_t)ne01*nth0, (size_t)ny*nth1, (size_t)ne12*ne13};
+ size_t global_work_size[] = {(size_t)nx*nth0, (size_t)ny*nth1, (size_t)ne12*ne13};
size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1};
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_0_f32.cl
index 27ff7edd3..a46293c24 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_0_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_0_f32.cl
@@ -1,7 +1,19 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#ifndef Q2_0_MV_NSG
-#define Q2_0_MV_NSG 8
+#if Q2_0_MV_WIDE
+#define Q2_0_MV_ROWS 8
+#define Q2_0_MV_LOAD_D vload8
+#define Q2_0_MV_STORE vstore8
+#define Q2_0_MV_FLOAT convert_float8
+typedef float8 q2_0_mv_acc_t;
+typedef uchar8 q2_0_mv_quant_t;
+#else
+#define Q2_0_MV_ROWS 4
+#define Q2_0_MV_LOAD_D vload4
+#define Q2_0_MV_STORE vstore4
+#define Q2_0_MV_FLOAT convert_float4
+typedef float4 q2_0_mv_acc_t;
+typedef uchar4 q2_0_mv_quant_t;
#endif
kernel void kernel_gemv_noshuffle_q2_0_f32(
@@ -19,46 +31,47 @@ kernel void kernel_gemv_noshuffle_q2_0_f32(
src1 = (global const float *)((global const char *)src1 + offset1);
dst = (global float *)((global char *)dst + offsetd);
- const uint m = (uint)ne01;
- const uint mh = m >> 1;
+ const uint mr = (uint)ne01/Q2_0_MV_ROWS;
const uint lid = get_local_id(0);
const uint sgi = get_local_id(1);
+ const uint lanes = get_local_size(0);
+ const uint nsg = get_local_size(1);
const uint col = get_group_id(1);
- const uint j = get_group_id(0)*64u + lid;
- const uint row = j << 1;
+ const uint j = get_group_id(0)*lanes + lid;
global const float * y = src1 + (ulong)col * (uint)ne10;
- float2 sum = 0.0f;
- if (j < mh) {
- global const ushort * qs = (global const ushort *)src0_qs;
- for (uint ib = sgi; ib < (uint)ne00/64; ib += Q2_0_MV_NSG) {
- const float2 d = convert_float2(vload2(j + ib*mh, src0_d));
- float2 acc = 0.0f;
+ q2_0_mv_acc_t sum = 0.0f;
+ if (j < mr) {
+ for (uint ib = sgi; ib < (uint)ne00/64; ib += nsg) {
+ const q2_0_mv_acc_t d = Q2_0_MV_FLOAT(Q2_0_MV_LOAD_D(j + ib*mr, src0_d));
+ q2_0_mv_acc_t acc = 0.0f;
for (uint u = 0; u < 16; ++u) {
const uint group = ib*16 + u;
- const uint q = qs[j + group*mh];
+#if Q2_0_MV_WIDE
+ const uchar8 q = as_uchar8(((global const uint2 *)src0_qs)[j + group*mr]);
+#else
+ const uchar4 q = as_uchar4(((global const uint *)src0_qs)[j + group*mr]);
+#endif
const float4 yv = vload4(group, y);
- const int4 v0 = convert_int4(((uint4)(q) >> (uint4)(0, 2, 4, 6)) & (uint4)(3)) - (int4)(1);
- const int4 v1 = convert_int4(((uint4)(q) >> (uint4)(8, 10, 12, 14)) & (uint4)(3)) - (int4)(1);
- acc.s0 += dot(yv, convert_float4(v0));
- acc.s1 += dot(yv, convert_float4(v1));
+ acc += (Q2_0_MV_FLOAT( q & (q2_0_mv_quant_t)(3)) - 1.0f)*yv.s0;
+ acc += (Q2_0_MV_FLOAT((q >> (q2_0_mv_quant_t)(2)) & (q2_0_mv_quant_t)(3)) - 1.0f)*yv.s1;
+ acc += (Q2_0_MV_FLOAT((q >> (q2_0_mv_quant_t)(4)) & (q2_0_mv_quant_t)(3)) - 1.0f)*yv.s2;
+ acc += (Q2_0_MV_FLOAT( q >> (q2_0_mv_quant_t)(6)) - 1.0f)*yv.s3;
}
sum += d*acc;
}
}
-#if Q2_0_MV_NSG > 1
- local float2 part[Q2_0_MV_NSG][64];
- part[sgi][lid] = sum;
+ local q2_0_mv_acc_t part[512];
+ part[sgi*lanes + lid] = sum;
barrier(CLK_LOCAL_MEM_FENCE);
if (sgi != 0) {
return;
}
- for (uint s = 1; s < Q2_0_MV_NSG; ++s) {
- sum += part[s][lid];
+ for (uint s = 1; s < nsg; ++s) {
+ sum += part[s*lanes + lid];
}
-#endif
- if (j < mh) {
- vstore2(sum, 0, dst + (ulong)col*(uint)ne0 + row);
+ if (j < mr) {
+ Q2_0_MV_STORE(sum, 0, dst + (ulong)col*(uint)ne0 + j*Q2_0_MV_ROWS);
}
}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl
index ec71b8756..1ac250a84 100644
--- a/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl
@@ -116,3 +116,70 @@ kernel void kernel_mul_mat_f32_f32(
}
}
}
+
+// Each group computes two rows and four columns, with 32 lanes per row.
+kernel void kernel_mul_mv_f32_f32_r2(
+ global const char * src0,
+ ulong offset0,
+ global const char * src1,
+ ulong offset1,
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ int ne02,
+ ulong nb00,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne10,
+ int ne11,
+ int ne12,
+ ulong nb10,
+ ulong nb11,
+ ulong nb12,
+ ulong nb13,
+ int ne0,
+ int ne1,
+ int r2,
+ int r3
+) {
+ src0 += offset0;
+ src1 += offset1;
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const int lid = get_local_id(0);
+ const int lane = lid % 32;
+ const int row = get_group_id(0)*2 + lid/32;
+ const int col = get_group_id(1)*4;
+ const int im = get_group_id(2);
+ const int i12 = im % ne12;
+ const int i13 = im / ne12;
+ global const float4 * x = (global const float4 *)(src0 + min(row, ne01 - 1)*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03);
+ global const char * y = src1 + col*nb11 + i12*nb12 + i13*nb13;
+
+ float4 sum = 0.0f;
+ for (int i = lane; i < ne00/4; i += 32) {
+ const float4 xv = x[i];
+ if (col + 0 < ne11) sum.s0 += dot(xv, ((global const float4 *)(y + 0*nb11))[i]);
+ if (col + 1 < ne11) sum.s1 += dot(xv, ((global const float4 *)(y + 1*nb11))[i]);
+ if (col + 2 < ne11) sum.s2 += dot(xv, ((global const float4 *)(y + 2*nb11))[i]);
+ if (col + 3 < ne11) sum.s3 += dot(xv, ((global const float4 *)(y + 3*nb11))[i]);
+ }
+
+ local float4 part[64];
+ part[lid] = sum;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ for (int s = 16; s > 0; s /= 2) {
+ if (lane < s) {
+ part[lid] += part[lid + s];
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+ if (lane == 0 && row < ne01) {
+ if (col + 0 < ne11) dst[im*ne1*ne0 + (col + 0)*ne0 + row] = part[lid].s0;
+ if (col + 1 < ne11) dst[im*ne1*ne0 + (col + 1)*ne0 + row] = part[lid].s1;
+ if (col + 2 < ne11) dst[im*ne1*ne0 + (col + 2)*ne0 + row] = part[lid].s2;
+ if (col + 3 < ne11) dst[im*ne1*ne0 + (col + 3)*ne0 + row] = part[lid].s3;
+ }
+}