Commit fee39dd92 for llama.cpp
commit fee39dd92673ba0c08c8da96040ce53368b35188
Author: shaofeiqi <shaoqi@qti.qualcomm.com>
Date: Wed Sep 23 10:37:04 2026 -0700
opencl: add A8 Q6_K non-MoE dp4a binary kernel (#29057)
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 91cd1a0ed..239821894 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -1259,6 +1259,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_gemm_noshuffle_q6_K_f32;
cl_kernel kernel_gemm_noshuffle_q6_K_f32_cok;
cl_kernel kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin;
+ cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin;
cl_kernel kernel_gemv_noshuffle_q6_k_f32_32b_trans;
cl_kernel kernel_gemv_noshuffle_q5_k_f32;
cl_kernel kernel_gemv_noshuffle_q5_k_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP)
@@ -4407,6 +4408,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans = nullptr;
backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin = nullptr;
+ backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin = nullptr;
if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E) {
{
std::string opts = std::string("-cl-std=") + opencl_c_std +
@@ -4439,6 +4441,17 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
CL_CHECK(clReleaseProgram(bin_prog));
GGML_LOG_CONT(".");
}
+
+ kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8", &bin_size);
+ if (kernel_bin && bin_size > 0) {
+ cl_program bin_prog =
+ build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size);
+
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin =
+ clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8", &err), err));
+ CL_CHECK(clReleaseProgram(bin_prog));
+ GGML_LOG_CONT(".");
+ }
}
}
@@ -21919,6 +21932,74 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno_ila(ggml_backend_t backend, const gg
const int gemm_tile_n = 64;
int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n;
+ static const char * q6_k_bin_dp4a_env = getenv("GGML_OPENCL_Q6_K_BIN_DP4A");
+ bool q6_k_bin_dp4a_on = q6_k_bin_dp4a_env
+ ? (atoi(q6_k_bin_dp4a_env) != 0)
+ : true;
+ // dot prod has to be available
+ q6_k_bin_dp4a_on = backend_ctx->has_integer_dot && q6_k_bin_dp4a_on;
+
+ if (q6_k_bin_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin) {
+ const int dp4a_N_pad = CEIL_DIV(N, 32) * 32;
+ const size_t n_blocks = (size_t)dp4a_N_pad * (K / 32);
+
+ backend_ctx->prealloc_moe_qa.allocate(context, (size_t)dp4a_N_pad * K * sizeof(cl_char));
+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half));
+
+ cl_mem b_sub = nullptr;
+ region.origin = offset1;
+ region.size = (size_t)K * N * sizeof(float);
+ CL_CHECK((b_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+
+ cl_int tb = (cl_int)((size_t)N * (K / 32));
+ cl_kernel qk = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub));
+ CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb));
+ size_t q_local[1] = { 64 };
+ size_t q_global[1] = { (size_t)CEIL_DIV(tb, 64) * 64 };
+ backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
+
+ cl_mem d_sub = nullptr;
+ cl_mem d_img = nullptr;
+ region.origin = offsetd;
+ region.size = (size_t)M * N * sizeof(float);
+ CL_CHECK((d_sub = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+
+ img_fmt = { CL_R, CL_FLOAT };
+ memset(&img_desc, 0, sizeof(img_desc));
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = (size_t)M * N;
+ img_desc.buffer = d_sub;
+ CL_CHECK((d_img = clCreateImage(context, CL_MEM_WRITE_ONLY, &img_fmt, &img_desc, NULL, &err), err));
+
+ kernel = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin;
+
+ cl_uint k_arg = 0;
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->ql_img));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->qh));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->s));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->d));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &d_img));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &K));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &M));
+ CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N));
+
+ size_t local_work_size[3] = { 64, 1, 1 };
+ size_t global_work_size[3] = { 64, (size_t)(M / 64), (size_t)(dp4a_N_pad / 32) };
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+
+ CL_CHECK(clReleaseMemObject(b_sub));
+ CL_CHECK(clReleaseMemObject(d_img));
+ CL_CHECK(clReleaseMemObject(d_sub));
+ return;
+ }
+
cl_mem b_sub_buf = nullptr;
cl_mem b_padded = nullptr;
cl_mem b_buf = nullptr;