Commit 1b29621ba for llama.cpp
commit 1b29621bacc6a446bd523fc790316a24a06e886f
Author: lhez <lih@qti.qualcomm.com>
Date: Mon Oct 19 12:30:39 2026 -0700
opencl: add cooperative-k dp4a gemm for q4_0, q4_k and q6_k (#30741)
* opencl: add cooperative-k dp4a GEMM for q4_0
* opencl: add cooperative-k dp4a GEMM for q4_k
* opencl: add cooperative-k dp4a GEMM for q6_k
* opencl: fix alignment
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt
index 5ba63dade..84087162d 100644
--- a/ggml/src/ggml-opencl/CMakeLists.txt
+++ b/ggml/src/ggml-opencl/CMakeLists.txt
@@ -140,6 +140,9 @@ set(GGML_OPENCL_KERNELS
gemm_noshuffle_q5_0_q8_1_dp4a
gemm_noshuffle_iq4_nl_q8_1_dp4a
gemm_noshuffle_q4_0_q8_1_dp4a
+ gemm_coopk_q4_0_q8_1_dp4a
+ gemm_coopk_q4_k_q8_1_dp4a
+ gemm_coopk_q6_k_q8_1_dp4a
quant_a_q8_1
gemv_moe_q4_k_f32_ns
gemm_moe_q5_k_f32_ns
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 96461bcd9..ad65de00a 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -1296,6 +1296,22 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_gemm_noshuffle_iq4_nl_f32;
cl_kernel kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = nullptr; // dp4a (int8) dense IQ4_NL prefill GEMM
cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q4_0 prefill GEMM
+ cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c4 = nullptr;
+ cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c2 = nullptr;
+ int nsg_coopk_dp4a_q4_0 = 8;
+ int rows_coopk_dp4a_q4_0 = 4;
+ cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_c4 = nullptr;
+ cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_c2 = nullptr;
+ int nsg_coopk_dp4a_q4_k = 8;
+ int rows_coopk_dp4a_q4_k = 4;
+ cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4 = nullptr;
+ cl_kernel kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2 = nullptr;
+ int nsg_coopk_dp4a_q4_k_bin_c4 = 8;
+ int nsg_coopk_dp4a_q4_k_bin_c2 = 8;
+ cl_kernel kernel_gemm_coopk_q6_k_q8_1_dp4a_c4 = nullptr;
+ cl_kernel kernel_gemm_coopk_q6_k_q8_1_dp4a_c2 = nullptr;
+ int nsg_coopk_dp4a_q6_k = 8;
+ int rows_coopk_dp4a_q6_k = 4;
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
void free() {
@@ -1492,6 +1508,180 @@ static bool use_adreno_bin_kernels(ggml_backend_opencl_context * backend_ctx) {
#endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS
}
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+static bool ggml_cl_coopk_dp4a_enabled(const ggml_backend_opencl_context * backend_ctx) {
+ static const char * const env = getenv("GGML_OPENCL_COOPK_DP4A");
+ if (env && *env && atoi(env) == 0) {
+ return false;
+ }
+ if (!backend_ctx->has_integer_dot) {
+ return false;
+ }
+ if (backend_ctx->gpu_family == GPU_FAMILY::ADRENO) {
+ if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::ADRENO_UNKNOWN ||
+ backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X ||
+ backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X) {
+ return false;
+ }
+ if (backend_ctx->adreno_cl_compiler_version.type == ADRENO_CL_COMPILER_TYPE::E17) {
+ return false;
+ }
+ }
+ return true;
+}
+
+static bool ggml_cl_coopk_dp4a_narrow_on(const ggml_backend_opencl_context * backend_ctx,
+ const ggml_tensor * op, int rows) {
+ if (!ggml_cl_coopk_dp4a_enabled(backend_ctx)) {
+ return false;
+ }
+ // Each lane handles rows output rows; subgroups split K for the same rows.
+ // Full row groups and 32-value K blocks avoid tail checks in the kernels.
+ return op->ne[1] >= 2 && op->ne[1] <= 4
+ && rows > 0 && (op->src[0]->ne[1] % (64 * rows)) == 0
+ && (op->src[0]->ne[0] % 32) == 0;
+}
+
+// COOPK_NSG sizes the local reduction buffer, so a smaller workgroup requires a rebuild.
+static cl_program ggml_cl_build_program_coopk(ggml_backend_opencl_context * backend_ctx,
+ const char * kernel_src, const char * kernel_name,
+ const std::string & opts_base, int & nsg) {
+ while (nsg >= 2) {
+ const std::string opts = opts_base + " -DCOOPK_NSG=" + std::to_string(nsg);
+ cl_program program = build_program_from_source(backend_ctx, kernel_src, opts);
+ cl_int err;
+ cl_kernel kernel = clCreateKernel(program, kernel_name, &err);
+ CL_CHECK(err);
+ size_t max_workgroup_size = 0;
+ CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE,
+ sizeof(max_workgroup_size), &max_workgroup_size, NULL));
+ CL_CHECK(clReleaseKernel(kernel));
+ if ((size_t) 64 * nsg <= max_workgroup_size) {
+ return program;
+ }
+ CL_CHECK(clReleaseProgram(program));
+ nsg /= 2;
+ }
+ return nullptr;
+}
+
+static void ggml_cl_coopk_build_narrow(ggml_backend_opencl_context * backend_ctx,
+ const std::string & kernel_src, const char * kernel_name,
+ int rows, int & nsg, cl_kernel & kernel_c4, cl_kernel & kernel_c2) {
+ const std::string base_opts = backend_ctx->kernel_compile_opts
+ + " -DCOOPK_ROWS=" + std::to_string(rows);
+ for (int cols = 4; cols >= 2; cols -= 2) {
+ int nsg_eff = nsg;
+ cl_program program = ggml_cl_build_program_coopk(
+ backend_ctx, kernel_src.c_str(), kernel_name,
+ base_opts + " -DCOOPK_COLS=" + std::to_string(cols), nsg_eff);
+ if (program == nullptr) {
+ return;
+ }
+ cl_int err;
+ cl_kernel kernel = clCreateKernel(program, kernel_name, &err);
+ if (err != CL_SUCCESS) { kernel = nullptr; }
+ CL_CHECK(clReleaseProgram(program));
+ if (cols == 4) {
+ kernel_c4 = kernel;
+ nsg = nsg_eff;
+ if (kernel == nullptr) {
+ return;
+ }
+ } else {
+ // Both widths share one stored subgroup count for launch.
+ if (kernel != nullptr && nsg_eff != nsg) {
+ CL_CHECK(clReleaseKernel(kernel));
+ kernel = nullptr;
+ }
+ kernel_c2 = kernel;
+ }
+ }
+}
+
+static void load_cl_kernels_gemm_coopk(ggml_backend_opencl_context * backend_ctx) {
+ if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO || !ggml_cl_coopk_dp4a_enabled(backend_ctx)) {
+ return;
+ }
+
+ // Q4_0
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_coopk_q4_0_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_coopk_q4_0_q8_1_dp4a.cl");
+#endif
+ ggml_cl_coopk_build_narrow(backend_ctx, kernel_src, "kernel_gemm_coopk_q4_0_q8_1_dp4a",
+ backend_ctx->rows_coopk_dp4a_q4_0, backend_ctx->nsg_coopk_dp4a_q4_0,
+ backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4,
+ backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c2);
+ GGML_LOG_CONT(".");
+ }
+
+ // Q4_K
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_coopk_q4_k_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_coopk_q4_k_q8_1_dp4a.cl");
+#endif
+ ggml_cl_coopk_build_narrow(backend_ctx, kernel_src, "kernel_gemm_coopk_q4_k_q8_1_dp4a",
+ backend_ctx->rows_coopk_dp4a_q4_k, backend_ctx->nsg_coopk_dp4a_q4_k,
+ backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4,
+ backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c2);
+
+ const char * env = getenv("GGML_OPENCL_COOPK_Q4_K_BIN");
+ if (use_adreno_bin_kernels(backend_ctx) && (!env || !*env || atoi(env) != 0)) {
+ const std::string base_opts = backend_ctx->kernel_compile_opts
+ + " -DCOOPK_Q_BIN=1 -DCOOPK_ROWS=" + std::to_string(backend_ctx->rows_coopk_dp4a_q4_k);
+ for (int cols = 4; cols >= 2; cols -= 2) {
+ int & nsg = (cols == 4) ? backend_ctx->nsg_coopk_dp4a_q4_k_bin_c4
+ : backend_ctx->nsg_coopk_dp4a_q4_k_bin_c2;
+ int nsg_eff = nsg;
+ cl_program program = ggml_cl_build_program_coopk(
+ backend_ctx, kernel_src.c_str(), "kernel_gemm_coopk_q4_k_q8_1_dp4a",
+ base_opts + " -DCOOPK_COLS=" + std::to_string(cols), nsg_eff);
+ if (program == nullptr) {
+ continue;
+ }
+ cl_int err;
+ cl_kernel kernel = clCreateKernel(program, "kernel_gemm_coopk_q4_k_q8_1_dp4a", &err);
+ if (err != CL_SUCCESS) { kernel = nullptr; }
+ CL_CHECK(clReleaseProgram(program));
+ if (cols == 4) {
+ backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4 = kernel;
+ } else {
+ backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2 = kernel;
+ }
+ nsg = nsg_eff;
+ }
+ }
+ GGML_LOG_CONT(".");
+ }
+
+ // Q6_K
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_coopk_q6_k_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_coopk_q6_k_q8_1_dp4a.cl");
+#endif
+ ggml_cl_coopk_build_narrow(backend_ctx, kernel_src, "kernel_gemm_coopk_q6_k_q8_1_dp4a",
+ backend_ctx->rows_coopk_dp4a_q6_k, backend_ctx->nsg_coopk_dp4a_q6_k,
+ backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4,
+ backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c2);
+ GGML_LOG_CONT(".");
+ }
+}
+
+#endif
+
static std::string ggml_opencl_make_compile_opts(const ggml_backend_opencl_context * backend_ctx) {
auto opencl_c_std =
std::string("CL") + std::to_string(backend_ctx->opencl_c_version.major) + "." + std::to_string(backend_ctx->opencl_c_version.minor);
@@ -5440,6 +5630,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
}
}
}
+ load_cl_kernels_gemm_coopk(backend_ctx);
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
GGML_LOG_CONT("\n");
backend_ctx->kernels_loaded = true;
@@ -20052,6 +20243,7 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t
warned = true;
}
}
+ // This bin layout is not supported by CoopK.
ggml_cl_mul_mat_q4_0_f32_adreno_ila(backend, src0, src1, dst);
return;
}
@@ -20139,6 +20331,71 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clReleaseMemObject(b_sub_buf));
CL_CHECK(clReleaseMemObject(b_img));
} else {
+ // CoopK uses the same untiled weight layout as gemm_noshuffle_q4_0_f32, with K-major Q8_1 activations (K contiguous).
+ if (ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q4_0)
+ && backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4 != nullptr
+ && ((N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c2
+ : backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4) != nullptr) {
+ cl_mem b_sub_buf = nullptr;
+ region.origin = offset1;
+ region.size = (size_t)K * N * sizeof(float);
+ CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+
+ const int cols = (N <= 2) ? 2 : 4;
+ const size_t n_blocks_padded = (size_t)cols * (K / 32);
+ const size_t size_qa = (size_t)cols * K * sizeof(cl_char);
+ const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size;
+ const size_t size_da_prev = backend_ctx->prealloc_moe_da.size;
+ backend_ctx->prealloc_moe_qa.allocate(context, size_qa);
+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half));
+ if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) {
+ const cl_char z8 = 0;
+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer,
+ &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL));
+ }
+ if (backend_ctx->prealloc_moe_da.size != size_da_prev) {
+ const cl_half z16 = 0;
+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer,
+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL));
+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer,
+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL));
+ }
+
+ cl_int n_blocks = (cl_int)((size_t)N * (K / 32));
+ cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf));
+ CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks));
+ size_t local_work_size_quant[1] = { 64 };
+ size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) };
+ backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst);
+
+ cl_kernel kernel_coopk = (N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c2
+ : backend_ctx->kernel_gemm_coopk_q4_0_q8_1_dp4a_c4;
+ const cl_int n_padded = cols;
+ CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q4_0->q));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q4_0->d));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_int), &M));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_int), &n_padded));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_int), &K));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &N));
+
+ const size_t nsg = (size_t)backend_ctx->nsg_coopk_dp4a_q4_0;
+ size_t local_work_size[3] = { 64, nsg, 1 };
+ size_t global_work_size[3] = { (size_t)(M / backend_ctx->rows_coopk_dp4a_q4_0), nsg, 1 };
+ backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst);
+
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+ return;
+ }
+
// dp4a (int8) dense prefill GEMM, default off
static const char * q4_0_dense_dp4a_env = getenv("GGML_OPENCL_Q4_0_DENSE_DP4A");
bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env
@@ -21711,6 +21968,86 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const gg
int N = ne1;
int K = ne00;
+ // CoopK handles with _bin use the bin weight layout (uint-transposed q), with K-major Q8_1 activations (K contiguous).
+ // This switch disables only this bin-layout path; weight conversion is unchanged.
+ static const char * env_coopk_bin = getenv("GGML_OPENCL_COOPK_Q4_K_BIN");
+ const bool use_coopk_bin = !env_coopk_bin || !*env_coopk_bin || atoi(env_coopk_bin) != 0;
+ if (use_coopk_bin && ne1 >= 2 && ne1 <= 4 &&
+ ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q4_k) &&
+ backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4 != nullptr &&
+ ((ne1 <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2
+ : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4) != nullptr) {
+ cl_mem b_sub_buf = nullptr;
+ region.origin = offset1;
+ region.size = (size_t)K * N * sizeof(float);
+ CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+
+ // Allocate all kernel columns; padded columns have independent accumulators and are not stored.
+ const int cols = (ne1 <= 2) ? 2 : 4;
+ const size_t size_qa = (size_t)cols * K * sizeof(cl_char);
+ const size_t n_blocks_padded = (size_t)cols * (K / 32);
+ const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size;
+ const size_t size_da_prev = backend_ctx->prealloc_moe_da.size;
+ backend_ctx->prealloc_moe_qa.allocate(context, size_qa);
+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half));
+ if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) {
+ const cl_char z8 = 0;
+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer,
+ &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL));
+ }
+ if (backend_ctx->prealloc_moe_da.size != size_da_prev) {
+ const cl_half z16 = 0;
+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer,
+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL));
+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer,
+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL));
+ }
+
+ cl_int n_blocks = (cl_int)((size_t)N * (K / 32));
+ cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf));
+ CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks));
+ size_t local_work_size_quant[1] = { 64 };
+ size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) };
+ backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst);
+
+ cl_kernel kernel_coopk = (ne1 <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c2
+ : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_bin_c4;
+ const int nsg_selected = (ne1 <= 2) ? backend_ctx->nsg_coopk_dp4a_q4_k_bin_c2
+ : backend_ctx->nsg_coopk_dp4a_q4_k_bin_c4;
+ const cl_uchar mask_d6 = 0x3F, mask_d4 = 0x0F, mask_hi2 = 0xC0;
+ const cl_int n_padded = cols;
+ CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q4_k->q));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q4_k->s));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &extra0_q4_k->d));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &extra0_q4_k->dm));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 10, sizeof(cl_int), &n_padded));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 11, sizeof(cl_int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 12, sizeof(cl_int), &ne1));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 13, sizeof(cl_uchar), &mask_d6));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 14, sizeof(cl_uchar), &mask_d4));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 15, sizeof(cl_uchar), &mask_hi2));
+
+ // Each width can compile with a different subgroup count.
+ const size_t nsg = (size_t)nsg_selected;
+ size_t local_work_size[3] = { 64, nsg, 1 };
+ size_t global_work_size[3] = { (size_t)(ne01 / backend_ctx->rows_coopk_dp4a_q4_k), nsg, 1 };
+ backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst);
+
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+ return;
+ }
+
if (ne1 == 1) {
cl_mem b_sub_buf = nullptr;
cl_mem b_img = nullptr;
@@ -22158,6 +22495,74 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t
region.size = K * N * sizeof(float);
CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+ // CoopK handles without _bin use the same untiled weight layout as gemm_noshuffle_q4_k_f32.
+ // Activations use K-major Q8_1 SoA (K contiguous).
+ if (ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q4_k)
+ && backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4 != nullptr
+ && ((N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c2
+ : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4) != nullptr) {
+ const int cols = (N <= 2) ? 2 : 4;
+ const size_t size_qa = (size_t)cols * K * sizeof(cl_char);
+ const size_t n_blocks_padded = (size_t)cols * (K / 32);
+ const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size;
+ const size_t size_da_prev = backend_ctx->prealloc_moe_da.size;
+ backend_ctx->prealloc_moe_qa.allocate(context, size_qa);
+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half));
+ if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) {
+ const cl_char z8 = 0;
+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer,
+ &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL));
+ }
+ if (backend_ctx->prealloc_moe_da.size != size_da_prev) {
+ const cl_half z16 = 0;
+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer,
+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL));
+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer,
+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL));
+ }
+
+ cl_int n_blocks = (cl_int)((size_t)N * (K / 32));
+ cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf));
+ CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks));
+ size_t local_work_size_quant[1] = { 64 };
+ size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) };
+ backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst);
+
+ cl_kernel kernel_coopk = (N <= 2) ? backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c2
+ : backend_ctx->kernel_gemm_coopk_q4_k_q8_1_dp4a_c4;
+ const cl_uchar mask_d6 = 0x3F, mask_d4 = 0x0F, mask_hi2 = 0xC0;
+ const cl_int n_padded = cols;
+ CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q4_k->q));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q4_k->s));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &extra0_q4_k->d));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &extra0_q4_k->dm));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &M));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 10, sizeof(cl_int), &n_padded));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 11, sizeof(cl_int), &K));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 12, sizeof(cl_int), &N));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 13, sizeof(cl_uchar), &mask_d6));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 14, sizeof(cl_uchar), &mask_d4));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 15, sizeof(cl_uchar), &mask_hi2));
+
+ const size_t nsg = (size_t)backend_ctx->nsg_coopk_dp4a_q4_k;
+ size_t local_work_size[3] = { 64, nsg, 1 };
+ size_t global_work_size[3] = { (size_t)(M / backend_ctx->rows_coopk_dp4a_q4_k), nsg, 1 };
+ backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst);
+
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+ return;
+ }
+
// image for activations
img_fmt = {CL_RGBA, CL_FLOAT};
memset(&img_desc, 0, sizeof(img_desc));
@@ -22672,6 +23077,7 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t
warned = true;
}
}
+ // This bin layout is not supported by CoopK.
ggml_cl_mul_mat_q6_K_f32_adreno_ila(backend, src0, src1, dst);
return;
}
@@ -22827,6 +23233,77 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t
region.size = ne00 * ne1 * sizeof(float);
CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+ // CoopK uses the same untiled weight layout as gemm_noshuffle_q6_K_f32, with K-major Q8_1 activations (K contiguous).
+ // Keep output and embedding weights on the existing path for logit stability.
+ if (ggml_cl_coopk_dp4a_narrow_on(backend_ctx, dst, backend_ctx->rows_coopk_dp4a_q6_k)
+ && backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4 != nullptr
+ && strncmp(src0->name, "output", 6) != 0
+ && strncmp(src0->name, "token_embd", 10) != 0
+ && ((ne1 <= 2) ? backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c2
+ : backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4) != nullptr) {
+ const int N = (int)ne1, K = (int)ne00;
+ const int cols = (N <= 2) ? 2 : 4;
+ const size_t n_blocks_padded = (size_t)cols * (K / 32);
+ const size_t size_qa = (size_t)cols * K * sizeof(cl_char);
+ const size_t size_qa_prev = backend_ctx->prealloc_moe_qa.size;
+ const size_t size_da_prev = backend_ctx->prealloc_moe_da.size;
+ backend_ctx->prealloc_moe_qa.allocate(context, size_qa);
+ backend_ctx->prealloc_moe_da.allocate(context, n_blocks_padded * sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(context, n_blocks_padded * sizeof(cl_half));
+ if (backend_ctx->prealloc_moe_qa.size != size_qa_prev) {
+ const cl_char z8 = 0;
+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_qa.buffer,
+ &z8, sizeof(z8), 0, backend_ctx->prealloc_moe_qa.size, 0, NULL, NULL));
+ }
+ if (backend_ctx->prealloc_moe_da.size != size_da_prev) {
+ const cl_half z16 = 0;
+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_da.buffer,
+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_da.size, 0, NULL, NULL));
+ CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, backend_ctx->prealloc_moe_sa.buffer,
+ &z16, sizeof(z16), 0, backend_ctx->prealloc_moe_sa.size, 0, NULL, NULL));
+ }
+
+ cl_int n_blocks = (cl_int)((size_t)N * (K / 32));
+ cl_kernel kernel_quant = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(kernel_quant, 0, sizeof(cl_mem), &b_sub_buf));
+ CL_CHECK(clSetKernelArg(kernel_quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(kernel_quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(kernel_quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(kernel_quant, 4, sizeof(cl_int), &n_blocks));
+ size_t local_work_size_quant[1] = { 64 };
+ size_t global_work_size_quant[1] = { (size_t)((((size_t)n_blocks + 63) / 64) * 64) };
+ backend_ctx->enqueue_ndrange_kernel(kernel_quant, 1, global_work_size_quant, local_work_size_quant, dst);
+
+ cl_kernel kernel_coopk = (N <= 2) ? backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c2
+ : backend_ctx->kernel_gemm_coopk_q6_k_q8_1_dp4a_c4;
+ const cl_ushort mask_f000 = 0xF000;
+ const cl_uchar mask_c0 = 0xC0;
+ const cl_int n_padded = cols;
+ const cl_int m = (cl_int)ne01, k = (cl_int)ne00, n = (cl_int)ne1;
+ CL_CHECK(clSetKernelArg(kernel_coopk, 0, sizeof(cl_mem), &extra0_q6_K->ql));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 1, sizeof(cl_mem), &extra0_q6_K->qh));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 2, sizeof(cl_mem), &extra0_q6_K->s));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 3, sizeof(cl_mem), &extra0_q6_K->d));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 6, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 7, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 8, sizeof(cl_int), &m));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 9, sizeof(cl_int), &n_padded));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 10, sizeof(cl_int), &k));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 11, sizeof(cl_int), &n));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 12, sizeof(cl_ushort), &mask_f000));
+ CL_CHECK(clSetKernelArg(kernel_coopk, 13, sizeof(cl_uchar), &mask_c0));
+
+ const size_t nsg = (size_t)backend_ctx->nsg_coopk_dp4a_q6_k;
+ size_t local_work_size[3] = { 64, nsg, 1 };
+ size_t global_work_size[3] = { (size_t)(ne01 / backend_ctx->rows_coopk_dp4a_q6_k), nsg, 1 };
+ backend_ctx->enqueue_ndrange_kernel(kernel_coopk, 3, global_work_size, local_work_size, dst);
+
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+ return;
+ }
+
// dp4a (int8) dense q6_K prefill GEMM
static const char * q6k_dense_dp4a_env = getenv("GGML_OPENCL_Q6K_DENSE_DP4A");
bool q6k_dense_dp4a_on = (q6k_dense_dp4a_env != nullptr)
diff --git a/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_0_q8_1_dp4a.cl
new file mode 100644
index 000000000..a35957409
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_0_q8_1_dp4a.cl
@@ -0,0 +1,205 @@
+// Cooperative-K DP4A GEMM for 2 to 4 columns with Q4_0 weights and Q8_1 activations.
+// Named vector components avoid dynamic private-array indexing, which can cause register spills.
+
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#ifndef COOPK_NSG
+#define COOPK_NSG 4
+#endif
+#define COOPK_SG 64
+
+#ifndef COOPK_ROWS
+#define COOPK_ROWS 4
+#endif
+#ifndef COOPK_COLS
+#define COOPK_COLS 4
+#endif
+
+#if COOPK_COLS == 4
+typedef float4 coopk_accv;
+typedef int4 coopk_dotv;
+#define COOPK_CONVF convert_float4
+#else
+typedef float2 coopk_accv;
+typedef int2 coopk_dotv;
+#define COOPK_CONVF convert_float2
+#endif
+
+// Pack four Q4_0 weights as signed bytes after subtracting 8.
+#define EXP40(l) \
+ ( (((uint)((int)(( (l) & 0x000Fu) - 8) & 0xFF))) \
+ | (((uint)((int)((((l) & 0x00F0u) >> 4) - 8) & 0xFF)) << 8) \
+ | (((uint)((int)((((l) & 0x0F00u) >> 8) - 8) & 0xFF)) << 16) \
+ | (((uint)((int)((((l) & 0xF000u) >> 12) - 8) & 0xFF)) << 24) )
+
+#if COOPK_ROWS == 4
+#define COOPK_DOT(ci, t) \
+ d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \
+ d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci); \
+ d2.s##ci = dot_acc_sat_4x8packed_ss_int(w2, A##ci.s##t, d2.s##ci); \
+ d3.s##ci = dot_acc_sat_4x8packed_ss_int(w3, A##ci.s##t, d3.s##ci);
+#else
+#define COOPK_DOT(ci, t) \
+ d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \
+ d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci);
+#endif
+
+#if COOPK_COLS == 4
+#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) COOPK_DOT(2,t) COOPK_DOT(3,t)
+#define COOPK_FOR_COLS(F) F(0) F(1) F(2) F(3)
+#else
+#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t)
+#define COOPK_FOR_COLS(F) F(0) F(1)
+#endif
+
+#if COOPK_ROWS == 4
+#define COOPK_KSTEP(t) \
+ { \
+ ushort4 bl = vload4(0, src0_q + row0 + (ku0 + t) * m); \
+ const uint w0 = EXP40(bl.s0); \
+ const uint w1 = EXP40(bl.s1); \
+ const uint w2 = EXP40(bl.s2); \
+ const uint w3 = EXP40(bl.s3); \
+ COOPK_DOTS_AT(t) \
+ }
+#else
+#define COOPK_KSTEP(t) \
+ { \
+ ushort2 bl = vload2(0, src0_q + row0 + (ku0 + t) * m); \
+ const uint w0 = EXP40(bl.s0); \
+ const uint w1 = EXP40(bl.s1); \
+ COOPK_DOTS_AT(t) \
+ }
+#endif
+
+// Weights use the untiled Adreno noshuffle SoA layout, with the output row contiguous in each plane.
+// Q8_1 activation planes are K-major (K contiguous); no activation transpose is used.
+kernel void kernel_gemm_coopk_q4_0_q8_1_dp4a(
+ global const ushort * src0_q, // four consecutive K nibbles per ushort, [k/4][m]
+ global const half * src0_d, // block scales, [k/32][m]
+ global const uint * src1_qa, // four signed activation bytes per uint, [n][k/4]
+ global const half * src1_da, // activation scales, [n][k/32]
+ global float * dst,
+ ulong offsetd,
+ int m,
+ int n,
+ int k,
+ int n_no_padding
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const int gx = get_global_id(0);
+ const int sg = get_local_id(1);
+ const int lane = get_local_id(0);
+
+ const int row0 = gx * COOPK_ROWS;
+ const int num_32blk = k / 32;
+ const int k_u = k >> 2;
+ const int k_b = k >> 5;
+
+ // The host pads activations to COOPK_COLS; stores mask columns beyond n_no_padding.
+ const int c0 = 0, c1 = 1;
+#if COOPK_COLS == 4
+ const int c2 = 2, c3 = 3;
+#endif
+
+ coopk_accv acc0 = (coopk_accv)(0.0f), acc1 = (coopk_accv)(0.0f);
+#if COOPK_ROWS == 4
+ coopk_accv acc2 = (coopk_accv)(0.0f), acc3 = (coopk_accv)(0.0f);
+#endif
+
+ for (int blk = sg; blk < num_32blk; blk += COOPK_NSG) {
+#if COOPK_ROWS == 4
+ half4 scale = vload4(0, src0_d + row0 + blk * m);
+#else
+ half2 scale = vload2(0, src0_d + row0 + blk * m);
+#endif
+
+ coopk_accv da;
+ da.s0 = (float)src1_da[c0*k_b + blk];
+ da.s1 = (float)src1_da[c1*k_b + blk];
+#if COOPK_COLS == 4
+ da.s2 = (float)src1_da[c2*k_b + blk];
+ da.s3 = (float)src1_da[c3*k_b + blk];
+#endif
+
+ // Q4_0 uses one weight scale per 32 K values.
+ coopk_dotv d0 = (coopk_dotv)(0), d1 = (coopk_dotv)(0);
+#if COOPK_ROWS == 4
+ coopk_dotv d2 = (coopk_dotv)(0), d3 = (coopk_dotv)(0);
+#endif
+ for (int half_idx = 0; half_idx < 2; ++half_idx) {
+ const int ku0 = (blk << 3) + half_idx * 4;
+
+ uint4 A0 = vload4(0, src1_qa + (uint)c0 * k_u + ku0);
+ uint4 A1 = vload4(0, src1_qa + (uint)c1 * k_u + ku0);
+#if COOPK_COLS == 4
+ uint4 A2 = vload4(0, src1_qa + (uint)c2 * k_u + ku0);
+ uint4 A3 = vload4(0, src1_qa + (uint)c3 * k_u + ku0);
+#endif
+ COOPK_KSTEP(0)
+ COOPK_KSTEP(1)
+ COOPK_KSTEP(2)
+ COOPK_KSTEP(3)
+ }
+
+ acc0 += (float)scale.s0 * da * COOPK_CONVF(d0);
+ acc1 += (float)scale.s1 * da * COOPK_CONVF(d1);
+#if COOPK_ROWS == 4
+ acc2 += (float)scale.s2 * da * COOPK_CONVF(d2);
+ acc3 += (float)scale.s3 * da * COOPK_CONVF(d3);
+#endif
+ }
+
+ // Reuse one local buffer for all rows; the first barrier protects the previous row's reads.
+ local coopk_accv reduceLM[COOPK_SG * (COOPK_NSG - 1)];
+ coopk_accv out0 = (coopk_accv)(0.0f), out1 = (coopk_accv)(0.0f);
+#if COOPK_ROWS == 4
+ coopk_accv out2 = (coopk_accv)(0.0f), out3 = (coopk_accv)(0.0f);
+#endif
+
+#define COOPK_REDUCE(accv, outv) \
+ barrier(CLK_LOCAL_MEM_FENCE); \
+ if (sg > 0) { reduceLM[(sg - 1) * COOPK_SG + lane] = (accv); } \
+ barrier(CLK_LOCAL_MEM_FENCE); \
+ if (sg == 0) { \
+ coopk_accv sum = (accv); \
+ for (int s = 0; s < COOPK_NSG - 1; s++) { \
+ sum += reduceLM[s * COOPK_SG + lane]; \
+ } \
+ (outv) = sum; \
+ }
+
+ COOPK_REDUCE(acc0, out0)
+ COOPK_REDUCE(acc1, out1)
+#if COOPK_ROWS == 4
+ COOPK_REDUCE(acc2, out2)
+ COOPK_REDUCE(acc3, out3)
+#endif
+
+#undef COOPK_REDUCE
+
+#if COOPK_ROWS == 4
+#define COOPK_STORE_COL(ci) \
+ if (idx < m*n_no_padding) { \
+ vstore4((float4)(out0.s##ci, out1.s##ci, out2.s##ci, out3.s##ci), 0, dst + idx); \
+ idx += m; \
+ }
+#else
+#define COOPK_STORE_COL(ci) \
+ if (idx < m*n_no_padding) { \
+ vstore2((float2)(out0.s##ci, out1.s##ci), 0, dst + idx); \
+ idx += m; \
+ }
+#endif
+
+ if (sg == 0) {
+ int idx = row0;
+ COOPK_FOR_COLS(COOPK_STORE_COL)
+ }
+
+#undef COOPK_STORE_COL
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_k_q8_1_dp4a.cl
new file mode 100644
index 000000000..31dce626f
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_coopk_q4_k_q8_1_dp4a.cl
@@ -0,0 +1,264 @@
+// Cooperative-K DP4A GEMM for 2 to 4 columns with Q4_K weights and Q8_1 activations.
+// Named vector components avoid dynamic private-array indexing, which can cause register spills.
+
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#define QK_K 256
+#define K_SCALE_SIZE 12
+
+#ifndef COOPK_NSG
+#define COOPK_NSG 4
+#endif
+#define COOPK_SG 64
+
+#ifndef COOPK_ROWS
+#define COOPK_ROWS 4
+#endif
+#ifndef COOPK_COLS
+#define COOPK_COLS 4
+#endif
+
+#if COOPK_COLS == 4
+typedef float4 coopk_accv;
+typedef int4 coopk_dotv;
+#define COOPK_CONVF convert_float4
+#else
+typedef float2 coopk_accv;
+typedef int2 coopk_dotv;
+#define COOPK_CONVF convert_float2
+#endif
+
+// Expand four Q4_K nibbles into bytes for DP4A.
+#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \
+ (((uint)((u) & 0x00F0u)) << 4) | \
+ (((uint)((u) & 0x0F00u)) << 8) | \
+ (((uint)((u) & 0xF000u)) << 12) )
+
+#if COOPK_ROWS == 4
+#define COOPK_DOT(ci, t) \
+ s0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, s0.s##ci); \
+ s1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, s1.s##ci); \
+ s2.s##ci = dot_acc_sat_4x8packed_ss_int(w2, A##ci.s##t, s2.s##ci); \
+ s3.s##ci = dot_acc_sat_4x8packed_ss_int(w3, A##ci.s##t, s3.s##ci);
+#else
+#define COOPK_DOT(ci, t) \
+ s0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, s0.s##ci); \
+ s1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, s1.s##ci);
+#endif
+
+#if COOPK_COLS == 4
+#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) COOPK_DOT(2,t) COOPK_DOT(3,t)
+#define COOPK_FOR_COLS(F) F(0) F(1) F(2) F(3)
+#else
+#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t)
+#define COOPK_FOR_COLS(F) F(0) F(1)
+#endif
+
+// The bin layout packs eight K values in each uint, with four nibbles in each 16-bit half.
+// Since ku0 is a multiple of four, t / 2 selects the uint and t & 1 selects its half.
+#ifdef COOPK_Q_BIN
+#define COOPK_QBITS4(t) convert_ushort4(((t) & 1) ? (vload4(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) >> 16) \
+ : (vload4(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) & 0xFFFFu))
+#define COOPK_QBITS2(t) convert_ushort2(((t) & 1) ? (vload2(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) >> 16) \
+ : (vload2(0, src0_q + row0 + ((ku0 >> 1) + ((t) >> 1)) * m) & 0xFFFFu))
+#else
+#define COOPK_QBITS4(t) vload4(0, src0_q + row0 + (ku0 + t) * m)
+#define COOPK_QBITS2(t) vload2(0, src0_q + row0 + (ku0 + t) * m)
+#endif
+
+#if COOPK_ROWS == 4
+#define COOPK_KSTEP(t) \
+ { \
+ ushort4 bits = COOPK_QBITS4(t); \
+ const uint w0 = EXP4(bits.s0); \
+ const uint w1 = EXP4(bits.s1); \
+ const uint w2 = EXP4(bits.s2); \
+ const uint w3 = EXP4(bits.s3); \
+ COOPK_DOTS_AT(t) \
+ }
+#else
+#define COOPK_KSTEP(t) \
+ { \
+ ushort2 bits = COOPK_QBITS2(t); \
+ const uint w0 = EXP4(bits.s0); \
+ const uint w1 = EXP4(bits.s1); \
+ COOPK_DOTS_AT(t) \
+ }
+#endif
+
+inline void get_scale_min_k4_c(int j, global const uchar * q, int stride,
+ uchar * d, uchar * m,
+ uchar mask_d6, uchar mask_d4, uchar mask_hi2) {
+ if (j < 4) {
+ *d = q[j*stride] & mask_d6;
+ *m = q[(j + 4)*stride] & mask_d6;
+ } else {
+ *d = (q[(j + 4)*stride] & mask_d4) | ((q[(j - 4)*stride] >> 6) << 4);
+ *m = (q[(j + 4)*stride] >> 4) | ((q[(j )*stride] >> 6) << 4);
+ }
+}
+
+// Weights use the untiled Adreno noshuffle SoA layout; COOPK_Q_BIN selects uint-transposed q instead of ushort-transposed q.
+// Q8_1 activation planes are K-major (K contiguous); no activation transpose is used.
+kernel void kernel_gemm_coopk_q4_k_q8_1_dp4a(
+#ifdef COOPK_Q_BIN
+ global const uint * src0_q, // eight consecutive K nibbles per uint, [k/8][m]
+#else
+ global const ushort * src0_q, // four consecutive K nibbles per ushort, [k/4][m]
+#endif
+ global const uchar * src0_s, // packed 6-bit scales/mins, [k/256][12][m]
+ global const half * src0_d, // super-block scales, [k/256][m]
+ global const half * src0_dm, // super-block mins, [k/256][m]
+ global const uint * src1_qa, // four signed activation bytes per uint, [n][k/4]
+ global const half * src1_da, // activation scales, [n][k/32]
+ global const half * src1_sa, // scaled activation sums, [n][k/32]
+ global float * dst,
+ ulong offsetd,
+ int m,
+ int n,
+ int k,
+ int n_no_padding,
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const int gx = get_global_id(0); // row group
+ const int sg = get_local_id(1); // K-split subgroup
+ const int lane = get_local_id(0);
+
+ const int row0 = gx * COOPK_ROWS;
+ const int num_32blk = k / 32;
+ const int k_u = k >> 2; // K in uint (int8x4) units
+ const int k_b = k >> 5; // 32-blocks along K
+
+ // The host pads activations to COOPK_COLS; stores mask columns beyond n_no_padding.
+ const int nl = n - 1;
+ const int c0 = 0;
+ const int c1 = (1 < n) ? 1 : nl;
+#if COOPK_COLS >= 4
+ const int c2 = (2 < n) ? 2 : nl;
+ const int c3 = (3 < n) ? 3 : nl;
+#endif
+
+ coopk_accv acc0 = (coopk_accv)(0.0f), acc1 = (coopk_accv)(0.0f);
+#if COOPK_ROWS == 4
+ coopk_accv acc2 = (coopk_accv)(0.0f), acc3 = (coopk_accv)(0.0f);
+#endif
+
+ for (int blk = sg; blk < num_32blk; blk += COOPK_NSG) {
+ const int i = blk << 5;
+ const int sb_idx = blk >> 3;
+ const int sub_idx = blk & 7;
+
+ global const uchar * sc = src0_s + sb_idx * K_SCALE_SIZE * m + row0;
+ uchar sv0, mn0, sv1, mn1;
+ get_scale_min_k4_c(sub_idx, sc + 0, m, &sv0, &mn0, mask_d6, mask_d4, mask_hi2);
+ get_scale_min_k4_c(sub_idx, sc + 1, m, &sv1, &mn1, mask_d6, mask_d4, mask_hi2);
+#if COOPK_ROWS == 4
+ half4 dd = vload4(0, src0_d + row0 + sb_idx * m);
+ half4 dmm = vload4(0, src0_dm + row0 + sb_idx * m);
+ uchar sv2, mn2, sv3, mn3;
+ get_scale_min_k4_c(sub_idx, sc + 2, m, &sv2, &mn2, mask_d6, mask_d4, mask_hi2);
+ get_scale_min_k4_c(sub_idx, sc + 3, m, &sv3, &mn3, mask_d6, mask_d4, mask_hi2);
+ const float sc2 = (float)dd.s2 * (float)sv2, mv2 = (float)dmm.s2 * (float)mn2;
+ const float sc3 = (float)dd.s3 * (float)sv3, mv3 = (float)dmm.s3 * (float)mn3;
+#else
+ half2 dd = vload2(0, src0_d + row0 + sb_idx * m);
+ half2 dmm = vload2(0, src0_dm + row0 + sb_idx * m);
+#endif
+ const float sc0 = (float)dd.s0 * (float)sv0, mv0 = (float)dmm.s0 * (float)mn0;
+ const float sc1 = (float)dd.s1 * (float)sv1, mv1 = (float)dmm.s1 * (float)mn1;
+
+ // Q4_K scale and min values change every 32 K values.
+ coopk_dotv s0 = (coopk_dotv)(0), s1 = (coopk_dotv)(0);
+#if COOPK_ROWS == 4
+ coopk_dotv s2 = (coopk_dotv)(0), s3 = (coopk_dotv)(0);
+#endif
+
+ for (int uq = 0; uq < 2; ++uq) {
+ const int ku0 = (i >> 2) + uq * 4;
+
+ uint4 A0 = vload4(0, src1_qa + (uint)c0 * k_u + ku0);
+ uint4 A1 = vload4(0, src1_qa + (uint)c1 * k_u + ku0);
+#if COOPK_COLS >= 4
+ uint4 A2 = vload4(0, src1_qa + (uint)c2 * k_u + ku0);
+ uint4 A3 = vload4(0, src1_qa + (uint)c3 * k_u + ku0);
+#endif
+ COOPK_KSTEP(0)
+ COOPK_KSTEP(1)
+ COOPK_KSTEP(2)
+ COOPK_KSTEP(3)
+ }
+
+ // Q4_K stores q * scale - min for each 32-value block.
+ // src1_sa already includes the activation scale, so the min term needs no extra factor of da.
+ coopk_accv da, sa;
+ da.s0 = (float)src1_da[c0*k_b + blk]; sa.s0 = (float)src1_sa[c0*k_b + blk];
+ da.s1 = (float)src1_da[c1*k_b + blk]; sa.s1 = (float)src1_sa[c1*k_b + blk];
+#if COOPK_COLS >= 4
+ da.s2 = (float)src1_da[c2*k_b + blk]; sa.s2 = (float)src1_sa[c2*k_b + blk];
+ da.s3 = (float)src1_da[c3*k_b + blk]; sa.s3 = (float)src1_sa[c3*k_b + blk];
+#endif
+
+ acc0 += sc0 * da * COOPK_CONVF(s0) - mv0 * sa;
+ acc1 += sc1 * da * COOPK_CONVF(s1) - mv1 * sa;
+#if COOPK_ROWS == 4
+ acc2 += sc2 * da * COOPK_CONVF(s2) - mv2 * sa;
+ acc3 += sc3 * da * COOPK_CONVF(s3) - mv3 * sa;
+#endif
+ }
+
+ // Reuse one local buffer for all rows; the first barrier protects the previous row's reads.
+ local coopk_accv reduceLM[COOPK_SG * (COOPK_NSG - 1)];
+ coopk_accv out0 = (coopk_accv)(0.0f), out1 = (coopk_accv)(0.0f);
+#if COOPK_ROWS == 4
+ coopk_accv out2 = (coopk_accv)(0.0f), out3 = (coopk_accv)(0.0f);
+#endif
+
+#define COOPK_REDUCE(accv, outv) \
+ barrier(CLK_LOCAL_MEM_FENCE); \
+ if (sg > 0) { reduceLM[(sg - 1) * COOPK_SG + lane] = (accv); } \
+ barrier(CLK_LOCAL_MEM_FENCE); \
+ if (sg == 0) { \
+ coopk_accv sum = (accv); \
+ for (int s = 0; s < COOPK_NSG - 1; s++) { \
+ sum += reduceLM[s * COOPK_SG + lane]; \
+ } \
+ (outv) = sum; \
+ }
+
+ COOPK_REDUCE(acc0, out0)
+ COOPK_REDUCE(acc1, out1)
+#if COOPK_ROWS == 4
+ COOPK_REDUCE(acc2, out2)
+ COOPK_REDUCE(acc3, out3)
+#endif
+
+#undef COOPK_REDUCE
+
+#if COOPK_ROWS == 4
+#define COOPK_STORE_COL(ci) \
+ if (idx < m*n_no_padding) { \
+ vstore4((float4)(out0.s##ci, out1.s##ci, out2.s##ci, out3.s##ci), 0, dst + idx); \
+ idx += m; \
+ }
+#else
+#define COOPK_STORE_COL(ci) \
+ if (idx < m*n_no_padding) { \
+ vstore2((float2)(out0.s##ci, out1.s##ci), 0, dst + idx); \
+ idx += m; \
+ }
+#endif
+
+ if (sg == 0) {
+ int idx = row0;
+ COOPK_FOR_COLS(COOPK_STORE_COL)
+ }
+
+#undef COOPK_STORE_COL
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_coopk_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_coopk_q6_k_q8_1_dp4a.cl
new file mode 100644
index 000000000..4410a230d
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_coopk_q6_k_q8_1_dp4a.cl
@@ -0,0 +1,216 @@
+// Cooperative-K DP4A GEMM for 2 to 4 columns with Q6_K weights and Q8_1 activations.
+// Named vector components avoid dynamic private-array indexing, which can cause register spills.
+
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#ifndef COOPK_NSG
+#define COOPK_NSG 4
+#endif
+#define COOPK_SG 64
+
+#ifndef COOPK_ROWS
+#define COOPK_ROWS 4
+#endif
+#ifndef COOPK_COLS
+#define COOPK_COLS 4
+#endif
+
+#if COOPK_COLS == 4
+typedef float4 coopk_accv;
+typedef int4 coopk_dotv;
+#define COOPK_CONVF convert_float4
+#else
+typedef float2 coopk_accv;
+typedef int2 coopk_dotv;
+#define COOPK_CONVF convert_float2
+#endif
+
+// Combine low nibbles and high bits, subtract 32, and pack four signed bytes for DP4A.
+#define EXP6(l, h, mf, mc) \
+ ( (((uint)((int)((( (l) & 0x000Fu) | (((h) & 0x03u) << 4)) - 32) & 0xFF))) \
+ | (((uint)((int)(((((l) & 0x00F0u) >> 4) | (((h) & 0x0Cu) << 2)) - 32) & 0xFF)) << 8) \
+ | (((uint)((int)(((((l) & 0x0F00u) >> 8) | ((h) & 0x30u)) - 32) & 0xFF)) << 16) \
+ | (((uint)((int)(((((l) & (mf)) >> 12) | (((h) & (mc)) >> 2)) - 32) & 0xFF)) << 24) )
+
+#if COOPK_ROWS == 4
+#define COOPK_DOT(ci, t) \
+ d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \
+ d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci); \
+ d2.s##ci = dot_acc_sat_4x8packed_ss_int(w2, A##ci.s##t, d2.s##ci); \
+ d3.s##ci = dot_acc_sat_4x8packed_ss_int(w3, A##ci.s##t, d3.s##ci);
+#else
+#define COOPK_DOT(ci, t) \
+ d0.s##ci = dot_acc_sat_4x8packed_ss_int(w0, A##ci.s##t, d0.s##ci); \
+ d1.s##ci = dot_acc_sat_4x8packed_ss_int(w1, A##ci.s##t, d1.s##ci);
+#endif
+
+#if COOPK_COLS == 4
+#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t) COOPK_DOT(2,t) COOPK_DOT(3,t)
+#define COOPK_FOR_COLS(F) F(0) F(1) F(2) F(3)
+#else
+#define COOPK_DOTS_AT(t) COOPK_DOT(0,t) COOPK_DOT(1,t)
+#define COOPK_FOR_COLS(F) F(0) F(1)
+#endif
+
+#if COOPK_ROWS == 4
+#define COOPK_KSTEP(t) \
+ { \
+ ushort4 bl = vload4(0, src0_ql + row0 + (ku0 + t) * m); \
+ uchar4 bh = vload4(0, src0_qh + row0 + (ku0 + t) * m); \
+ const uint w0 = EXP6(bl.s0, bh.s0, mask_f000, mask_c0); \
+ const uint w1 = EXP6(bl.s1, bh.s1, mask_f000, mask_c0); \
+ const uint w2 = EXP6(bl.s2, bh.s2, mask_f000, mask_c0); \
+ const uint w3 = EXP6(bl.s3, bh.s3, mask_f000, mask_c0); \
+ COOPK_DOTS_AT(t) \
+ }
+#else
+#define COOPK_KSTEP(t) \
+ { \
+ ushort2 bl = vload2(0, src0_ql + row0 + (ku0 + t) * m); \
+ uchar2 bh = vload2(0, src0_qh + row0 + (ku0 + t) * m); \
+ const uint w0 = EXP6(bl.s0, bh.s0, mask_f000, mask_c0); \
+ const uint w1 = EXP6(bl.s1, bh.s1, mask_f000, mask_c0); \
+ COOPK_DOTS_AT(t) \
+ }
+#endif
+
+// Weights use the untiled Adreno noshuffle SoA layout, with the output row contiguous in each plane.
+// Q8_1 activation planes are K-major (K contiguous); no activation transpose is used.
+kernel void kernel_gemm_coopk_q6_k_q8_1_dp4a(
+ global const ushort * src0_ql, // four consecutive low nibbles per ushort, [k/4][m]
+ global const uchar * src0_qh, // four consecutive high 2-bit codes per uchar, [k/4][m]
+ global const ushort * src0_s, // two signed int8 scales per ushort, [k/32][m]
+ global const half * src0_d, // super-block scales, [k/256][m]
+ global const uint * src1_qa, // four signed activation bytes per uint, [n][k/4]
+ global const half * src1_da, // activation scales, [n][k/32]
+ global float * dst,
+ ulong offsetd,
+ int m,
+ int n,
+ int k,
+ int n_no_padding,
+ ushort mask_f000,
+ uchar mask_c0
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const int gx = get_global_id(0);
+ const int sg = get_local_id(1);
+ const int lane = get_local_id(0);
+
+ const int row0 = gx * COOPK_ROWS;
+ const int num_32blk = k / 32;
+ const int k_u = k >> 2;
+ const int k_b = k >> 5;
+
+ // The host pads activations to COOPK_COLS; stores mask columns beyond n_no_padding.
+ const int c0 = 0, c1 = 1;
+#if COOPK_COLS == 4
+ const int c2 = 2, c3 = 3;
+#endif
+
+ coopk_accv acc0 = (coopk_accv)(0.0f), acc1 = (coopk_accv)(0.0f);
+#if COOPK_ROWS == 4
+ coopk_accv acc2 = (coopk_accv)(0.0f), acc3 = (coopk_accv)(0.0f);
+#endif
+
+ for (int blk = sg; blk < num_32blk; blk += COOPK_NSG) {
+ const int i = blk << 5;
+
+ // Each ushort packs two signed scale bytes, one for each 16-value half of this 32-value block.
+ // The half-precision super-block scale is shared by 256 K values.
+ ushort4 spk = vload4(0, src0_s + row0 + blk * m);
+ half4 scd = vload4(0, src0_d + row0 + (i >> 8) * m);
+
+ coopk_accv da;
+ da.s0 = (float)src1_da[c0*k_b + blk];
+ da.s1 = (float)src1_da[c1*k_b + blk];
+#if COOPK_COLS == 4
+ da.s2 = (float)src1_da[c2*k_b + blk];
+ da.s3 = (float)src1_da[c3*k_b + blk];
+#endif
+
+ // Apply each signed weight scale to its 16-value dot product.
+ for (int half_idx = 0; half_idx < 2; ++half_idx) {
+ const int ku0 = (i >> 2) + half_idx * 4;
+
+ coopk_dotv d0 = (coopk_dotv)(0), d1 = (coopk_dotv)(0);
+#if COOPK_ROWS == 4
+ coopk_dotv d2 = (coopk_dotv)(0), d3 = (coopk_dotv)(0);
+#endif
+ uint4 A0 = vload4(0, src1_qa + (uint)c0 * k_u + ku0);
+ uint4 A1 = vload4(0, src1_qa + (uint)c1 * k_u + ku0);
+#if COOPK_COLS == 4
+ uint4 A2 = vload4(0, src1_qa + (uint)c2 * k_u + ku0);
+ uint4 A3 = vload4(0, src1_qa + (uint)c3 * k_u + ku0);
+#endif
+ COOPK_KSTEP(0)
+ COOPK_KSTEP(1)
+ COOPK_KSTEP(2)
+ COOPK_KSTEP(3)
+
+ const float s0f = (float)(half_idx ? as_char2(spk.s0).s1 : as_char2(spk.s0).s0) * (float)scd.s0;
+ const float s1f = (float)(half_idx ? as_char2(spk.s1).s1 : as_char2(spk.s1).s0) * (float)scd.s1;
+ acc0 += s0f * da * COOPK_CONVF(d0);
+ acc1 += s1f * da * COOPK_CONVF(d1);
+#if COOPK_ROWS == 4
+ const float s2f = (float)(half_idx ? as_char2(spk.s2).s1 : as_char2(spk.s2).s0) * (float)scd.s2;
+ const float s3f = (float)(half_idx ? as_char2(spk.s3).s1 : as_char2(spk.s3).s0) * (float)scd.s3;
+ acc2 += s2f * da * COOPK_CONVF(d2);
+ acc3 += s3f * da * COOPK_CONVF(d3);
+#endif
+ }
+ }
+
+ // Reuse one local buffer for all rows; the first barrier protects the previous row's reads.
+ local coopk_accv reduceLM[COOPK_SG * (COOPK_NSG - 1)];
+ coopk_accv out0 = (coopk_accv)(0.0f), out1 = (coopk_accv)(0.0f);
+#if COOPK_ROWS == 4
+ coopk_accv out2 = (coopk_accv)(0.0f), out3 = (coopk_accv)(0.0f);
+#endif
+
+#define COOPK_REDUCE(accv, outv) \
+ barrier(CLK_LOCAL_MEM_FENCE); \
+ if (sg > 0) { reduceLM[(sg - 1) * COOPK_SG + lane] = (accv); } \
+ barrier(CLK_LOCAL_MEM_FENCE); \
+ if (sg == 0) { \
+ coopk_accv sum = (accv); \
+ for (int s = 0; s < COOPK_NSG - 1; s++) { \
+ sum += reduceLM[s * COOPK_SG + lane]; \
+ } \
+ (outv) = sum; \
+ }
+
+ COOPK_REDUCE(acc0, out0)
+ COOPK_REDUCE(acc1, out1)
+#if COOPK_ROWS == 4
+ COOPK_REDUCE(acc2, out2)
+ COOPK_REDUCE(acc3, out3)
+#endif
+
+#undef COOPK_REDUCE
+
+#if COOPK_ROWS == 4
+#define COOPK_STORE_COL(ci) \
+ if (idx < m*n_no_padding) { \
+ vstore4((float4)(out0.s##ci, out1.s##ci, out2.s##ci, out3.s##ci), 0, dst + idx); \
+ idx += m; \
+ }
+#else
+#define COOPK_STORE_COL(ci) \
+ if (idx < m*n_no_padding) { \
+ vstore2((float2)(out0.s##ci, out1.s##ci), 0, dst + idx); \
+ idx += m; \
+ }
+#endif
+
+ if (sg == 0) {
+ int idx = row0;
+ COOPK_FOR_COLS(COOPK_STORE_COL)
+ }
+
+#undef COOPK_STORE_COL
+}
diff --git a/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl b/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl
index 3ce06305a..5b6dea773 100644
--- a/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl
+++ b/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl
@@ -1,9 +1,7 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-// Quantize a contiguous [N, K] f32 activation buffer (token-major, K contiguous
-// per token) into q8_1 blocks of 32: int8 quants + per-block scale d + per-block
-// sum s (= d * Sum(qs)). Consumed by kernel_gemm_noshuffle_q4_k_q8_1_dp4a for the
-// dp4a (int8) dense q4_K prefill GEMM. One work-item per 32-element block.
+// Quantize a contiguous K-major [N, K] f32 activation buffer (K contiguous) into 32-value Q8_1 SoA blocks.
+// Store int8 quants, per-block scale d, and scaled sum s = d * sum(qs); one work-item handles each block.
__kernel void kernel_quant_a_q8_1(
__global const float * src, // [N * K]
__global char * qa, // [N * K]