Commit b531e4c74 for llama.cpp
commit b531e4c7494e5a9349e40d2ea9efc3686f635cdf
Author: lhez <lih@qti.qualcomm.com>
Date: Wed Oct 21 00:48:13 2026 -0700
opencl: add dp4a GEMM variants to vector load activation (#30753)
* opencl: add dp4a GEMM variants with vector activation loads
* opencl: rename kernels and env vars
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 70f199dba..6071b6e58 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -1243,6 +1243,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_gemm_noshuffle_q4_1_f32;
cl_kernel kernel_gemm_noshuffle_q8_0_f32, kernel_gemm_noshuffle_q8_0_f32_bin;
cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q8_0 prefill GEMM (opt-in)
+ cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4 = nullptr;
cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = nullptr; // q8_0 dense dp4a, weights via texture (opt-in)
cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = nullptr;
cl_kernel kernel_gemv_noshuffle_q8_0_f32;
@@ -1264,6 +1265,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin;
cl_kernel kernel_gemv_noshuffle_q4_k_f32_32b_trans;
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM
+ cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4 = nullptr;
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = nullptr; // dp4a dense prefill GEMM, weights via texture (X1 opt-in)
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow = nullptr;
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow_wimg = nullptr;
@@ -1271,7 +1273,9 @@ struct ggml_backend_opencl_context {
int tile_n_dp4a_narrow_q4_k = 32;
int max_n_dp4a_narrow_q4_k = 16;
cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_K prefill GEMM
+ cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4 = nullptr;
cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q6_K prefill GEMM
+ cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4 = nullptr;
cl_kernel kernel_quant_a_q8_1; // plain activation q8_1 pre-pass
cl_kernel kernel_gemm_noshuffle_q4_k_f32_r1;
cl_kernel kernel_gemm_noshuffle_q4_k_f32_kimg;
@@ -1305,6 +1309,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_gemm_noshuffle_iq4_nl_f32;
cl_kernel kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = nullptr; // dp4a (int8) dense IQ4_NL prefill GEMM
cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q4_0 prefill GEMM
+ cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 = nullptr;
cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c4 = nullptr;
cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c2 = nullptr;
int nsg_coopk_dp4a_q4_0 = 8;
@@ -4347,6 +4352,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -4486,6 +4492,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), dp4a_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a", &err), err));
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4", &err), err));
CL_CHECK(clReleaseProgram(prog));
// Compile a smaller N tile to avoid unused columns in small batches.
@@ -4518,6 +4525,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a", &err), err));
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -4533,6 +4541,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -4548,6 +4557,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -20430,11 +20440,13 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t
return;
}
- // dp4a (int8) dense prefill GEMM, default off
+ static const char * q4_0_act_vload4_env = getenv("GGML_OPENCL_Q4_0_DP4A_ACT_VLOAD4");
+ const bool q4_0_act_vload4_on = q4_0_act_vload4_env == nullptr || atoi(q4_0_act_vload4_env) != 0;
+ // Enable dense prefill by default on X2E when the uint4 activation tile is enabled.
static const char * q4_0_dense_dp4a_env = getenv("GGML_OPENCL_Q4_0_DENSE_DP4A");
bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env
? (atoi(q4_0_dense_dp4a_env) != 0)
- : false;
+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && q4_0_act_vload4_on);
// dot prod has to be available
q4_0_dense_dp4a_on = backend_ctx->has_integer_dot && q4_0_dense_dp4a_on;
@@ -20461,7 +20473,9 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t
size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };
backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
- cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a;
+ cl_kernel dk = q4_0_act_vload4_on
+ ? backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4
+ : backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a;
int ai = 0;
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->q));
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->d));
@@ -21754,8 +21768,13 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t
}
}
- cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg
- : backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a;
+ static const char * q8_0_act_vload4_env = getenv("GGML_OPENCL_Q8_0_DP4A_ACT_VLOAD4");
+ const bool q8_0_act_vload4_on = q8_0_act_vload4_env == nullptr || atoi(q8_0_act_vload4_env) != 0;
+ cl_kernel dk = use_wimg
+ ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg
+ : q8_0_act_vload4_on
+ ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4
+ : backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a;
int ai = 0;
if (use_wimg) {
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q8_q_img));
@@ -22703,11 +22722,16 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t
const bool use_narrow = backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow != nullptr
&& N <= backend_ctx->max_n_dp4a_narrow_q4_k;
+ static const char * q4_k_act_vload4_env = getenv("GGML_OPENCL_Q4_K_DP4A_ACT_VLOAD4");
+ const bool q4_k_act_vload4_on = q4_k_act_vload4_env == nullptr || atoi(q4_k_act_vload4_env) != 0;
cl_kernel dk = use_narrow
? (use_wimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow_wimg
: backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow)
- : (use_wimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg
- : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a);
+ : use_wimg
+ ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg
+ : q4_k_act_vload4_on
+ ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4
+ : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a;
int ai = 0;
if (use_wimg) {
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q4k_q_img));
@@ -23369,7 +23393,11 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t
size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };
backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
- cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a;
+ static const char * q6_k_act_vload4_env = getenv("GGML_OPENCL_Q6_K_DP4A_ACT_VLOAD4");
+ const bool q6_k_act_vload4_on = q6_k_act_vload4_env == nullptr || atoi(q6_k_act_vload4_env) != 0;
+ cl_kernel dk = q6_k_act_vload4_on
+ ? backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4
+ : backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a;
int ai = 0;
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->ql));
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->qh));
@@ -23925,7 +23953,11 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t
size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };
backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
- cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a;
+ static const char * q5_k_act_vload4_env = getenv("GGML_OPENCL_Q5_K_DP4A_ACT_VLOAD4");
+ const bool q5_k_act_vload4_on = q5_k_act_vload4_env == nullptr || atoi(q5_k_act_vload4_env) != 0;
+ cl_kernel dk = q5_k_act_vload4_on
+ ? backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4
+ : backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a;
int ai = 0;
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->q));
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->qh));
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl
index 446a8eb18..6418d0f8f 100644
--- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl
@@ -125,3 +125,129 @@ kernel void kernel_gemm_noshuffle_q4_0_q8_1_dp4a(
}
#undef NGROUPS
}
+
+// Stage each 32-K activation block as two uint4 values.
+inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
+ int r = 0;
+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
+ return r;
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4(
+ __global const ushort * src0_q,
+ __global const half * src0_d,
+ __global const uint * src1_qa,
+ __global const half * src1_da,
+ __global const half * src1_sa,
+ __global float * dst,
+ ulong offsetd,
+ int m,
+ int n_no_padding,
+ int k
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0);
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0;
+
+ const uint k_u = (uint)k >> 2;
+ const uint k_b = (uint)k >> 5;
+
+ __local uint4 sh_qa4[TILESIZE_N][2];
+ __local half sh_d[TILESIZE_N];
+ __local half sh_s[TILESIZE_N];
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ acc[g] = (float4)(0.0f);
+ }
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+
+ const float d_w = (float)src0_d[rrow + sub * (uint)m];
+
+ const uint qsbase = rrow + (step >> 2) * (uint)m;
+ uint8 qw;
+ qw.s0 = EXP4(src0_q[qsbase + 0 * m]);
+ qw.s1 = EXP4(src0_q[qsbase + 1 * m]);
+ qw.s2 = EXP4(src0_q[qsbase + 2 * m]);
+ qw.s3 = EXP4(src0_q[qsbase + 3 * m]);
+ qw.s4 = EXP4(src0_q[qsbase + 4 * m]);
+ qw.s5 = EXP4(src0_q[qsbase + 5 * m]);
+ qw.s6 = EXP4(src0_q[qsbase + 6 * m]);
+ qw.s7 = EXP4(src0_q[qsbase + 7 * m]);
+
+ for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
+ const uint t = idx >> 1;
+ const uint v = idx & 1;
+ const uint c = col_base + t;
+ sh_qa4[t][v] = (c < (uint)n_no_padding)
+ ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
+ : (uint4)(0u);
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1])
+ rf.s0 = (float)DOTV(b+0);
+ rf.s1 = (float)DOTV(b+1);
+ rf.s2 = (float)DOTV(b+2);
+ rf.s3 = (float)DOTV(b+3);
+#undef DOTV
+
+ acc[g] += d_w * (LD4(sh_d, b) * rf - 8.0f * LD4(sh_s, b));
+ }
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) {
+ dst[(c0 + 0) * (uint)m + row] = a.s0;
+ }
+ if (c0 + 1 < (uint)n_no_padding) {
+ dst[(c0 + 1) * (uint)m + row] = a.s1;
+ }
+ if (c0 + 2 < (uint)n_no_padding) {
+ dst[(c0 + 2) * (uint)m + row] = a.s2;
+ }
+ if (c0 + 3 < (uint)n_no_padding) {
+ dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+ }
+#undef NGROUPS
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl
index a3b39b6aa..69cabb946 100644
--- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl
@@ -279,3 +279,141 @@ kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg(
}
#undef NGROUPS
}
+
+// Stage each 32-K activation block as two uint4 values.
+inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
+ int r = 0;
+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
+ return r;
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4(
+ __global const ushort * src0_q,
+ __global const uchar * src0_s,
+ __global const half * src0_d,
+ __global const half * src0_dm,
+ __global const uint * src1_qa,
+ __global const half * src1_da,
+ __global const half * src1_sa,
+ __global float * dst,
+ ulong offsetd,
+ int m,
+ int n_no_padding,
+ int k,
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0);
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0;
+
+ const uint k_u = (uint)k >> 2;
+ const uint k_b = (uint)k >> 5;
+
+ __local uint4 sh_qa4[TILESIZE_N][2];
+ __local half sh_d[TILESIZE_N];
+ __local half sh_s[TILESIZE_N];
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ acc[g] = (float4)(0.0f);
+ }
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+ const uint sb_idx = step / QK_K;
+ const uint sub_idx = sub & 7;
+
+ const float dd = (float)src0_d [rrow + sb_idx * m];
+ const float dmm = (float)src0_dm[rrow + sb_idx * m];
+ global const uchar * sc = src0_s + sb_idx * K_SCALE_SIZE * (uint)m + rrow;
+ uchar sv, mn;
+ get_scale_min_k4(sub_idx, sc, (uint)m, &sv, &mn, mask_d6, mask_d4, mask_hi2);
+ const float scale = dd * (float)sv;
+ const float minv = dmm * (float)mn;
+
+ const uint wbase = rrow + (step >> 2) * (uint)m;
+ uint8 qw;
+ qw.s0 = EXP4(src0_q[wbase + 0 * m]);
+ qw.s1 = EXP4(src0_q[wbase + 1 * m]);
+ qw.s2 = EXP4(src0_q[wbase + 2 * m]);
+ qw.s3 = EXP4(src0_q[wbase + 3 * m]);
+ qw.s4 = EXP4(src0_q[wbase + 4 * m]);
+ qw.s5 = EXP4(src0_q[wbase + 5 * m]);
+ qw.s6 = EXP4(src0_q[wbase + 6 * m]);
+ qw.s7 = EXP4(src0_q[wbase + 7 * m]);
+
+ for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
+ const uint t = idx >> 1;
+ const uint v = idx & 1;
+ const uint c = col_base + t;
+ sh_qa4[t][v] = (c < (uint)n_no_padding)
+ ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
+ : (uint4)(0u);
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1])
+ rf.s0 = (float)DOTV(b+0);
+ rf.s1 = (float)DOTV(b+1);
+ rf.s2 = (float)DOTV(b+2);
+ rf.s3 = (float)DOTV(b+3);
+#undef DOTV
+ acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);
+ }
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) {
+ dst[(c0 + 0) * (uint)m + row] = a.s0;
+ }
+ if (c0 + 1 < (uint)n_no_padding) {
+ dst[(c0 + 1) * (uint)m + row] = a.s1;
+ }
+ if (c0 + 2 < (uint)n_no_padding) {
+ dst[(c0 + 2) * (uint)m + row] = a.s2;
+ }
+ if (c0 + 3 < (uint)n_no_padding) {
+ dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+ }
+#undef NGROUPS
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl
index aaeed68f6..eef474b47 100644
--- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl
@@ -162,3 +162,141 @@ kernel void kernel_gemm_noshuffle_q5_k_q8_1_dp4a(
}
#undef NGROUPS
}
+
+// Stage each 32-K activation block as two uint4 values.
+inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
+ int r = 0;
+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
+ return r;
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4(
+ __global const ushort * src0_q,
+ __global const uchar * src0_qh,
+ __global const uchar * src0_s,
+ __global const half * src0_d,
+ __global const half * src0_dm,
+ __global const uint * src1_qa,
+ __global const half * src1_da,
+ __global const half * src1_sa,
+ __global float * dst,
+ ulong offsetd,
+ int m,
+ int n_no_padding,
+ int k,
+ uchar mask_d6,
+ uchar mask_d4,
+ uchar mask_hi2
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0);
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0;
+
+ const uint num_superblocks = (uint)k / QK_K;
+ const uint k_u = (uint)k >> 2;
+ const uint k_b = (uint)k >> 5;
+
+ __local uint4 sh_qa4[TILESIZE_N][2];
+ __local half sh_d[TILESIZE_N];
+ __local half sh_s[TILESIZE_N];
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ acc[g] = (float4)(0.0f);
+ }
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+ const uint sb_idx = step / QK_K;
+ const uint sub_idx = sub & 7;
+
+ const float dd = (float)src0_d [rrow + sb_idx * m];
+ const float dmm = (float)src0_dm[rrow + sb_idx * m];
+ global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE;
+ uchar sv, mn;
+ get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2);
+ const float scale = dd * (float)sv;
+ const float minv = dmm * (float)mn;
+
+ const uint wbase = rrow + (step >> 2) * (uint)m;
+ const uint qhbase = rrow + (step >> 3) * (uint)m;
+ uint8 qw;
+#define QWU(u) ( EXP4((uint)src0_q[wbase + (uint)(u) * m]) \
+ | EXP1( (uint)((src0_qh[qhbase + (uint)((u) >> 1) * m] >> (((u) & 1) * 4)) & 0x0Fu) ) )
+ qw.s0 = QWU(0); qw.s1 = QWU(1); qw.s2 = QWU(2); qw.s3 = QWU(3);
+ qw.s4 = QWU(4); qw.s5 = QWU(5); qw.s6 = QWU(6); qw.s7 = QWU(7);
+#undef QWU
+
+ for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
+ const uint t = idx >> 1;
+ const uint v = idx & 1;
+ const uint c = col_base + t;
+ sh_qa4[t][v] = (c < (uint)n_no_padding)
+ ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
+ : (uint4)(0u);
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1])
+ rf.s0 = (float)DOTV(b+0);
+ rf.s1 = (float)DOTV(b+1);
+ rf.s2 = (float)DOTV(b+2);
+ rf.s3 = (float)DOTV(b+3);
+#undef DOTV
+ acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);
+ }
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) {
+ dst[(c0 + 0) * (uint)m + row] = a.s0;
+ }
+ if (c0 + 1 < (uint)n_no_padding) {
+ dst[(c0 + 1) * (uint)m + row] = a.s1;
+ }
+ if (c0 + 2 < (uint)n_no_padding) {
+ dst[(c0 + 2) * (uint)m + row] = a.s2;
+ }
+ if (c0 + 3 < (uint)n_no_padding) {
+ dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+ }
+#undef NGROUPS
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl
index 382d79fdd..f17403a72 100644
--- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl
@@ -142,3 +142,117 @@ kernel void kernel_gemm_noshuffle_q6_k_q8_1_dp4a(
}
#undef NGROUPS
}
+
+// Stage each 32-K activation block as two uint4 values.
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4(
+ __global const ushort * src0_ql,
+ __global const uchar * src0_qh,
+ __global const ushort * src0_s,
+ __global const half * src0_d,
+ __global const uint * src1_qa,
+ __global const half * src1_da,
+ __global float * dst,
+ ulong offsetd,
+ int m,
+ int n_no_padding,
+ int k
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0);
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0;
+
+ const uint k_u = (uint)k >> 2;
+ const uint k_b = (uint)k >> 5;
+
+ __local uint4 sh_qa4[TILESIZE_N][2];
+ __local half sh_d[TILESIZE_N];
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ acc[g] = (float4)(0.0f);
+ }
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+ const uint sb_idx = step / QK_K;
+
+ const float dd = (float)src0_d[rrow + sb_idx * m];
+ const char2 sc = as_char2(src0_s[rrow + sub * m]);
+ const float scale0 = dd * (float)sc.s0;
+ const float scale1 = dd * (float)sc.s1;
+
+ const uint wbase = rrow + (step >> 2) * (uint)m;
+ uint qw[8];
+ #pragma unroll
+ for (int u = 0; u < 8; ++u) {
+ const uint o = wbase + (uint)u * (uint)m;
+ qw[u] = SIGN6(EXP4((uint)src0_ql[o]) | EXP2((uint)src0_qh[o]));
+ }
+
+ for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
+ const uint t = idx >> 1;
+ const uint v = idx & 1;
+ const uint c = col_base + t;
+ sh_qa4[t][v] = (c < (uint)n_no_padding)
+ ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
+ : (uint4)(0u);
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+ #define DOT_TOK(j) { \
+ const uint4 a0 = sh_qa4[b + (j)][0]; \
+ const uint4 a1 = sh_qa4[b + (j)][1]; \
+ const int raw1 = dot4_q8a(qw[0], qw[1], qw[2], qw[3], a0.x, a0.y, a0.z, a0.w); \
+ const int raw2 = dot4_q8a(qw[4], qw[5], qw[6], qw[7], a1.x, a1.y, a1.z, a1.w); \
+ rf.s##j = scale0 * (float)raw1 + scale1 * (float)raw2; \
+ }
+ DOT_TOK(0); DOT_TOK(1); DOT_TOK(2); DOT_TOK(3);
+ #undef DOT_TOK
+ const float4 ad = (float4)((float)sh_d[b+0], (float)sh_d[b+1], (float)sh_d[b+2], (float)sh_d[b+3]);
+ acc[g] += ad * rf;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) {
+ dst[(c0 + 0) * (uint)m + row] = a.s0;
+ }
+ if (c0 + 1 < (uint)n_no_padding) {
+ dst[(c0 + 1) * (uint)m + row] = a.s1;
+ }
+ if (c0 + 2 < (uint)n_no_padding) {
+ dst[(c0 + 2) * (uint)m + row] = a.s2;
+ }
+ if (c0 + 3 < (uint)n_no_padding) {
+ dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+ }
+#undef NGROUPS
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl
index a481636c2..32ff7b4f2 100644
--- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl
@@ -210,3 +210,125 @@ kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg(
}
#undef NGROUPS
}
+
+// Stage each 32-K activation block as two uint4 values.
+inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
+ int r = 0;
+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
+ return r;
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4(
+ __global const uint * src0_q,
+ __global const half * src0_d,
+ __global const uint * src1_qa,
+ __global const half * src1_da,
+ __global float * dst,
+ ulong offsetd,
+ int m,
+ int n_no_padding,
+ int k
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0);
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0;
+
+ const uint k_u = (uint)k >> 2;
+ const uint k_b = (uint)k >> 5;
+
+ __local uint4 sh_qa4[TILESIZE_N][2];
+ __local half sh_d[TILESIZE_N];
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ acc[g] = (float4)(0.0f);
+ }
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+
+ const float d_w = (float)src0_d[rrow + sub * (uint)m];
+
+ const uint wbase = rrow + (step >> 2) * (uint)m;
+ uint8 qw;
+ qw.s0 = src0_q[wbase + 0 * m];
+ qw.s1 = src0_q[wbase + 1 * m];
+ qw.s2 = src0_q[wbase + 2 * m];
+ qw.s3 = src0_q[wbase + 3 * m];
+ qw.s4 = src0_q[wbase + 4 * m];
+ qw.s5 = src0_q[wbase + 5 * m];
+ qw.s6 = src0_q[wbase + 6 * m];
+ qw.s7 = src0_q[wbase + 7 * m];
+
+ for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
+ const uint t = idx >> 1;
+ const uint v = idx & 1;
+ const uint c = col_base + t;
+ sh_qa4[t][v] = (c < (uint)n_no_padding)
+ ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
+ : (uint4)(0u);
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1])
+ rf.s0 = (float)DOTV(b+0);
+ rf.s1 = (float)DOTV(b+1);
+ rf.s2 = (float)DOTV(b+2);
+ rf.s3 = (float)DOTV(b+3);
+#undef DOTV
+ acc[g] += d_w * LD4(sh_d, b) * rf;
+ }
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) {
+ dst[(c0 + 0) * (uint)m + row] = a.s0;
+ }
+ if (c0 + 1 < (uint)n_no_padding) {
+ dst[(c0 + 1) * (uint)m + row] = a.s1;
+ }
+ if (c0 + 2 < (uint)n_no_padding) {
+ dst[(c0 + 2) * (uint)m + row] = a.s2;
+ }
+ if (c0 + 3 < (uint)n_no_padding) {
+ dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+ }
+#undef NGROUPS
+}