Commit b083c384f for llama.cpp
commit b083c384f735ee0c4b5b187edcf75a22a675370d
Author: lhez <lih@qti.qualcomm.com>
Date: Sun Oct 25 02:09:56 2026 -0700
opencl: add initial q2_k support (#30999)
* opencl: add `mul_mv_q2_k_f32`
* opencl: add q2_k SOA mv and tiled mm
* opencl: add q2_k noshuffle gemm/gemv for adreno
* opencl: add q2_k dp4a gemm for adreno
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
* opencl: tune q2_k kernels for adreno
---------
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt
index 1c6edc036..b705e4e33 100644
--- a/ggml/src/ggml-opencl/CMakeLists.txt
+++ b/ggml/src/ggml-opencl/CMakeLists.txt
@@ -99,6 +99,12 @@ set(GGML_OPENCL_KERNELS
mul_mv_q4_0_f32_1d_16x_flat
mul_mv_q4_1_f32
mul_mv_q4_1_f32_flat
+ gemm_noshuffle_q2_k_q8_1_dp4a
+ gemv_noshuffle_q2_k_f32
+ gemm_noshuffle_q2_k_f32
+ mul_mv_q2_k_f32
+ mul_mv_q2_k_f32_flat
+ mul_mm_q2_k_f32_l4_lm
mul_mv_q4_k_f32
mul_mv_q4_k_f32_flat
mul_mv_q5_0_f32
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index fc9877e2a..c441f6443 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -959,6 +959,18 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_mul_mv_q5_0_f32_flat;
cl_kernel kernel_mul_mv_q5_1_f32;
cl_kernel kernel_mul_mv_q5_1_f32_flat;
+ cl_kernel kernel_convert_block_q2_K, kernel_restore_block_q2_K;
+ cl_kernel kernel_mul_mv_q2_K_f32_flat;
+ cl_kernel kernel_mul_mm_q2_k_f32_l4_lm;
+ cl_kernel kernel_convert_block_q2_k_noshuffle;
+ cl_kernel kernel_restore_block_q2_k_noshuffle;
+ cl_kernel kernel_gemv_noshuffle_q2_k_f32 = nullptr;
+ cl_kernel kernel_gemm_noshuffle_q2_k_f32 = nullptr;
+ cl_kernel kernel_gemm_noshuffle_q2_k_q8_1_dp4a = nullptr;
+ cl_kernel kernel_gemm_noshuffle_q2_k_q8_1_dp4a_narrow = nullptr;
+ int q2_k_mv_rows = 4;
+ int q2_k_mv_nsg = 4;
+ cl_kernel kernel_mul_mv_q2_K_f32;
cl_kernel kernel_mul_mv_q4_K_f32;
cl_kernel kernel_mul_mv_q4_K_f32_flat;
cl_kernel kernel_mul_mv_q5_K_f32;
@@ -1509,6 +1521,23 @@ static void load_cl_kernels_argmax(ggml_backend_opencl_context *backend_ctx) {
}
}
+static std::string ggml_cl_q2_k_mm_options(const ggml_backend_opencl_context * backend_ctx) {
+ const bool x2 = backend_ctx->gpu_family == ADRENO &&
+ (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);
+ int bk = x2 ? 16 : 32;
+ bool half = x2;
+ if (const char * env = getenv("GGML_OPENCL_LM_BK")) {
+ const int value = atoi(env);
+ if (value == 16 || value == 32) {
+ bk = value;
+ }
+ }
+ if (const char * env = getenv("GGML_OPENCL_LM_HALF")) {
+ half = atoi(env) != 0;
+ }
+ return " -DBK=" + std::to_string(bk) + " -DLM_HALF=" + std::to_string(half ? 1 : 0);
+}
+
static void load_cl_kernels_argsort(ggml_backend_opencl_context *backend_ctx) {
// compiler options for general kernels
auto opencl_c_std =
@@ -1957,6 +1986,10 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
CL_CHECK((backend_ctx->kernel_restore_block_q5_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_K", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q5_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_K_noshuffle", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_q5_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_K_noshuffle", &err), err));
+ CL_CHECK((backend_ctx->kernel_convert_block_q2_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_k_noshuffle", &err), err));
+ CL_CHECK((backend_ctx->kernel_restore_block_q2_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_k_noshuffle", &err), err));
+ CL_CHECK((backend_ctx->kernel_convert_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_K", &err), err));
+ CL_CHECK((backend_ctx->kernel_restore_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_K", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q6_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_K", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_q6_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q6_K", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q6_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_K_noshuffle", &err), err));
@@ -2283,6 +2316,53 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
+ // mul_mv_q2_k_f32_flat
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mv_q2_k_f32_flat.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mv_q2_k_f32_flat.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_mul_mv_q2_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q2_K_f32_flat", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // mul_mm_q2_k_f32_l4_lm
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mm_q2_k_f32_l4_lm.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mm_q2_k_f32_l4_lm.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + ggml_cl_q2_k_mm_options(backend_ctx));
+ CL_CHECK((backend_ctx->kernel_mul_mm_q2_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q2_k_f32_l4_lm", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // mul_mv_q2_k_f32
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mv_q2_k_f32.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mv_q2_k_f32.cl");
+#endif
+ cl_program prog =
+ build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+
+ CL_CHECK((backend_ctx->kernel_mul_mv_q2_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q2_K_f32", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
// mul_mv_q4_k_f32
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -4091,6 +4171,80 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
+ // gemm_noshuffle_q2_k_f32
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_noshuffle_q2_k_f32.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_noshuffle_q2_k_f32.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_k_f32", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemm_noshuffle_q2_k_q8_1_dp4a
+ if (backend_ctx->has_integer_dot) {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_noshuffle_q2_k_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_noshuffle_q2_k_q8_1_dp4a.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_k_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + " -DTILESIZE_N=8");
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_k_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemv_noshuffle_q2_k_f32
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemv_noshuffle_q2_k_f32.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemv_noshuffle_q2_k_f32.cl");
+#endif
+ if (const char * env = getenv("GGML_OPENCL_Q2_K_MV_R")) {
+ const int rows = atoi(env);
+ if (rows == 1 || rows == 2 || rows == 4) {
+ backend_ctx->q2_k_mv_rows = rows;
+ }
+ }
+ if (const char * env = getenv("GGML_OPENCL_Q2_K_MV_NSG")) {
+ const int nsg = atoi(env);
+ if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) {
+ backend_ctx->q2_k_mv_nsg = nsg;
+ }
+ }
+ for (;;) {
+ const std::string opts = compile_opts + " -DQ2_K_MV_R=" + std::to_string(backend_ctx->q2_k_mv_rows) +
+ " -DQ2_K_MV_NSG=" + std::to_string(backend_ctx->q2_k_mv_nsg);
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts);
+ cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q2_k_f32", &err);
+ CL_CHECK(err);
+ CL_CHECK(clReleaseProgram(prog));
+ size_t max_workgroup;
+ CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr));
+ if (64*(size_t)backend_ctx->q2_k_mv_nsg <= max_workgroup) {
+ backend_ctx->kernel_gemv_noshuffle_q2_k_f32 = kernel;
+ break;
+ }
+ CL_CHECK(clReleaseKernel(kernel));
+ GGML_ASSERT(backend_ctx->q2_k_mv_nsg > 1);
+ backend_ctx->q2_k_mv_nsg /= 2;
+ }
+ GGML_LOG_CONT(".");
+ }
+
// gemv_noshuffle_general
{
std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +
@@ -8038,6 +8192,25 @@ struct ggml_tensor_extra_cl_q5_K {
}
};
+struct ggml_tensor_extra_cl_q2_K {
+ bool noshuffle = false;
+ cl_mem qs = nullptr; // quantized values
+ cl_mem sc = nullptr; // one uchar per 16 weights: scale nibble + min nibble
+ cl_mem dm = nullptr; // half2 per super-block: d, dmin
+
+ size_t size_qs = 0, size_sc = 0, size_dm = 0;
+
+ ~ggml_tensor_extra_cl_q2_K() { reset(); }
+
+ void reset() {
+ noshuffle = false;
+ if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; }
+ if (sc != nullptr) { CL_CHECK(clReleaseMemObject(sc)); sc = nullptr; }
+ if (dm != nullptr) { CL_CHECK(clReleaseMemObject(dm)); dm = nullptr; }
+ size_qs = size_sc = size_dm = 0;
+ }
+};
+
struct ggml_tensor_extra_cl_q6_K {
// Lower 4 bits of quantized weights.
cl_mem ql = nullptr;
@@ -9120,6 +9293,24 @@ static bool adreno_e17_compiler_quirks(const ggml_backend_opencl_context *backen
return !(env && env[0] == '0');
}
+static bool ggml_cl_q2_k_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) {
+#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS)
+ if (const char * env = getenv("GGML_OPENCL_Q2_K_NOSHUFFLE")) {
+ if (env[0] == '0') {
+ return false;
+ }
+ }
+ return tensor->type == GGML_TYPE_Q2_K &&
+ backend_ctx->gpu_family == ADRENO &&
+ ggml_is_contiguous(tensor) && !tensor->view_src &&
+ use_adreno_kernels(backend_ctx, tensor);
+#else
+ GGML_UNUSED(backend_ctx);
+ GGML_UNUSED(tensor);
+ return false;
+#endif
+}
+
inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
// The moe weight repack kernels *_trans4_ns alias a private ushort8 through a uchar*.
// Certain compilers (found with some A7x and A6x) miscompiles this, corrupting the weights.
@@ -9643,6 +9834,13 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
} else if (op->src[0]->type == GGML_TYPE_Q4_0) {
// Non-contig src0 routes through on-device dequant-to-f16.
return op->src[1]->type == GGML_TYPE_F32;
+ } else if (op->src[0]->type == GGML_TYPE_Q2_K) {
+ const ggml_tensor * weight = op->src[0]->view_src;
+ if (weight && ggml_cl_q2_k_use_noshuffle(backend_ctx, weight)) {
+ return false;
+ }
+ return op->src[1]->type == GGML_TYPE_F32 &&
+ ggml_is_contiguous_rows(op->src[1]);
} else if (op->src[0]->type == GGML_TYPE_Q4_1 ||
op->src[0]->type == GGML_TYPE_Q5_0 || op->src[0]->type == GGML_TYPE_Q5_1 ||
op->src[0]->type == GGML_TYPE_MXFP4 ||
@@ -10011,9 +10209,15 @@ struct ggml_backend_opencl_buffer_context {
for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K) {
delete e;
}
+ for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K) {
+ delete e;
+ }
for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) {
delete e;
}
+ for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K_in_use) {
+ delete e;
+ }
for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K) {
delete e;
}
@@ -10187,6 +10391,21 @@ struct ggml_backend_opencl_buffer_context {
return extra;
}
+ ggml_tensor_extra_cl_q2_K * ggml_opencl_alloc_temp_tensor_extra_q2_K() {
+ ggml_tensor_extra_cl_q2_K * extra;
+ if (temp_tensor_extras_q2_K.empty()) {
+ extra = new ggml_tensor_extra_cl_q2_K();
+ } else {
+ extra = temp_tensor_extras_q2_K.back();
+ temp_tensor_extras_q2_K.pop_back();
+ }
+
+ temp_tensor_extras_q2_K_in_use.push_back(extra);
+
+ extra->reset();
+ return extra;
+ }
+
ggml_tensor_extra_cl_q6_K * ggml_opencl_alloc_temp_tensor_extra_q6_K() {
ggml_tensor_extra_cl_q6_K * extra;
if (temp_tensor_extras_q6_K.empty()) {
@@ -10263,6 +10482,11 @@ struct ggml_backend_opencl_buffer_context {
}
temp_tensor_extras_q6_K_in_use.clear();
+ for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K_in_use) {
+ temp_tensor_extras_q2_K.push_back(e);
+ }
+ temp_tensor_extras_q2_K_in_use.clear();
+
q8_0_soa_tensors.clear();
q4_0_soa_tensors.clear();
}
@@ -10294,6 +10518,8 @@ struct ggml_backend_opencl_buffer_context {
std::vector<ggml_tensor_extra_cl_q4_K *> temp_tensor_extras_q4_K_in_use;
std::vector<ggml_tensor_extra_cl_q5_K *> temp_tensor_extras_q5_K;
std::vector<ggml_tensor_extra_cl_q5_K *> temp_tensor_extras_q5_K_in_use;
+ std::vector<ggml_tensor_extra_cl_q2_K *> temp_tensor_extras_q2_K;
+ std::vector<ggml_tensor_extra_cl_q2_K *> temp_tensor_extras_q2_K_in_use;
std::vector<ggml_tensor_extra_cl_q6_K *> temp_tensor_extras_q6_K;
std::vector<ggml_tensor_extra_cl_q6_K *> temp_tensor_extras_q6_K_in_use;
@@ -10473,6 +10699,83 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
cl_command_queue queue = backend_ctx->queue;
#ifdef GGML_OPENCL_SOA_Q
+ if (tensor->type == GGML_TYPE_Q2_K) {
+ GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor));
+ ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;
+ GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized");
+
+ ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;
+ ggml_tensor_extra_cl_q2_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q2_K();
+ extra->noshuffle = ggml_cl_q2_k_use_noshuffle(backend_ctx, tensor);
+
+ const size_t blck = (size_t)ggml_blck_size(tensor->type);
+ const size_t n_blk = ggml_nelements(tensor) / blck;
+
+ const size_t size_qs = n_blk * (blck/4);
+ const size_t size_sc = n_blk * (blck/16);
+ const size_t size_dm = n_blk * 2 * sizeof(ggml_fp16_t);
+ GGML_ASSERT(size_qs + size_sc + size_dm == ggml_nbytes(tensor) && "Incorrect tensor size");
+
+ cl_int err;
+ cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,
+ ggml_nbytes(tensor), NULL, &err);
+ CL_CHECK(err);
+ CL_CHECK(clEnqueueWriteBuffer(
+ queue, data_device, CL_TRUE, 0,
+ ggml_nbytes(tensor), data, 0, NULL, NULL));
+
+ cl_buffer_region region;
+ region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment);
+ region.size = size_qs;
+ CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+ auto previous_origin = region.origin;
+
+ region.origin = align_to(previous_origin + size_qs, backend_ctx->alignment);
+ region.size = size_sc;
+ CL_CHECK((extra->sc = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+ previous_origin = region.origin;
+
+ region.origin = align_to(previous_origin + size_sc, backend_ctx->alignment);
+ region.size = size_dm;
+ CL_CHECK((extra->dm = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+
+ cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_convert_block_q2_k_noshuffle
+ : backend_ctx->kernel_convert_block_q2_K;
+ cl_ulong nb_arg = (cl_ulong)n_blk;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sc));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->dm));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg));
+
+ size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 };
+ size_t local_work_size[] = { 64, 1, 1 };
+
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clReleaseEvent(evt));
+ CL_CHECK(clReleaseMemObject(data_device));
+
+
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ if (extra->noshuffle) {
+ const int m = tensor->ne[1];
+ const int k = tensor->ne[0];
+ transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true);
+ transpose_2d_as_8b(backend_ctx, extra->sc, extra->sc, size_sc, k/16, m, true, true);
+ transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->dm, extra->dm, size_dm, k/(int)blck, m, true, true);
+ }
+#endif
+
+ extra->size_qs = size_qs;
+ extra->size_sc = size_sc;
+ extra->size_dm = size_dm;
+
+ tensor->extra = extra;
+ return;
+ }
+
if (tensor->type == GGML_TYPE_Q1_0) {
ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;
GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized");
@@ -12143,6 +12446,52 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,
sync_with_other_backends(backend_ctx);
#ifdef GGML_OPENCL_SOA_Q
+ if (tensor->type == GGML_TYPE_Q2_K) {
+ ggml_tensor_extra_cl_q2_K * extra = (ggml_tensor_extra_cl_q2_K *)tensor->extra;
+
+ const size_t blck = (size_t)ggml_blck_size(tensor->type);
+
+ ggml_cl_buffer buf_unpacked;
+ buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor));
+
+ const size_t n_blk = (size_t)ggml_nelements(tensor) / blck;
+ cl_ulong nb_arg = (cl_ulong)n_blk;
+
+ cl_mem qs = extra->qs, sc = extra->sc, dm = extra->dm;
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ ggml_cl_buffer buf_qs, buf_sc, buf_dm;
+ if (extra->noshuffle) {
+ buf_qs.allocate(context, extra->size_qs);
+ buf_sc.allocate(context, extra->size_sc);
+ buf_dm.allocate(context, extra->size_dm);
+ qs = buf_qs.buffer;
+ sc = buf_sc.buffer;
+ dm = buf_dm.buffer;
+ const int m = tensor->ne[1];
+ const int k = tensor->ne[0];
+ transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true);
+ transpose_2d_as_8b(backend_ctx, extra->sc, sc, extra->size_sc, m, k/16, true, true);
+ transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->dm, dm, extra->size_dm, m, k/(int)blck, true, true);
+ }
+#endif
+ cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_restore_block_q2_k_noshuffle
+ : backend_ctx->kernel_restore_block_q2_K;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sc));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &dm));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg));
+
+ size_t gws[] = { (n_blk + 63) / 64 * 64 };
+ size_t lws[] = { 64 };
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clReleaseEvent(evt));
+ CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL));
+ return;
+ }
+
if (tensor->type == GGML_TYPE_Q1_0) {
ggml_tensor_extra_cl_q1_0 * extra = (ggml_tensor_extra_cl_q1_0 *)tensor->extra;
@@ -24455,6 +24804,166 @@ static cl_mem ggml_cl_img_pool_get_or_create(
return img;
}
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+static bool ggml_cl_mul_mat_q2_K_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
+ ggml_tensor_extra_cl_q2_K * extra0 = (ggml_tensor_extra_cl_q2_K *)src0->extra;
+ ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
+
+ cl_ulong offset1 = extra1->offset + src1->view_offs;
+ cl_ulong offsetd = extrad->offset + dst->view_offs;
+ const int m = src0->ne[1];
+ const int n = src1->ne[1];
+ const int k = src0->ne[0];
+
+ const int tile_n = n <= 16 ? 8 : 32;
+ cl_kernel gemm = n <= 16 ? backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a_narrow
+ : backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a;
+ const char * env = getenv("GGML_OPENCL_Q2_K_DENSE_DP4A");
+ if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') ||
+ n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
+ return false;
+ }
+
+ cl_int err;
+ cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) };
+ cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+ const size_t n_blocks = (size_t)n*(k/32);
+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k);
+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half));
+
+ cl_int nb = n_blocks;
+ cl_kernel quant = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation));
+ CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb));
+ size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 };
+ size_t q_local[] = { 64 };
+ backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst);
+
+ CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs));
+ CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sc));
+ CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->dm));
+ CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(gemm, 7, sizeof(int), &m));
+ CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &n));
+ CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &k));
+ size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) };
+ size_t local[] = { 64, 1, 1 };
+ backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst);
+ CL_CHECK(clReleaseMemObject(activation));
+ return true;
+}
+
+static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
+ ggml_tensor_extra_cl_q2_K * extra0 = (ggml_tensor_extra_cl_q2_K *)src0->extra;
+ ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
+
+ cl_ulong offset1 = extra1->offset + src1->view_offs;
+ cl_ulong offsetd = extrad->offset + dst->view_offs;
+ const int m = src0->ne[1];
+ const int n = src1->ne[1];
+ const int k = src0->ne[0];
+
+ if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
+ cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_q2_k_f32;
+ const int stride_d = dst->ne[0];
+ const int stride_b = src1->nb[1]/sizeof(float);
+ CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs));
+ CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->sc));
+ CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->dm));
+ CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(gemv, 7, sizeof(int), &k));
+ CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &m));
+ CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &stride_b));
+ CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_d));
+ const size_t rows = 64*(size_t)backend_ctx->q2_k_mv_rows;
+ const size_t nsg = backend_ctx->q2_k_mv_nsg;
+ size_t global[] = { CEIL_DIV((size_t)m, rows)*64, (size_t)n*nsg, 1 };
+ size_t local[] = { 64, nsg, 1 };
+ backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst);
+ return;
+ }
+
+ if (ggml_cl_mul_mat_q2_K_dp4a(backend, src0, src1, dst)) {
+ return;
+ }
+
+ cl_context context = backend_ctx->context;
+ cl_int err;
+ cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) };
+ cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+
+ cl_image_format img_fmt = { CL_RGBA, CL_FLOAT };
+ cl_image_desc img_desc = {};
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = (size_t)k*n/4;
+ img_desc.buffer = b_sub_buf;
+ cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err);
+ CL_CHECK(err);
+
+ const int padded_n = CEIL_DIV(n, 8)*8;
+ region.origin = 0;
+ region.size = (size_t)k*padded_n*sizeof(cl_half);
+ backend_ctx->prealloc_act_trans.allocate(context, region.size);
+ cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+
+ img_fmt = { CL_RGBA, CL_HALF_FLOAT };
+ img_desc.image_width = (size_t)k*padded_n/4;
+ img_desc.buffer = b_sub_buf_trans;
+ cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err);
+ CL_CHECK(err);
+
+ const int height_b = std::max(n/4, 1);
+ const int width_b = k/4;
+ const int padded_height_b = padded_n/4;
+ cl_kernel kernel = backend_ctx->kernel_transpose_32_16;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b));
+ size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b };
+ size_t t_local[] = { 1, 16 };
+ backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst);
+
+ kernel = backend_ctx->kernel_gemm_noshuffle_q2_k_f32;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sc));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->dm));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &m));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &padded_n));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &k));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n));
+ size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 };
+ size_t local[] = { 1, 128, 1 };
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst);
+
+ CL_CHECK(clReleaseMemObject(b_img_trans));
+ CL_CHECK(clReleaseMemObject(b_sub_buf_trans));
+ CL_CHECK(clReleaseMemObject(b_img));
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+}
+#endif
+
static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
GGML_ASSERT(src0);
GGML_ASSERT(src0->extra);
@@ -24523,6 +25032,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)soa0_src->extra;
ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra;
ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)soa0_src->extra;
+ ggml_tensor_extra_cl_q2_K * extra0_q2_K = (ggml_tensor_extra_cl_q2_K *)soa0_src->extra;
ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)soa0_src->extra;
ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)soa0_src->extra;
ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)soa0_src->extra;
@@ -24553,6 +25063,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
ggml_tensor d = *dst;
d.ne[2] = 1; d.ne[3] = 1;
d.view_offs = dst->view_offs + (size_t)i12*nb2 + (size_t)i13*nb3;
+ d.src[1] = &s1;
ggml_cl_mul_mat(backend, src0, &s1, &d);
}
}
@@ -24917,6 +25428,12 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
return;
}
+ // q2_k x fp32
+ if (src0t == GGML_TYPE_Q2_K && src1t == GGML_TYPE_F32 && extra0_q2_K->noshuffle) {
+ ggml_cl_mul_mat_q2_k_f32_adreno(backend, src0, src1, dst);
+ return;
+ }
+
// q4_k x fp32
if (src0t == GGML_TYPE_Q4_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q4_K(backend_ctx, src0)) {
ggml_cl_mul_mat_q4_k_f32_adreno(backend, src0, src1, dst);
@@ -25423,6 +25940,50 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
return;
}
+ case GGML_TYPE_Q2_K: {
+ if (ne11 < 32) {
+ break;
+ }
+ if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {
+ break;
+ }
+
+ kernel = backend_ctx->kernel_mul_mm_q2_k_f32_l4_lm;
+ nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128;
+ offset0 = src0->view_offs;
+
+ int batch_stride_a = ne00*ne01;
+ int batch_stride_b = ne10*ne11;
+ int batch_stride_d = ne0*ne1;
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q2_K->qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q2_K->sc));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q2_K->dm));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d));
+ CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2));
+ CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3));
+
+ size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13};
+ size_t local_work_size[] = {(size_t)nth0, 1, 1};
+
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+ return;
+ }
case GGML_TYPE_Q4_K: {
if (ne11 < 32) {
break;
@@ -26339,7 +26900,69 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
#endif // GGML_OPENCL_SOA_Q
break;
}
- case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q2_K: {
+ if (backend_ctx->gpu_family == INTEL) {
+ nth0 = 16;
+ nth1 = 1;
+ ndst = 4;
+ } else if (backend_ctx->gpu_family == ADRENO) {
+ nth0 = 64;
+ nth1 = 1;
+ ndst = 4;
+ } else {
+ GGML_ASSERT(false && "TODO: Unknown GPU");
+ }
+
+#ifdef GGML_OPENCL_SOA_Q
+ kernel = backend_ctx->kernel_mul_mv_q2_K_f32_flat;
+ offset0 = src0->view_offs;
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q2_K->qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q2_K->sc));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q2_K->dm));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1));
+ CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2));
+ CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3));
+#else
+ kernel = backend_ctx->kernel_mul_mv_q2_K_f32;
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));
+#endif // GGML_OPENCL_SOA_Q
+ break;
+ }
case GGML_TYPE_Q3_K:
case GGML_TYPE_Q4_K: {
#ifdef GGML_OPENCL_SOA_Q
diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.cl
index acc8f9807..f3309a298 100644
--- a/ggml/src/ggml-opencl/kernels/cvt.cl
+++ b/ggml/src/ggml-opencl/kernels/cvt.cl
@@ -2661,3 +2661,115 @@ kernel void kernel_moe_expand_scale_q5_K(
dst_min[((long)e*ne01 + row)*nblk32 + sub] = (half)(dm * (float)mn);
}
}
+
+struct block_q2_K {
+ uint8_t scales[QK_K/16]; // low nibble scale, high nibble min, one per 16 weights
+ uint8_t qs[QK_K/4]; // quants, 2 bits each
+ half d; // super-block scale for the quantized scales
+ half dmin; // super-block scale for the quantized mins
+};
+
+kernel void kernel_convert_block_q2_K(
+ global struct block_q2_K * src0,
+ global uchar * dst_qs,
+ global uchar * dst_sc,
+ global half * dst_dm,
+ ulong n_blk
+) {
+ const ulong i = get_global_id(0);
+ if (i >= n_blk) {
+ return;
+ }
+ global struct block_q2_K * b = src0 + i;
+ for (int j = 0; j < 64; ++j) {
+ dst_qs[64*i + j] = b->qs[j];
+ }
+ for (int j = 0; j < 16; ++j) {
+ dst_sc[16*i + j] = b->scales[j];
+ }
+ dst_dm[2*i + 0] = b->d;
+ dst_dm[2*i + 1] = b->dmin;
+}
+
+kernel void kernel_restore_block_q2_K(
+ global uchar * src_qs,
+ global uchar * src_sc,
+ global half * src_dm,
+ global struct block_q2_K * dst,
+ ulong n_blk
+) {
+ const ulong i = get_global_id(0);
+ if (i >= n_blk) {
+ return;
+ }
+ global struct block_q2_K * b = dst + i;
+ for (int j = 0; j < 64; ++j) {
+ b->qs[j] = src_qs[64*i + j];
+ }
+ for (int j = 0; j < 16; ++j) {
+ b->scales[j] = src_sc[16*i + j];
+ }
+ b->d = src_dm[2*i + 0];
+ b->dmin = src_dm[2*i + 1];
+}
+
+kernel void kernel_convert_block_q2_k_noshuffle(
+ global struct block_q2_K * src0,
+ global uchar * dst_qs, // QK_K/4 per block
+ global uchar * dst_sc, // QK_K/16 per block
+ global half * dst_dm, // 2 per block (d, dmin)
+ ulong n_blk
+) {
+ const ulong i = get_global_id(0);
+ if (i >= n_blk) {
+ return;
+ }
+ global struct block_q2_K * b = (global struct block_q2_K *) src0 + i;
+
+ dst_dm[2*i + 0] = b->d;
+ dst_dm[2*i + 1] = b->dmin;
+
+ for (int j = 0; j < QK_K/16; ++j) { dst_sc[(QK_K/16) * i + j] = b->scales[j]; }
+
+ for (int g = 0; g < QK_K/4; ++g) {
+ uint pk = 0;
+ for (int t = 0; t < 4; ++t) {
+ const int e = 4*g + t;
+ const int sb = e >> 5;
+ const int ee = e & 31;
+ pk |= (uint)(((b->qs[32*(sb >> 2) + ee] >> (2*(sb & 3))) & 3) << (2*t));
+ }
+ dst_qs[(QK_K/4) * i + g] = (uchar)pk;
+ }
+}
+
+kernel void kernel_restore_block_q2_k_noshuffle(
+ global uchar * src_qs,
+ global uchar * src_sc,
+ global half * src_dm,
+ global struct block_q2_K * dst,
+ ulong n_blk
+) {
+ const ulong i = get_global_id(0);
+ if (i >= n_blk) {
+ return;
+ }
+ global struct block_q2_K * b = (global struct block_q2_K *) dst + i;
+
+ b->d = src_dm[2*i + 0];
+ b->dmin = src_dm[2*i + 1];
+
+ for (int j = 0; j < QK_K/16; ++j) { b->scales[j] = src_sc[(QK_K/16) * i + j]; }
+
+ for (int n2 = 0; n2 < 2; ++n2) {
+ for (int ee = 0; ee < 32; ++ee) {
+ uint v = 0;
+ for (int j = 0; j < 4; ++j) {
+ const int sb = 4*n2 + j;
+ const int e = 32*sb + ee;
+ v |= ((uint)(src_qs[(QK_K/4) * i + (e >> 2)] >> (2*(e & 3))) & 3u) << (2*j);
+ }
+ b->qs[32*n2 + ee] = (uchar)v;
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_f32.cl
new file mode 100644
index 000000000..2749e1c14
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_f32.cl
@@ -0,0 +1,83 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_qcom_reqd_sub_group_size
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+__attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+kernel void kernel_gemm_noshuffle_q2_k_f32(
+ global const uchar * src0_qs,
+ global const uchar * src0_sc,
+ global const half * src0_dm,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int m,
+ int n,
+ int k,
+ int n_no_padding
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+ const int gy = get_global_id(0);
+ const int row = get_global_id(1) * 4;
+ const int n_4 = n / 4;
+ if (row >= m) {
+ return;
+ }
+
+ half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
+ for (int i = 0; i < k; i += 16) {
+ const half8 dm = vload8(0, src0_dm + 2 * (row + (i / 256) * m));
+ const uchar4 sc = vload4(0, src0_sc + row + (i / 16) * m);
+ const half4 scale = convert_half4(convert_float4(dm.s0246) * convert_float4(sc & (uchar4)(15)));
+ const half4 mval = convert_half4(convert_float4(dm.s1357) * convert_float4(sc >> (uchar4)(4)));
+
+ for (int l = 0; l < 16; l += 4) {
+ const int ki = i + l;
+ const uchar4 bits = vload4(0, src0_qs + row + (ki / 4) * m);
+ #pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ half8 b;
+ b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4);
+ b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
+ const half4 w = convert_half4((bits >> (uchar4)(2 * j)) & (uchar4)(3)) * scale - mval;
+ c0 += b * w.s0;
+ c1 += b * w.s1;
+ c2 += b * w.s2;
+ c3 += b * w.s3;
+ }
+ }
+ }
+
+ int idx = gy * 8 * m + row;
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_q8_1_dp4a.cl
new file mode 100644
index 000000000..7db7e6d71
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q2_k_q8_1_dp4a.cl
@@ -0,0 +1,146 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#define QK_K 256
+
+#ifndef TILESIZE_N
+#define TILESIZE_N 32
+#endif
+
+inline uint q2_k_pack(uint pk) {
+ return ( (pk ) & 3u)
+ | (((pk >> 2) & 3u) << 8)
+ | (((pk >> 4) & 3u) << 16)
+ | (((pk >> 6) & 3u) << 24);
+}
+
+// Pass staged operands by value; local dp4a operands miscompile on X2.
+#define KQ_STAGE(p) vload4(0, (p))
+
+inline int dot4_q8a_v(uint4 qw, uint4 a) {
+ int r = 0;
+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r);
+ return r;
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_q2_k_q8_1_dp4a(
+ __global const uchar * src0_qs,
+ __global const uchar * src0_sc,
+ __global const half * src0_dm, // half pairs: d, dmin
+ __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K]
+ __global const half * src1_da, // q8_1 per-block scale [N, K/32]
+ __global float * dst,
+ ulong offsetd,
+ int m,
+ int n_no_padding,
+ int k
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0);
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0;
+
+ const uint k_u = (uint)k >> 2;
+ const uint k_b = (uint)k >> 5;
+
+ __local uint4 sh_qa4[TILESIZE_N][2];
+ __local float sh_s [TILESIZE_N][2]; // per-16 activation sums, in qa units
+ __local half sh_d [TILESIZE_N];
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+ const uint ib = sub >> 3;
+
+ const half2 dm = vload2(rrow + ib * (uint)m, src0_dm);
+ const float dv = (float)dm.s0;
+ const float dmv = (float)dm.s1;
+
+ const uint scb = rrow + (sub << 1) * (uint)m;
+ const uint sc0 = (uint)src0_sc[scb + 0u * (uint)m];
+ const uint sc1 = (uint)src0_sc[scb + 1u * (uint)m];
+ const float dl0 = dv * (float)( sc0 & 0xFu);
+ const float ml0 = dmv * (float)( sc0 >> 4);
+ const float dl1 = dv * (float)( sc1 & 0xFu);
+ const float ml1 = dmv * (float)( sc1 >> 4);
+
+ const uint qsb = rrow + (step >> 2) * (uint)m;
+ uint4 qlo, qhi;
+ qlo.s0 = q2_k_pack((uint)src0_qs[qsb + 0u * (uint)m]);
+ qlo.s1 = q2_k_pack((uint)src0_qs[qsb + 1u * (uint)m]);
+ qlo.s2 = q2_k_pack((uint)src0_qs[qsb + 2u * (uint)m]);
+ qlo.s3 = q2_k_pack((uint)src0_qs[qsb + 3u * (uint)m]);
+ qhi.s0 = q2_k_pack((uint)src0_qs[qsb + 4u * (uint)m]);
+ qhi.s1 = q2_k_pack((uint)src0_qs[qsb + 5u * (uint)m]);
+ qhi.s2 = q2_k_pack((uint)src0_qs[qsb + 6u * (uint)m]);
+ qhi.s3 = q2_k_pack((uint)src0_qs[qsb + 7u * (uint)m]);
+
+ for (uint idx = lid; idx < TILESIZE_N * 2u; idx += 64u) {
+ const uint t = idx >> 1;
+ const uint h = idx & 1u;
+ const uint c = col_base + t;
+ const bool ok = c < (uint)n_no_padding;
+ const uint4 w = ok ? KQ_STAGE(src1_qa + c * k_u + (step >> 2) + (h << 2))
+ : (uint4)(0u);
+ sh_qa4[t][h] = w;
+ int s = 0;
+ s = dot_acc_sat_4x8packed_ss_int(w.x, 0x01010101u, s);
+ s = dot_acc_sat_4x8packed_ss_int(w.y, 0x01010101u, s);
+ s = dot_acc_sat_4x8packed_ss_int(w.z, 0x01010101u, s);
+ s = dot_acc_sat_4x8packed_ss_int(w.w, 0x01010101u, s);
+ sh_s[t][h] = (float)s;
+ if (h == 0u) {
+ sh_d[t] = ok ? src1_da[c * k_b + sub] : (half)0;
+ }
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+#define Q2_K_COL(b) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) - ml0 * sh_s[b][0] + \
+ dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1])) - ml1 * sh_s[b][1])
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+ rf.s0 = Q2_K_COL(b+0); rf.s1 = Q2_K_COL(b+1);
+ rf.s2 = Q2_K_COL(b+2); rf.s3 = Q2_K_COL(b+3);
+ acc[g] += LD4(sh_d, b) * rf;
+ }
+#undef Q2_K_COL
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+#undef NGROUPS
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_k_f32.cl
new file mode 100644
index 000000000..efcc31077
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_k_f32.cl
@@ -0,0 +1,231 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#define QK_K 256
+
+#ifndef Q2_K_MV_NSG
+#define Q2_K_MV_NSG 8
+#endif
+
+#ifndef Q2_K_MV_R
+#define Q2_K_MV_R 4
+#endif
+
+inline float4 q2_k_vals(uint pk) {
+ return (float4)((float)( pk & 3u), (float)((pk >> 2) & 3u),
+ (float)((pk >> 4) & 3u), (float)((pk >> 6) & 3u));
+}
+
+kernel void kernel_gemv_noshuffle_q2_k_f32(
+ global const uchar * src0_qs,
+ global const uchar * src0_sc,
+ global const half * src0_dm,
+ global const float * src1,
+ ulong offset1,
+ global float * dst,
+ ulong offsetd,
+ int ne00, // K
+ int ne01, // M
+ int ne10, // activation row stride, == K
+ int ne0 // dst row stride
+) {
+ src1 = (global const float *)((global const char *)src1 + offset1);
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint m = (uint)ne01;
+ const uint K = (uint)ne00;
+ const uint nsb = K / QK_K;
+
+ const uint lid = get_local_id(0);
+ const uint sgi = get_local_id(1);
+ const uint col = get_group_id(1);
+
+ global const float * y = src1 + (ulong)col * (uint)ne10;
+
+ const uint mr = m / Q2_K_MV_R; // row groups
+ const uint j = get_group_id(0) * 64u + lid; // row group this lane owns
+ const uint row = j * Q2_K_MV_R;
+
+#if Q2_K_MV_R == 4
+ float sumf0 = 0.f;
+ float sumf1 = 0.f;
+ float sumf2 = 0.f;
+ float sumf3 = 0.f;
+
+ if (j < mr) {
+ global const uint * qsu = (global const uint *)src0_qs;
+ global const uint * scu = (global const uint *)src0_sc;
+
+ for (uint ib = sgi; ib < nsb; ib += Q2_K_MV_NSG) {
+ const half4 dma = vload4(2u*(j + ib*mr) + 0u, src0_dm); // d0, dmin0, d1, dmin1
+ const half4 dmb = vload4(2u*(j + ib*mr) + 1u, src0_dm); // d2, dmin2, d3, dmin3
+
+ float ad0 = 0.f, am0 = 0.f, ad1 = 0.f, am1 = 0.f;
+ float ad2 = 0.f, am2 = 0.f, ad3 = 0.f, am3 = 0.f;
+
+ for (uint sb = 0; sb < 8u; ++sb) {
+ const uint grp = ib * 64u + sb * 8u;
+ const uint qsb = j + grp * mr;
+
+ for (uint h = 0; h < 2u; ++h) {
+ const uint scv = scu[j + (2u * (ib * 8u + sb) + h) * mr];
+
+ float a0 = 0.f, a1 = 0.f, a2 = 0.f, a3 = 0.f;
+ float4 as = (float4)(0.f);
+ for (uint u = 0; u < 4u; ++u) {
+ const uint gg = 4u*h + u;
+ const uint qsv = qsu[qsb + gg * mr]; // four rows, one load
+ const float4 yv = vload4(grp + gg, y);
+ as += yv;
+ a0 += dot(yv, q2_k_vals( qsv & 0xFFu));
+ a1 += dot(yv, q2_k_vals((qsv >> 8) & 0xFFu));
+ a2 += dot(yv, q2_k_vals((qsv >> 16) & 0xFFu));
+ a3 += dot(yv, q2_k_vals((qsv >> 24) & 0xFFu));
+ }
+ const float asum = as.s0 + as.s1 + as.s2 + as.s3;
+
+ const uint s0 = scv & 0xFFu;
+ const uint s1 = (scv >> 8) & 0xFFu;
+ const uint s2 = (scv >> 16) & 0xFFu;
+ const uint s3 = (scv >> 24) & 0xFFu;
+ ad0 += (float)(s0 & 0xFu) * a0; am0 += (float)(s0 >> 4) * asum;
+ ad1 += (float)(s1 & 0xFu) * a1; am1 += (float)(s1 >> 4) * asum;
+ ad2 += (float)(s2 & 0xFu) * a2; am2 += (float)(s2 >> 4) * asum;
+ ad3 += (float)(s3 & 0xFu) * a3; am3 += (float)(s3 >> 4) * asum;
+ }
+ }
+ sumf0 += (float)dma.s0 * ad0 - (float)dma.s1 * am0;
+ sumf1 += (float)dma.s2 * ad1 - (float)dma.s3 * am1;
+ sumf2 += (float)dmb.s0 * ad2 - (float)dmb.s1 * am2;
+ sumf3 += (float)dmb.s2 * ad3 - (float)dmb.s3 * am3;
+ }
+ }
+
+#if Q2_K_MV_NSG > 1
+ __local float4 part[Q2_K_MV_NSG][64];
+ part[sgi][lid] = (float4)(sumf0, sumf1, sumf2, sumf3);
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (sgi != 0) {
+ return;
+ }
+ for (uint s = 1; s < Q2_K_MV_NSG; ++s) {
+ const float4 p = part[s][lid];
+ sumf0 += p.s0;
+ sumf1 += p.s1;
+ sumf2 += p.s2;
+ sumf3 += p.s3;
+ }
+#endif
+
+ if (j < mr) {
+ global float * o = dst + (ulong)col * (uint)ne0 + row;
+ o[0] = sumf0; o[1] = sumf1; o[2] = sumf2; o[3] = sumf3;
+ }
+
+#elif Q2_K_MV_R == 2
+ float sumf0 = 0.f;
+ float sumf1 = 0.f;
+
+ if (j < mr) {
+ global const ushort * qsu = (global const ushort *)src0_qs;
+ global const ushort * scu = (global const ushort *)src0_sc;
+
+ for (uint ib = sgi; ib < nsb; ib += Q2_K_MV_NSG) {
+ const half4 dm = vload4(j + ib * mr, src0_dm); // d0, dmin0, d1, dmin1
+
+ float ad0 = 0.f, am0 = 0.f, ad1 = 0.f, am1 = 0.f;
+ for (uint sb = 0; sb < 8u; ++sb) {
+ const uint grp = ib * 64u + sb * 8u;
+ const uint qsb = j + grp * mr;
+
+ for (uint h = 0; h < 2u; ++h) {
+ const uint scv = (uint)scu[j + (2u * (ib * 8u + sb) + h) * mr];
+ const uint s0 = scv & 0xFFu;
+ const uint s1 = scv >> 8;
+
+ float a0 = 0.f, a1 = 0.f;
+ float4 as = (float4)(0.f);
+ for (uint u = 0; u < 4u; ++u) {
+ const uint gg = 4u*h + u;
+ const uint qsv = (uint)qsu[qsb + gg * mr];
+ const float4 yv = vload4(grp + gg, y);
+ as += yv;
+ a0 += dot(yv, q2_k_vals( qsv & 0xFFu));
+ a1 += dot(yv, q2_k_vals((qsv >> 8) & 0xFFu));
+ }
+ const float asum = as.s0 + as.s1 + as.s2 + as.s3;
+ ad0 += (float)(s0 & 0xFu) * a0; am0 += (float)(s0 >> 4) * asum;
+ ad1 += (float)(s1 & 0xFu) * a1; am1 += (float)(s1 >> 4) * asum;
+ }
+ }
+ sumf0 += (float)dm.s0 * ad0 - (float)dm.s1 * am0;
+ sumf1 += (float)dm.s2 * ad1 - (float)dm.s3 * am1;
+ }
+ }
+
+#if Q2_K_MV_NSG > 1
+ __local float2 part[Q2_K_MV_NSG][64];
+ part[sgi][lid] = (float2)(sumf0, sumf1);
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (sgi != 0) {
+ return;
+ }
+ for (uint s = 1; s < Q2_K_MV_NSG; ++s) {
+ const float2 p = part[s][lid];
+ sumf0 += p.s0;
+ sumf1 += p.s1;
+ }
+#endif
+
+ if (j < mr) {
+ vstore2((float2)(sumf0, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
+ }
+
+#else
+ float sumf0 = 0.f;
+
+ if (j < mr) {
+ for (uint ib = sgi; ib < nsb; ib += Q2_K_MV_NSG) {
+ const half2 dm = vload2(row + ib * m, src0_dm);
+
+ float ad = 0.f, am = 0.f;
+ for (uint sb = 0; sb < 8u; ++sb) {
+ const uint grp = ib * 64u + sb * 8u;
+ const uint qsb = row + grp * m;
+
+ for (uint h = 0; h < 2u; ++h) {
+ const uint sc = (uint)src0_sc[row + (2u * (ib * 8u + sb) + h) * m];
+
+ float a = 0.f;
+ float4 as = (float4)(0.f);
+ for (uint u = 0; u < 4u; ++u) {
+ const uint gg = 4u*h + u;
+ const float4 yv = vload4(grp + gg, y);
+ as += yv;
+ const uint pkv = (uint)src0_qs[qsb + gg * m];
+ a += dot(yv, q2_k_vals(pkv));
+ }
+ const float asum = as.s0 + as.s1 + as.s2 + as.s3;
+ ad += (float)(sc & 0xFu) * a; am += (float)(sc >> 4) * asum;
+ }
+ }
+ sumf0 += (float)dm.s0 * ad - (float)dm.s1 * am;
+ }
+ }
+
+#if Q2_K_MV_NSG > 1
+ __local float part[Q2_K_MV_NSG][64];
+ part[sgi][lid] = sumf0;
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (sgi != 0) {
+ return;
+ }
+ for (uint s = 1; s < Q2_K_MV_NSG; ++s) {
+ sumf0 += part[s][lid];
+ }
+#endif
+
+ if (j < mr) {
+ dst[(ulong)col * (uint)ne0 + row] = sumf0;
+ }
+#endif
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q2_k_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q2_k_f32_l4_lm.cl
new file mode 100644
index 000000000..e192bdefb
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mm_q2_k_f32_l4_lm.cl
@@ -0,0 +1,203 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_required_subgroup_size
+#define INTEL_GPU 1
+#endif
+
+#define QK_K 256
+
+struct block_q2_K {
+ uchar scales[QK_K/16];
+ uchar qs[QK_K/4];
+ half d;
+ half dmin;
+};
+
+#define LOAD_VEC_A 4
+#define LOAD_VEC_B 4
+
+#define BM 64
+#define BN 64
+#ifndef BK
+#define BK 32
+#endif
+#ifdef INTEL_GPU
+#define TM 8
+#else
+#define TM 4
+#endif
+#define TN 8
+
+#ifndef LM_HALF
+#define LM_HALF 0
+#endif
+#if LM_HALF
+typedef half lm_st;
+#define LM_LD4(p, i) convert_float4(vload4((i), (p)))
+#else
+typedef float lm_st;
+#define LM_LD4(p, i) vload4((i), (p))
+#endif
+
+kernel void kernel_mul_mm_q2_k_f32_l4_lm(
+ global uchar * src0_qs,
+ global uchar * src0_sc,
+ global half * src0_dm,
+ ulong offset0,
+ global float4 * src1,
+ ulong offset1,
+ global float * dst,
+ ulong offsetd,
+
+ int ne00,
+ int ne01,
+ int ne02,
+ int ne11,
+ int ne12,
+
+ int stride_a,
+ int stride_b,
+ int stride_d,
+
+ int batch_stride_a,
+ int batch_stride_b,
+ int batch_stride_d,
+
+ int r2,
+ int r3
+) {
+ src1 = (global float4*)((global char*)src1 + offset1);
+ dst = (global float *)((global char*)dst + offsetd);
+
+ local lm_st buf_a[BM * BK];
+ local lm_st buf_b[BN * BK];
+
+ const int batch_idx = get_global_id(2);
+
+ const int i13 = batch_idx / ne12;
+ const int i12 = batch_idx % ne12;
+
+ const int i03 = i13 / r3;
+ const int i02 = i12 / r2;
+
+ const int batch_idx_a = i03 * ne02 + i02;
+
+ const int ir = get_group_id(0);
+ const int ic = get_group_id(1);
+
+ const int tid = get_local_id(0);
+ const int th_r = tid % (BM / TM);
+ const int th_c = tid / (BM / TM);
+
+ const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
+ const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
+ const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
+ const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
+
+ const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
+ const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
+
+ // pos_a counts elements, not blocks - one q2_K super block holds QK_K of them
+ int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
+ int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
+
+ float4 sums4[(TM/4) * TN];
+ float4 cache_a4[TM/4];
+
+ for (int i = 0; i < (TM/4) * TN; i++) {
+ sums4[i] = (float4)(0.0f);
+ }
+
+ for (int block = 0; block < ne00; block += BK) {
+ for (int l = 0; l < BM; l += loadstride_a) {
+ if (ir*BM + loadc_a + l < ne01) {
+ int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
+ int ib = idx / QK_K;
+ ib += offset0/sizeof(struct block_q2_K);
+ int e = idx % QK_K;
+
+ // dequantize_row_q2_K walks 2 halves of 128, each as 4 shifts of
+ // 2 groups of 16. Recover that position from the element index.
+ int n = e >> 7;
+ int rem = e & 127;
+ int j = rem >> 5;
+ int s = (rem >> 4) & 1;
+ int l16 = rem & 15;
+ int isc = 8*n + 2*j + s;
+ int shift = 2*j;
+
+ uchar sc = src0_sc[16*ib + isc];
+ float d = (float)src0_dm[2*ib] * (float)(sc & 0xF);
+ float m = -(float)src0_dm[2*ib + 1] * (float)(sc >> 4);
+
+ uchar4 q = vload4(0, src0_qs + 64*ib + 32*n + 16*s + l16);
+ float4 v1 = convert_float4((uchar4)((q.s0 >> shift) & 3,
+ (q.s1 >> shift) & 3,
+ (q.s2 >> shift) & 3,
+ (q.s3 >> shift) & 3)) * d + m;
+
+ buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
+ buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
+ buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
+ buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
+ } else {
+ buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
+ buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
+ buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
+ buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
+ }
+ }
+
+ for (int l = 0; l < BN; l += loadstride_b) {
+ if (ic*BN + loadc_b + l < ne11) {
+ int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
+ buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
+ buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
+ buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
+ buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
+ } else {
+ buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
+ buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
+ buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
+ buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
+ }
+ }
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ pos_a += BK;
+ pos_b += BK / LOAD_VEC_B;
+
+ for (int i = 0; i < BK; i++) {
+ for (int a = 0; a < TM/4; a++) {
+ cache_a4[a] = LM_LD4(buf_a + (i) * BM + th_r * TM, a);
+ }
+
+ for (int cc = 0; cc < TN; cc++) {
+ const float cache_b = buf_b[(i) * BN + th_c * TN + cc];
+ for (int a = 0; a < TM/4; a++) {
+ const int sums_idx = cc*(TM/4) + a;
+ sums4[sums_idx] = mad(cache_a4[a], (float4)cache_b, sums4[sums_idx]);
+ }
+ }
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ const int dr = ir * BM + th_r * TM;
+ const int dc = ic * BN + th_c * TN;
+
+ const ulong offsets = batch_idx * batch_stride_d;
+
+ for (int cc = 0; cc < TN; cc++) {
+ for (int a = 0; a < TM/4; a++) {
+ const float4 v = sums4[cc * (TM/4) + a];
+ const float vs[4] = { v.s0, v.s1, v.s2, v.s3 };
+ for (int k = 0; k < 4; k++) {
+ if (dr + 4*a + k < ne01 && dc + cc < ne11) {
+ dst[offsets + (dc + cc) * stride_d + dr + 4*a + k] = vs[k];
+ }
+ }
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32.cl
new file mode 100644
index 000000000..7f5328136
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32.cl
@@ -0,0 +1,161 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_required_subgroup_size
+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
+#define INTEL_GPU 1
+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
+#elif defined(cl_qcom_reqd_sub_group_size)
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+
+//------------------------------------------------------------------------------
+// block_q2_K
+//------------------------------------------------------------------------------
+#define QK_K 256
+
+// 16 blocks of 16 elements each
+// weight is represented as x = a * q + b
+typedef struct {
+ uchar scales[QK_K/16]; // scales and mins, quantized with 4 bits
+ uchar qs[QK_K/4]; // 2-bit quants
+ half d; // super-block scale for quantized scales
+ half dmin; // super-block scale for quantized mins
+} block_q2_K;
+
+#undef N_DST
+#undef N_SIMDGROUP
+#undef N_SIMDWIDTH
+
+#ifdef INTEL_GPU
+#define N_DST 4 // number of rows each SIMD group works on
+#define N_SIMDGROUP 1 // number of SIMD groups in a thread group
+#define N_SIMDWIDTH 16 // SIMD group size
+#elif defined (ADRENO_GPU)
+#define N_DST 4
+#define N_SIMDGROUP 1
+#define N_SIMDWIDTH 64
+#endif
+
+#undef BLOCK_STRIDE
+// number of (super) blocks each subgroup processes
+// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them
+#define BLOCK_STRIDE (N_SIMDWIDTH/8)
+
+#ifdef INTEL_GPU
+REQD_SUBGROUP_SIZE_16
+#elif defined (ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mv_q2_K_f32(
+ global char * src0,
+ ulong offset0,
+ global char * src1,
+ ulong offset1,
+ global char * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne12,
+ ulong nb11,
+ ulong nb12,
+ ulong nb13,
+ int ne0,
+ int ne1,
+ int r2,
+ int r3
+) {
+ src0 = src0 + offset0;
+ src1 = src1 + offset1;
+ dst = dst + offsetd;
+
+ int ix = get_sub_group_local_id()/8; // super block index
+ int it = get_sub_group_local_id()%8; // block index (inside super block)
+ int iq = it/4; // 0 or 1 - first or second half of the super block
+ int ir = it%4; // 0...3 - block index in the half super block
+ int is = (8*ir)/16; // 0 or 1 - scale pair
+
+ int nb = ne00/QK_K;
+
+ int r0 = get_group_id(0);
+ int r1 = get_group_id(1);
+ int im = get_group_id(2);
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
+
+ int i12 = im%ne12;
+ int i13 = im/ne12;
+
+ ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
+ ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;
+
+ global block_q2_K * x = (global block_q2_K *) (src0 + offset_src0);
+ global float * y = (global float *) (src1 + offset_src1);
+
+ float yl[32];
+ float sumf[N_DST] = {0.f};
+ float all_sum;
+
+ global float * y4 = y + ix * QK_K + 128 * iq + 8 * ir;
+
+ for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
+ float4 sumy = {0.f, 0.f, 0.f, 0.f};
+ for (int i = 0; i < 8; ++i) {
+ yl[i+ 0] = y4[i+ 0]; sumy.s0 += yl[i+ 0];
+ yl[i+ 8] = y4[i+32]; sumy.s1 += yl[i+ 8];
+ yl[i+16] = y4[i+64]; sumy.s2 += yl[i+16];
+ yl[i+24] = y4[i+96]; sumy.s3 += yl[i+24];
+ }
+
+ global uchar * sc = (global uchar *)x[ib].scales + 8*iq + is;
+ global ushort * qs = (global ushort *)x[ib].qs + 16 * iq + 4 * ir;
+ global half * dh = &x[ib].d;
+
+ // only ne01 output rows exist; reading past them can pull in an inf scale
+ for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
+ float4 acc1 = {0.f, 0.f, 0.f, 0.f};
+ float4 acc2 = {0.f, 0.f, 0.f, 0.f};
+ for (int i = 0; i < 8; i += 2) {
+ acc1.s0 += yl[i+ 0] * (qs[i/2] & 0x0003);
+ acc2.s0 += yl[i+ 1] * (qs[i/2] & 0x0300);
+ acc1.s1 += yl[i+ 8] * (qs[i/2] & 0x000c);
+ acc2.s1 += yl[i+ 9] * (qs[i/2] & 0x0c00);
+ acc1.s2 += yl[i+16] * (qs[i/2] & 0x0030);
+ acc2.s2 += yl[i+17] * (qs[i/2] & 0x3000);
+ acc1.s3 += yl[i+24] * (qs[i/2] & 0x00c0);
+ acc2.s3 += yl[i+25] * (qs[i/2] & 0xc000);
+ }
+
+ float dall = dh[0];
+ float dmin = dh[1] * 1.f/16.f;
+ sumf[row] += dall * ((acc1.s0 + 1.f/256.f * acc2.s0) * (sc[0] & 0xF) * 1.f/ 1.f +
+ (acc1.s1 + 1.f/256.f * acc2.s1) * (sc[2] & 0xF) * 1.f/ 4.f +
+ (acc1.s2 + 1.f/256.f * acc2.s2) * (sc[4] & 0xF) * 1.f/16.f +
+ (acc1.s3 + 1.f/256.f * acc2.s3) * (sc[6] & 0xF) * 1.f/64.f) -
+ dmin * (sumy.s0 * (sc[0] & 0xF0) + sumy.s1 * (sc[2] & 0xF0) +
+ sumy.s2 * (sc[4] & 0xF0) + sumy.s3 * (sc[6] & 0xF0));
+
+ qs += nb01/2;
+ sc += nb01;
+ dh += nb01/2;
+ }
+
+ y4 += BLOCK_STRIDE * QK_K;
+ }
+
+ global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = sub_group_reduce_add(sumf[row]);
+ if (first_row + row < ne01) {
+ if (get_sub_group_local_id() == 0) {
+ dst_f32[first_row + row] = all_sum;
+ }
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32_flat.cl
new file mode 100644
index 000000000..c993abebe
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_q2_k_f32_flat.cl
@@ -0,0 +1,162 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_required_subgroup_size
+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
+#define INTEL_GPU 1
+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
+#elif defined(cl_qcom_reqd_sub_group_size)
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+
+//------------------------------------------------------------------------------
+// struct block_q2_K
+//------------------------------------------------------------------------------
+#define QK_K 256
+
+// 16 blocks of 16 elements each
+// weight is represented as x = a * q + b
+struct block_q2_K {
+ uchar scales[QK_K/16]; // scales and mins, quantized with 4 bits
+ uchar qs[QK_K/4]; // 2-bit quants
+ half d; // super-block scale for quantized scales
+ half dmin; // super-block scale for quantized mins
+};
+
+#undef N_DST
+#undef N_SIMDGROUP
+#undef N_SIMDWIDTH
+
+#ifdef INTEL_GPU
+#define N_DST 4 // number of rows each SIMD group works on
+#define N_SIMDGROUP 1 // number of SIMD groups in a thread group
+#define N_SIMDWIDTH 16 // SIMD group size
+#elif defined (ADRENO_GPU)
+#define N_DST 4
+#define N_SIMDGROUP 1
+#define N_SIMDWIDTH 64
+#endif
+
+#undef BLOCK_STRIDE
+// number of (super) blocks each subgroup processes
+// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them
+#define BLOCK_STRIDE (N_SIMDWIDTH/8)
+
+#ifdef INTEL_GPU
+REQD_SUBGROUP_SIZE_16
+#elif defined (ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mv_q2_K_f32_flat(
+ global uchar * src0_qs,
+ global uchar * src0_sc,
+ global half * src0_dm,
+ ulong offset0,
+ global char * src1,
+ ulong offset1,
+ global char * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne12,
+ ulong nb11,
+ ulong nb12,
+ ulong nb13,
+ int ne0,
+ int ne1,
+ int r2,
+ int r3
+) {
+ src1 = src1 + offset1;
+ dst = dst + offsetd;
+
+ int ix = get_sub_group_local_id()/8; // super block index
+ int it = get_sub_group_local_id()%8; // block index (inside super block)
+ int iq = it/4; // 0 or 1 - first or second half of the super block
+ int ir = it%4; // 0...3 - block index in the half super block
+ int is = (8*ir)/16; // 0 or 1 - scale pair
+
+ int nb = ne00/QK_K;
+
+ int r0 = get_group_id(0);
+ int r1 = get_group_id(1);
+ int im = get_group_id(2);
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
+
+ int i12 = im%ne12;
+ int i13 = im/ne12;
+
+ ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(struct block_q2_K);
+ ulong row_stride = nb01/sizeof(struct block_q2_K);
+ ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;
+
+ global float * y = (global float *) (src1 + offset_src1);
+
+ float yl[32];
+ float sumf[N_DST] = {0.f};
+ float all_sum;
+
+ global float * y4 = y + ix * QK_K + 128 * iq + 8 * ir;
+
+ for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
+ float4 sumy = {0.f, 0.f, 0.f, 0.f};
+ for (int i = 0; i < 8; ++i) {
+ yl[i+ 0] = y4[i+ 0]; sumy.s0 += yl[i+ 0];
+ yl[i+ 8] = y4[i+32]; sumy.s1 += yl[i+ 8];
+ yl[i+16] = y4[i+64]; sumy.s2 += yl[i+16];
+ yl[i+24] = y4[i+96]; sumy.s3 += yl[i+24];
+ }
+
+ global uchar * sc = (src0_sc + 16*(offset_src0 + ib)) + 8*iq + is;
+ global ushort * qs = (global ushort *)(src0_qs + 64*(offset_src0 + ib)) + 16 * iq + 4 * ir;
+ global half * dh = src0_dm + 2*(offset_src0 + ib);
+
+ // only ne01 output rows exist; reading past them can pull in an inf scale
+ for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
+ float4 acc1 = {0.f, 0.f, 0.f, 0.f};
+ float4 acc2 = {0.f, 0.f, 0.f, 0.f};
+ for (int i = 0; i < 8; i += 2) {
+ acc1.s0 += yl[i+ 0] * (qs[i/2] & 0x0003);
+ acc2.s0 += yl[i+ 1] * (qs[i/2] & 0x0300);
+ acc1.s1 += yl[i+ 8] * (qs[i/2] & 0x000c);
+ acc2.s1 += yl[i+ 9] * (qs[i/2] & 0x0c00);
+ acc1.s2 += yl[i+16] * (qs[i/2] & 0x0030);
+ acc2.s2 += yl[i+17] * (qs[i/2] & 0x3000);
+ acc1.s3 += yl[i+24] * (qs[i/2] & 0x00c0);
+ acc2.s3 += yl[i+25] * (qs[i/2] & 0xc000);
+ }
+
+ float dall = dh[0];
+ float dmin = dh[1] * 1.f/16.f;
+ sumf[row] += dall * ((acc1.s0 + 1.f/256.f * acc2.s0) * (sc[0] & 0xF) * 1.f/ 1.f +
+ (acc1.s1 + 1.f/256.f * acc2.s1) * (sc[2] & 0xF) * 1.f/ 4.f +
+ (acc1.s2 + 1.f/256.f * acc2.s2) * (sc[4] & 0xF) * 1.f/16.f +
+ (acc1.s3 + 1.f/256.f * acc2.s3) * (sc[6] & 0xF) * 1.f/64.f) -
+ dmin * (sumy.s0 * (sc[0] & 0xF0) + sumy.s1 * (sc[2] & 0xF0) +
+ sumy.s2 * (sc[4] & 0xF0) + sumy.s3 * (sc[6] & 0xF0));
+
+ qs += row_stride*32;
+ sc += row_stride*16;
+ dh += row_stride*2;
+ }
+
+ y4 += BLOCK_STRIDE * QK_K;
+ }
+
+ global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = sub_group_reduce_add(sumf[row]);
+ if (first_row + row < ne01) {
+ if (get_sub_group_local_id() == 0) {
+ dst_f32[first_row + row] = all_sum;
+ }
+ }
+ }
+}