Commit 22183e51f for llama.cpp
commit 22183e51f920714cab6406c4d038106d4cc3ce49
Author: lhez <lih@qti.qualcomm.com>
Date: Thu Oct 29 01:12:11 2026 -0700
opencl: initial support for iq2 (#31267)
* opencl: add iq2_s AOS mv
* opencl: add iq2_s SOA mv and tiled mm
* opencl: add iq2_s noshuffle gemm/gemv for adreno
* opencl: add iq2_s dp4a for adreno
* opencl: optimize iq2_s adreno decode and narrow batches
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
* opencl: add iq2_xs AOS mv
* opencl: add iq2_xs SOA mv and tiled mm
* opencl: add iq2_xs noshuffle gemm/gemv for adreno
* opencl: add iq2_xs dp4a for adreno
* opencl: optimize iq2_xs adreno decode and narrow batches
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
* opencl: add iq2_xxs AOS mv
* opencl: add iq2_xxs SOA mv and tiled mm
* opencl: add iq2_xxs noshuffle gemm/gemv for adreno
* opencl: add iq2_xxs dp4a for adreno
* opencl: optimize iq2_xxs adreno decode and narrow batches
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
---------
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt
index 4d651ddb6..67b62829f 100644
--- a/ggml/src/ggml-opencl/CMakeLists.txt
+++ b/ggml/src/ggml-opencl/CMakeLists.txt
@@ -105,10 +105,31 @@ set(GGML_OPENCL_KERNELS
gemv_noshuffle_q2_k_f32
gemm_noshuffle_q2_k_f32
gemm_noshuffle_q3_k_f32
+ gemm_noshuffle_iq2_s_f32
+ gemm_noshuffle_iq2_xs_f32
+ gemm_noshuffle_iq2_xxs_f32
+ gemm_noshuffle_iq2_xxs_q8_1_dp4a
+ gemv_noshuffle_iq2_xxs_f32
+ gemm_noshuffle_iq2_xs_q8_1_dp4a
+ gemv_noshuffle_iq2_xs_f32
+ gemm_noshuffle_iq2_s_q8_1_dp4a
+ gemv_noshuffle_iq2_s_f32
gemm_noshuffle_q3_k_q8_1_dp4a
gemv_noshuffle_q3_k_f32
mul_mv_q2_k_f32
mul_mv_q3_k_f32
+ iq2_s_grid
+ mul_mv_iq2_s_f32
+ iq2_xs_grid
+ mul_mv_iq2_xs_f32
+ iq2_xxs_grid
+ mul_mv_iq2_xxs_f32
+ mul_mv_iq2_xxs_f32_flat
+ mul_mm_iq2_xxs_f32_l4_lm
+ mul_mv_iq2_xs_f32_flat
+ mul_mm_iq2_xs_f32_l4_lm
+ mul_mv_iq2_s_f32_flat
+ mul_mm_iq2_s_f32_l4_lm
mul_mv_q3_k_f32_flat
mul_mm_q3_k_f32_l4_lm
mul_mv_q2_k_f32_flat
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 8167e5e22..c68747c9a 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -986,6 +986,39 @@ struct ggml_backend_opencl_context {
int q3_k_mv_rows = 2;
int q3_k_mv_nsg = 8;
cl_kernel kernel_mul_mv_q3_K_f32;
+ cl_kernel kernel_convert_block_iq2_s, kernel_restore_block_iq2_s;
+ cl_kernel kernel_mul_mv_iq2_s_f32_flat;
+ cl_kernel kernel_mul_mm_iq2_s_f32_l4_lm;
+ cl_kernel kernel_gemv_noshuffle_iq2_s_f32 = nullptr;
+ cl_kernel kernel_gemm_noshuffle_iq2_s_f32 = nullptr;
+ cl_kernel kernel_gemm_noshuffle_iq2_s_q8_1_dp4a = nullptr;
+ cl_kernel kernel_gemm_noshuffle_iq2_s_q8_1_dp4a_narrow = nullptr;
+ int iq2_s_mv_nsg = 8;
+ ggml_cl_buffer iq2_s_grid_buffer;
+ cl_mem iq2_s_grid_image = nullptr;
+ cl_kernel kernel_mul_mv_iq2_s_f32;
+ cl_kernel kernel_convert_block_iq2_xs, kernel_restore_block_iq2_xs;
+ cl_kernel kernel_mul_mv_iq2_xs_f32_flat;
+ cl_kernel kernel_mul_mm_iq2_xs_f32_l4_lm;
+ cl_kernel kernel_gemv_noshuffle_iq2_xs_f32 = nullptr;
+ cl_kernel kernel_gemm_noshuffle_iq2_xs_f32 = nullptr;
+ cl_kernel kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a = nullptr;
+ cl_kernel kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a_narrow = nullptr;
+ int iq2_xs_mv_nsg = 4;
+ ggml_cl_buffer iq2_xs_grid_buffer;
+ cl_mem iq2_xs_grid_image = nullptr;
+ cl_kernel kernel_mul_mv_iq2_xs_f32;
+ cl_kernel kernel_convert_block_iq2_xxs, kernel_restore_block_iq2_xxs;
+ cl_kernel kernel_mul_mv_iq2_xxs_f32_flat;
+ cl_kernel kernel_mul_mm_iq2_xxs_f32_l4_lm;
+ cl_kernel kernel_gemv_noshuffle_iq2_xxs_f32 = nullptr;
+ cl_kernel kernel_gemm_noshuffle_iq2_xxs_f32 = nullptr;
+ cl_kernel kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a = nullptr;
+ cl_kernel kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a_narrow = nullptr;
+ int iq2_xxs_mv_nsg = 4;
+ ggml_cl_buffer iq2_xxs_grid_buffer;
+ cl_mem iq2_xxs_grid_image = nullptr;
+ cl_kernel kernel_mul_mv_iq2_xxs_f32;
cl_kernel kernel_mul_mv_q4_K_f32;
cl_kernel kernel_mul_mv_q4_K_f32_flat;
cl_kernel kernel_mul_mv_q5_K_f32;
@@ -1360,6 +1393,18 @@ struct ggml_backend_opencl_context {
int rows_coopk_dp4a_q6_k = 4;
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
+ ~ggml_backend_opencl_context() {
+ if (iq2_xxs_grid_image) {
+ CL_CHECK(clReleaseMemObject(iq2_xxs_grid_image));
+ }
+ if (iq2_xs_grid_image) {
+ CL_CHECK(clReleaseMemObject(iq2_xs_grid_image));
+ }
+ if (iq2_s_grid_image) {
+ CL_CHECK(clReleaseMemObject(iq2_s_grid_image));
+ }
+ }
+
void free() {
clFinish(queue);
@@ -2038,6 +2083,12 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
CL_CHECK((backend_ctx->kernel_restore_block_q2_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_k_noshuffle", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q3_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_k_noshuffle", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_q3_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_k_noshuffle", &err), err));
+ CL_CHECK((backend_ctx->kernel_convert_block_iq2_xxs = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq2_xxs", &err), err));
+ CL_CHECK((backend_ctx->kernel_restore_block_iq2_xxs = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq2_xxs", &err), err));
+ CL_CHECK((backend_ctx->kernel_convert_block_iq2_xs = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq2_xs", &err), err));
+ CL_CHECK((backend_ctx->kernel_restore_block_iq2_xs = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq2_xs", &err), err));
+ CL_CHECK((backend_ctx->kernel_convert_block_iq2_s = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq2_s", &err), err));
+ CL_CHECK((backend_ctx->kernel_restore_block_iq2_s = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq2_s", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_K", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_K", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_K", &err), err));
@@ -2476,6 +2527,165 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string iq2_s_grid_src {
+ #include "iq2_s_grid.cl.h"
+ };
+#else
+ const std::string iq2_s_grid_src = read_file("iq2_s_grid.cl");
+#endif
+
+ // mul_mv_iq2_s_f32_flat
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mv_iq2_s_f32_flat.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mv_iq2_s_f32_flat.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_s_grid_src + kernel_src).c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_mul_mv_iq2_s_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq2_s_f32_flat", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // mul_mm_iq2_s_f32_l4_lm
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mm_iq2_s_f32_l4_lm.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mm_iq2_s_f32_l4_lm.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_s_grid_src + kernel_src).c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_mul_mm_iq2_s_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq2_s_f32_l4_lm", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string iq2_xs_grid_src {
+ #include "iq2_xs_grid.cl.h"
+ };
+#else
+ const std::string iq2_xs_grid_src = read_file("iq2_xs_grid.cl");
+#endif
+
+ // mul_mv_iq2_xs_f32_flat
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mv_iq2_xs_f32_flat.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mv_iq2_xs_f32_flat.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_xs_grid_src + kernel_src).c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_mul_mv_iq2_xs_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq2_xs_f32_flat", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // mul_mm_iq2_xs_f32_l4_lm
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mm_iq2_xs_f32_l4_lm.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mm_iq2_xs_f32_l4_lm.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_xs_grid_src + kernel_src).c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_mul_mm_iq2_xs_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq2_xs_f32_l4_lm", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string iq2_xxs_grid_src {
+ #include "iq2_xxs_grid.cl.h"
+ };
+#else
+ const std::string iq2_xxs_grid_src = read_file("iq2_xxs_grid.cl");
+#endif
+
+ // mul_mv_iq2_xxs_f32_flat
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mv_iq2_xxs_f32_flat.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mv_iq2_xxs_f32_flat.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_xxs_grid_src + kernel_src).c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_mul_mv_iq2_xxs_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq2_xxs_f32_flat", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // mul_mm_iq2_xxs_f32_l4_lm
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mm_iq2_xxs_f32_l4_lm.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mm_iq2_xxs_f32_l4_lm.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_xxs_grid_src + kernel_src).c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_mul_mm_iq2_xxs_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq2_xxs_f32_l4_lm", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // mul_mv_iq2_xxs_f32
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mv_iq2_xxs_f32.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mv_iq2_xxs_f32.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_xxs_grid_src + kernel_src).c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_mul_mv_iq2_xxs_f32 = clCreateKernel(prog, "kernel_mul_mv_iq2_xxs_f32", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // mul_mv_iq2_xs_f32
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mv_iq2_xs_f32.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mv_iq2_xs_f32.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_xs_grid_src + kernel_src).c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_mul_mv_iq2_xs_f32 = clCreateKernel(prog, "kernel_mul_mv_iq2_xs_f32", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // mul_mv_iq2_s_f32
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mv_iq2_s_f32.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mv_iq2_s_f32.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_s_grid_src + kernel_src).c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_mul_mv_iq2_s_f32 = clCreateKernel(prog, "kernel_mul_mv_iq2_s_f32", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
// mul_mv_q3_k_f32
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -4319,195 +4529,456 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
- // gemm_noshuffle_q3_k_f32
+ // gemm_noshuffle_iq2_s_q8_1_dp4a
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
- #include "gemm_noshuffle_q3_k_f32.cl.h"
+ #include "gemm_noshuffle_iq2_s_q8_1_dp4a.cl.h"
};
#else
- const std::string kernel_src = read_file("gemm_noshuffle_q3_k_f32.cl");
+ const std::string kernel_src = read_file("gemm_noshuffle_iq2_s_q8_1_dp4a.cl");
#endif
- cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
- CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_f32", &err), err));
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_s_grid_src + kernel_src).c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_s_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_s_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ prog = build_program_from_source(backend_ctx, (iq2_s_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8");
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_s_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_s_q8_1_dp4a", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
- // gemv_noshuffle_q3_k_f32
+ // gemm_noshuffle_iq2_xs_q8_1_dp4a
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
- #include "gemv_noshuffle_q3_k_f32.cl.h"
+ #include "gemm_noshuffle_iq2_xs_q8_1_dp4a.cl.h"
};
#else
- const std::string kernel_src = read_file("gemv_noshuffle_q3_k_f32.cl");
+ const std::string kernel_src = read_file("gemm_noshuffle_iq2_xs_q8_1_dp4a.cl");
#endif
- if (const char * env = getenv("GGML_OPENCL_Q3_K_MV_R")) {
- const int rows = atoi(env);
- if (rows == 1 || rows == 2 || rows == 4) {
- backend_ctx->q3_k_mv_rows = rows;
- }
- }
- if (const char * env = getenv("GGML_OPENCL_Q3_K_MV_NSG")) {
- const int nsg = atoi(env);
- if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) {
- backend_ctx->q3_k_mv_nsg = nsg;
- }
- }
- for (;;) {
- const std::string opts = compile_opts + " -DQ3_K_MV_R=" + std::to_string(backend_ctx->q3_k_mv_rows) +
- " -DQ3_K_MV_NSG=" + std::to_string(backend_ctx->q3_k_mv_nsg);
- cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts);
- cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q3_k_f32", &err);
- CL_CHECK(err);
- CL_CHECK(clReleaseProgram(prog));
- size_t max_workgroup;
- CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr));
- if (64*(size_t)backend_ctx->q3_k_mv_nsg <= max_workgroup) {
- backend_ctx->kernel_gemv_noshuffle_q3_k_f32 = kernel;
- break;
- }
- CL_CHECK(clReleaseKernel(kernel));
- GGML_ASSERT(backend_ctx->q3_k_mv_nsg > 1);
- backend_ctx->q3_k_mv_nsg /= 2;
- }
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_xs_grid_src + kernel_src).c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ prog = build_program_from_source(backend_ctx, (iq2_xs_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8");
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
- // gemm_noshuffle_q2_k_f32
+ // gemm_noshuffle_iq2_xxs_q8_1_dp4a
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
- #include "gemm_noshuffle_q2_k_f32.cl.h"
+ #include "gemm_noshuffle_iq2_xxs_q8_1_dp4a.cl.h"
};
#else
- const std::string kernel_src = read_file("gemm_noshuffle_q2_k_f32.cl");
+ const std::string kernel_src = read_file("gemm_noshuffle_iq2_xxs_q8_1_dp4a.cl");
#endif
- cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
- CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_k_f32", &err), err));
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_xxs_grid_src + kernel_src).c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ prog = build_program_from_source(backend_ctx, (iq2_xxs_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8");
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
- // gemm_noshuffle_q2_k_q8_1_dp4a
- if (backend_ctx->has_integer_dot) {
+ // gemm_noshuffle_iq2_xxs_f32
+ {
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
- #include "gemm_noshuffle_q2_k_q8_1_dp4a.cl.h"
+ #include "gemm_noshuffle_iq2_xxs_f32.cl.h"
};
#else
- const std::string kernel_src = read_file("gemm_noshuffle_q2_k_q8_1_dp4a.cl");
+ const std::string kernel_src = read_file("gemm_noshuffle_iq2_xxs_f32.cl");
#endif
- cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
- CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_k_q8_1_dp4a", &err), err));
- CL_CHECK(clReleaseProgram(prog));
- prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + " -DTILESIZE_N=8");
- CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_k_q8_1_dp4a", &err), err));
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_xxs_grid_src + kernel_src).c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_xxs_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_xxs_f32", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
- // gemv_noshuffle_q2_k_f32
+ // gemv_noshuffle_iq2_xxs_f32
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
- #include "gemv_noshuffle_q2_k_f32.cl.h"
+ #include "gemv_noshuffle_iq2_xxs_f32.cl.h"
};
#else
- const std::string kernel_src = read_file("gemv_noshuffle_q2_k_f32.cl");
+ const std::string kernel_src = read_file("gemv_noshuffle_iq2_xxs_f32.cl");
#endif
- if (const char * env = getenv("GGML_OPENCL_Q2_K_MV_R")) {
- const int rows = atoi(env);
- if (rows == 1 || rows == 2 || rows == 4) {
- backend_ctx->q2_k_mv_rows = rows;
- }
- }
- if (const char * env = getenv("GGML_OPENCL_Q2_K_MV_NSG")) {
+ if (const char * env = getenv("GGML_OPENCL_IQ2_XXS_MV_NSG")) {
const int nsg = atoi(env);
if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) {
- backend_ctx->q2_k_mv_nsg = nsg;
+ backend_ctx->iq2_xxs_mv_nsg = nsg;
}
}
for (;;) {
- const std::string opts = compile_opts + " -DQ2_K_MV_R=" + std::to_string(backend_ctx->q2_k_mv_rows) +
- " -DQ2_K_MV_NSG=" + std::to_string(backend_ctx->q2_k_mv_nsg);
- cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts);
- cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q2_k_f32", &err);
+ const std::string opts = compile_opts + " -DIQ2_XXS_MV_NSG=" + std::to_string(backend_ctx->iq2_xxs_mv_nsg);
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_xxs_grid_src + kernel_src).c_str(), opts);
+ cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq2_xxs_f32", &err);
CL_CHECK(err);
- CL_CHECK(clReleaseProgram(prog));
size_t max_workgroup;
CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr));
- if (64*(size_t)backend_ctx->q2_k_mv_nsg <= max_workgroup) {
- backend_ctx->kernel_gemv_noshuffle_q2_k_f32 = kernel;
+ if (64*(size_t)backend_ctx->iq2_xxs_mv_nsg <= max_workgroup) {
+ backend_ctx->kernel_gemv_noshuffle_iq2_xxs_f32 = kernel;
+ cl_kernel export_grid = clCreateKernel(prog, "kernel_iq2_xxs_grid_export", &err);
+ CL_CHECK(err);
+ backend_ctx->iq2_xxs_grid_buffer.allocate(backend_ctx->context, 512*sizeof(cl_uint));
+ CL_CHECK(clSetKernelArg(export_grid, 0, sizeof(cl_mem), &backend_ctx->iq2_xxs_grid_buffer.buffer));
+ size_t global[] = { 512 };
+ size_t local[] = { 64 };
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, export_grid, 1, nullptr, global, local, 0, nullptr, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clReleaseEvent(evt));
+ CL_CHECK(clReleaseKernel(export_grid));
+ CL_CHECK(clReleaseProgram(prog));
+
+ cl_image_format format = { CL_R, CL_UNSIGNED_INT32 };
+ cl_image_desc desc = {};
+ desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ desc.image_width = 512;
+ desc.buffer = backend_ctx->iq2_xxs_grid_buffer.buffer;
+ backend_ctx->iq2_xxs_grid_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &format, &desc, nullptr, &err);
+ CL_CHECK(err);
break;
}
CL_CHECK(clReleaseKernel(kernel));
- GGML_ASSERT(backend_ctx->q2_k_mv_nsg > 1);
- backend_ctx->q2_k_mv_nsg /= 2;
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_ASSERT(backend_ctx->iq2_xxs_mv_nsg > 1);
+ backend_ctx->iq2_xxs_mv_nsg /= 2;
}
GGML_LOG_CONT(".");
}
- // gemv_noshuffle_general
+ // gemm_noshuffle_iq2_xs_f32
{
- std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +
- " -cl-mad-enable "
- " -DSIMDGROUP_WIDTH=" +
- std::to_string(backend_ctx->adreno_wave_size);
- if (backend_ctx->has_vector_subgroup_broadcast) {
- CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST ";
- }
- if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X) {
- CL_gemv_compile_opts += " -DGGML_CL_A6X_CONSTFOLD_FIX";
- }
-
#ifdef GGML_OPENCL_EMBED_KERNELS
- const std::string kernel_src_CL_gemv_general {
- #include "gemv_noshuffle_q4_0_f32.cl.h"
+ const std::string kernel_src {
+ #include "gemm_noshuffle_iq2_xs_f32.cl.h"
};
#else
- const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q4_0_f32.cl");
+ const std::string kernel_src = read_file("gemm_noshuffle_iq2_xs_f32.cl");
#endif
-
- cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts);
-
- CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err));
- CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_mc3", &err), err));
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_xs_grid_src + kernel_src).c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_xs_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_xs_f32", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
- // gemv_noshuffle
+ // gemv_noshuffle_iq2_xs_f32
{
- // Gemv 2048, 16384
- std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +
- " -cl-mad-enable "
- " -DLINE_STRIDE_A=2048 "
- " -DBLOCK_STRIDE_A=16384 "
- " -DSIMDGROUP_WIDTH=" +
- std::to_string(backend_ctx->adreno_wave_size);
- if (backend_ctx->has_vector_subgroup_broadcast) {
- CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST ";
- }
-
#ifdef GGML_OPENCL_EMBED_KERNELS
- const std::string kernel_src_CL_gemv {
- #include "gemv_noshuffle_q4_0_f32_spec.cl.h"
+ const std::string kernel_src {
+ #include "gemv_noshuffle_iq2_xs_f32.cl.h"
};
#else
- const std::string kernel_src_CL_gemv = read_file("gemv_noshuffle_q4_0_f32_spec.cl");
+ const std::string kernel_src = read_file("gemv_noshuffle_iq2_xs_f32.cl");
#endif
+ if (const char * env = getenv("GGML_OPENCL_IQ2_XS_MV_NSG")) {
+ const int nsg = atoi(env);
+ if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) {
+ backend_ctx->iq2_xs_mv_nsg = nsg;
+ }
+ }
+ for (;;) {
+ const std::string opts = compile_opts + " -DIQ2_XS_MV_NSG=" + std::to_string(backend_ctx->iq2_xs_mv_nsg);
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_xs_grid_src + kernel_src).c_str(), opts);
+ cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq2_xs_f32", &err);
+ CL_CHECK(err);
+ size_t max_workgroup;
+ CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr));
+ if (64*(size_t)backend_ctx->iq2_xs_mv_nsg <= max_workgroup) {
+ backend_ctx->kernel_gemv_noshuffle_iq2_xs_f32 = kernel;
+ cl_kernel export_grid = clCreateKernel(prog, "kernel_iq2_xs_grid_export", &err);
+ CL_CHECK(err);
+ backend_ctx->iq2_xs_grid_buffer.allocate(backend_ctx->context, 1024*sizeof(cl_uint));
+ CL_CHECK(clSetKernelArg(export_grid, 0, sizeof(cl_mem), &backend_ctx->iq2_xs_grid_buffer.buffer));
+ size_t global[] = { 1024 };
+ size_t local[] = { 64 };
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, export_grid, 1, nullptr, global, local, 0, nullptr, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clReleaseEvent(evt));
+ CL_CHECK(clReleaseKernel(export_grid));
+ CL_CHECK(clReleaseProgram(prog));
- cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts);
- CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_4096 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err));
- CL_CHECK(clReleaseProgram(prog));
+ cl_image_format format = { CL_R, CL_UNSIGNED_INT32 };
+ cl_image_desc desc = {};
+ desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ desc.image_width = 1024;
+ desc.buffer = backend_ctx->iq2_xs_grid_buffer.buffer;
+ backend_ctx->iq2_xs_grid_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &format, &desc, nullptr, &err);
+ CL_CHECK(err);
+ break;
+ }
+ CL_CHECK(clReleaseKernel(kernel));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_ASSERT(backend_ctx->iq2_xs_mv_nsg > 1);
+ backend_ctx->iq2_xs_mv_nsg /= 2;
+ }
GGML_LOG_CONT(".");
+ }
- // Gemv 2048, 16384
- CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +
- " -cl-mad-enable "
- " -DLINE_STRIDE_A=2048 "
+ // gemm_noshuffle_iq2_s_f32
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_noshuffle_iq2_s_f32.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_noshuffle_iq2_s_f32.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_s_grid_src + kernel_src).c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq2_s_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq2_s_f32", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemv_noshuffle_iq2_s_f32
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemv_noshuffle_iq2_s_f32.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemv_noshuffle_iq2_s_f32.cl");
+#endif
+ if (const char * env = getenv("GGML_OPENCL_IQ2_S_MV_NSG")) {
+ const int nsg = atoi(env);
+ if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) {
+ backend_ctx->iq2_s_mv_nsg = nsg;
+ }
+ }
+ for (;;) {
+ const std::string opts = compile_opts + " -DIQ2_S_MV_NSG=" + std::to_string(backend_ctx->iq2_s_mv_nsg);
+ cl_program prog = build_program_from_source(backend_ctx, (iq2_s_grid_src + kernel_src).c_str(), opts);
+ cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq2_s_f32", &err);
+ CL_CHECK(err);
+ size_t max_workgroup;
+ CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr));
+ if (64*(size_t)backend_ctx->iq2_s_mv_nsg <= max_workgroup) {
+ backend_ctx->kernel_gemv_noshuffle_iq2_s_f32 = kernel;
+ cl_kernel export_grid = clCreateKernel(prog, "kernel_iq2_s_grid_export", &err);
+ CL_CHECK(err);
+ backend_ctx->iq2_s_grid_buffer.allocate(backend_ctx->context, 2048*sizeof(cl_uint));
+ CL_CHECK(clSetKernelArg(export_grid, 0, sizeof(cl_mem), &backend_ctx->iq2_s_grid_buffer.buffer));
+ size_t global[] = { 2048 };
+ size_t local[] = { 64 };
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(backend_ctx->queue, export_grid, 1, nullptr, global, local, 0, nullptr, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clReleaseEvent(evt));
+ CL_CHECK(clReleaseKernel(export_grid));
+ CL_CHECK(clReleaseProgram(prog));
+
+ cl_image_format format = { CL_R, CL_UNSIGNED_INT32 };
+ cl_image_desc desc = {};
+ desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ desc.image_width = 2048;
+ desc.buffer = backend_ctx->iq2_s_grid_buffer.buffer;
+ backend_ctx->iq2_s_grid_image = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &format, &desc, nullptr, &err);
+ CL_CHECK(err);
+ break;
+ }
+ CL_CHECK(clReleaseKernel(kernel));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_ASSERT(backend_ctx->iq2_s_mv_nsg > 1);
+ backend_ctx->iq2_s_mv_nsg /= 2;
+ }
+ GGML_LOG_CONT(".");
+ }
+
+ // gemm_noshuffle_q3_k_f32
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_noshuffle_q3_k_f32.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_noshuffle_q3_k_f32.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_f32", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemv_noshuffle_q3_k_f32
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemv_noshuffle_q3_k_f32.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemv_noshuffle_q3_k_f32.cl");
+#endif
+ if (const char * env = getenv("GGML_OPENCL_Q3_K_MV_R")) {
+ const int rows = atoi(env);
+ if (rows == 1 || rows == 2 || rows == 4) {
+ backend_ctx->q3_k_mv_rows = rows;
+ }
+ }
+ if (const char * env = getenv("GGML_OPENCL_Q3_K_MV_NSG")) {
+ const int nsg = atoi(env);
+ if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) {
+ backend_ctx->q3_k_mv_nsg = nsg;
+ }
+ }
+ for (;;) {
+ const std::string opts = compile_opts + " -DQ3_K_MV_R=" + std::to_string(backend_ctx->q3_k_mv_rows) +
+ " -DQ3_K_MV_NSG=" + std::to_string(backend_ctx->q3_k_mv_nsg);
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts);
+ cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q3_k_f32", &err);
+ CL_CHECK(err);
+ CL_CHECK(clReleaseProgram(prog));
+ size_t max_workgroup;
+ CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr));
+ if (64*(size_t)backend_ctx->q3_k_mv_nsg <= max_workgroup) {
+ backend_ctx->kernel_gemv_noshuffle_q3_k_f32 = kernel;
+ break;
+ }
+ CL_CHECK(clReleaseKernel(kernel));
+ GGML_ASSERT(backend_ctx->q3_k_mv_nsg > 1);
+ backend_ctx->q3_k_mv_nsg /= 2;
+ }
+ GGML_LOG_CONT(".");
+ }
+
+ // gemm_noshuffle_q2_k_f32
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_noshuffle_q2_k_f32.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_noshuffle_q2_k_f32.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_k_f32", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemm_noshuffle_q2_k_q8_1_dp4a
+ if (backend_ctx->has_integer_dot) {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemm_noshuffle_q2_k_q8_1_dp4a.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemm_noshuffle_q2_k_q8_1_dp4a.cl");
+#endif
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_k_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + " -DTILESIZE_N=8");
+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q2_k_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_q2_k_q8_1_dp4a", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemv_noshuffle_q2_k_f32
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "gemv_noshuffle_q2_k_f32.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("gemv_noshuffle_q2_k_f32.cl");
+#endif
+ if (const char * env = getenv("GGML_OPENCL_Q2_K_MV_R")) {
+ const int rows = atoi(env);
+ if (rows == 1 || rows == 2 || rows == 4) {
+ backend_ctx->q2_k_mv_rows = rows;
+ }
+ }
+ if (const char * env = getenv("GGML_OPENCL_Q2_K_MV_NSG")) {
+ const int nsg = atoi(env);
+ if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) {
+ backend_ctx->q2_k_mv_nsg = nsg;
+ }
+ }
+ for (;;) {
+ const std::string opts = compile_opts + " -DQ2_K_MV_R=" + std::to_string(backend_ctx->q2_k_mv_rows) +
+ " -DQ2_K_MV_NSG=" + std::to_string(backend_ctx->q2_k_mv_nsg);
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts);
+ cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q2_k_f32", &err);
+ CL_CHECK(err);
+ CL_CHECK(clReleaseProgram(prog));
+ size_t max_workgroup;
+ CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr));
+ if (64*(size_t)backend_ctx->q2_k_mv_nsg <= max_workgroup) {
+ backend_ctx->kernel_gemv_noshuffle_q2_k_f32 = kernel;
+ break;
+ }
+ CL_CHECK(clReleaseKernel(kernel));
+ GGML_ASSERT(backend_ctx->q2_k_mv_nsg > 1);
+ backend_ctx->q2_k_mv_nsg /= 2;
+ }
+ GGML_LOG_CONT(".");
+ }
+
+ // gemv_noshuffle_general
+ {
+ std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +
+ " -cl-mad-enable "
+ " -DSIMDGROUP_WIDTH=" +
+ std::to_string(backend_ctx->adreno_wave_size);
+ if (backend_ctx->has_vector_subgroup_broadcast) {
+ CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST ";
+ }
+ if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A6X) {
+ CL_gemv_compile_opts += " -DGGML_CL_A6X_CONSTFOLD_FIX";
+ }
+
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src_CL_gemv_general {
+ #include "gemv_noshuffle_q4_0_f32.cl.h"
+ };
+#else
+ const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q4_0_f32.cl");
+#endif
+
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts);
+
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err));
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_mc3 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32_mc3", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
+ // gemv_noshuffle
+ {
+ // Gemv 2048, 16384
+ std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +
+ " -cl-mad-enable "
+ " -DLINE_STRIDE_A=2048 "
+ " -DBLOCK_STRIDE_A=16384 "
+ " -DSIMDGROUP_WIDTH=" +
+ std::to_string(backend_ctx->adreno_wave_size);
+ if (backend_ctx->has_vector_subgroup_broadcast) {
+ CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST ";
+ }
+
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src_CL_gemv {
+ #include "gemv_noshuffle_q4_0_f32_spec.cl.h"
+ };
+#else
+ const std::string kernel_src_CL_gemv = read_file("gemv_noshuffle_q4_0_f32_spec.cl");
+#endif
+
+ cl_program prog = build_program_from_source(backend_ctx, kernel_src_CL_gemv.c_str(), CL_gemv_compile_opts);
+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q4_0_f32_4096_1_4096 = clCreateKernel(prog, "kernel_gemv_noshuffle_q4_0_f32", &err), err));
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+
+ // Gemv 2048, 16384
+ CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +
+ " -cl-mad-enable "
+ " -DLINE_STRIDE_A=2048 "
" -DBLOCK_STRIDE_A=16384 "
" -DSIMDGROUP_WIDTH=" +
std::to_string(backend_ctx->adreno_wave_size);
@@ -8451,38 +8922,99 @@ struct ggml_tensor_extra_cl_q3_K {
}
};
-struct ggml_tensor_extra_cl_q6_K {
- // Lower 4 bits of quantized weights.
- cl_mem ql = nullptr;
- // Lower 4 bits as image1d_buffer_t
- cl_mem ql_img = nullptr;
- // Upper 2 bits of quantized weights.
- cl_mem qh = nullptr;
- // Upper 2 bits as image1d_buffer_t
- cl_mem qh_img = nullptr;
- // Scales for each block.
- cl_mem s = nullptr;
- // Scales for each super block.
- cl_mem d = nullptr;
+struct ggml_tensor_extra_cl_iq2_xxs {
+ bool noshuffle = false;
+ cl_mem qs = nullptr; // grid indices
+ cl_mem sas = nullptr; // packed sign codes and scales
+ cl_mem d = nullptr; // super-block scales
- size_t size_ql = 0;
- size_t size_qh = 0;
- size_t size_s = 0;
- size_t size_d = 0;
+ size_t size_qs = 0, size_sas = 0, size_d = 0;
- ~ggml_tensor_extra_cl_q6_K() {
- reset();
- }
+ ~ggml_tensor_extra_cl_iq2_xxs() { reset(); }
void reset() {
- if (ql != nullptr) {
- CL_CHECK(clReleaseMemObject(ql));
- ql = nullptr;
- }
- if (qh != nullptr) {
- CL_CHECK(clReleaseMemObject(qh));
- qh = nullptr;
- }
+ noshuffle = false;
+ if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; }
+ if (sas != nullptr) { CL_CHECK(clReleaseMemObject(sas)); sas = nullptr; }
+ if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; }
+ size_qs = size_sas = size_d = 0;
+ }
+};
+
+struct ggml_tensor_extra_cl_iq2_xs {
+ bool noshuffle = false;
+ cl_mem qs = nullptr; // packed grid indices and sign codes
+ cl_mem sc = nullptr; // packed scales
+ cl_mem d = nullptr; // super-block scales
+
+ size_t size_qs = 0, size_sc = 0, size_d = 0;
+
+ ~ggml_tensor_extra_cl_iq2_xs() { reset(); }
+
+ void reset() {
+ noshuffle = false;
+ if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; }
+ if (sc != nullptr) { CL_CHECK(clReleaseMemObject(sc)); sc = nullptr; }
+ if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; }
+ size_qs = size_sc = size_d = 0;
+ }
+};
+
+struct ggml_tensor_extra_cl_iq2_s {
+ bool noshuffle = false;
+ cl_mem qs = nullptr; // grid index low bits
+ cl_mem sg = nullptr; // sign bits
+ cl_mem qh = nullptr; // grid index high bits
+ cl_mem sc = nullptr; // packed scales
+ cl_mem d = nullptr; // super-block scales
+
+ size_t size_qs = 0, size_sg = 0, size_qh = 0, size_sc = 0, size_d = 0;
+
+ ~ggml_tensor_extra_cl_iq2_s() { reset(); }
+
+ void reset() {
+ noshuffle = false;
+ if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; }
+ if (sg != nullptr) { CL_CHECK(clReleaseMemObject(sg)); sg = nullptr; }
+ if (qh != nullptr) { CL_CHECK(clReleaseMemObject(qh)); qh = nullptr; }
+ if (sc != nullptr) { CL_CHECK(clReleaseMemObject(sc)); sc = nullptr; }
+ if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; }
+ size_qs = size_sg = size_qh = size_sc = size_d = 0;
+ }
+};
+
+struct ggml_tensor_extra_cl_q6_K {
+ // Lower 4 bits of quantized weights.
+ cl_mem ql = nullptr;
+ // Lower 4 bits as image1d_buffer_t
+ cl_mem ql_img = nullptr;
+ // Upper 2 bits of quantized weights.
+ cl_mem qh = nullptr;
+ // Upper 2 bits as image1d_buffer_t
+ cl_mem qh_img = nullptr;
+ // Scales for each block.
+ cl_mem s = nullptr;
+ // Scales for each super block.
+ cl_mem d = nullptr;
+
+ size_t size_ql = 0;
+ size_t size_qh = 0;
+ size_t size_s = 0;
+ size_t size_d = 0;
+
+ ~ggml_tensor_extra_cl_q6_K() {
+ reset();
+ }
+
+ void reset() {
+ if (ql != nullptr) {
+ CL_CHECK(clReleaseMemObject(ql));
+ ql = nullptr;
+ }
+ if (qh != nullptr) {
+ CL_CHECK(clReleaseMemObject(qh));
+ qh = nullptr;
+ }
if (s != nullptr) {
CL_CHECK(clReleaseMemObject(s));
s = nullptr;
@@ -9569,6 +10101,60 @@ static bool ggml_cl_q3_k_use_noshuffle(const ggml_backend_opencl_context * backe
#endif
}
+static bool ggml_cl_iq2_s_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) {
+#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS)
+ if (const char * env = getenv("GGML_OPENCL_IQ2_S_NOSHUFFLE")) {
+ if (env[0] == '0') {
+ return false;
+ }
+ }
+ return tensor->type == GGML_TYPE_IQ2_S &&
+ backend_ctx->gpu_family == ADRENO &&
+ ggml_is_contiguous(tensor) && !tensor->view_src &&
+ use_adreno_kernels(backend_ctx, tensor);
+#else
+ GGML_UNUSED(backend_ctx);
+ GGML_UNUSED(tensor);
+ return false;
+#endif
+}
+
+static bool ggml_cl_iq2_xs_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) {
+#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS)
+ if (const char * env = getenv("GGML_OPENCL_IQ2_XS_NOSHUFFLE")) {
+ if (env[0] == '0') {
+ return false;
+ }
+ }
+ return tensor->type == GGML_TYPE_IQ2_XS &&
+ backend_ctx->gpu_family == ADRENO &&
+ ggml_is_contiguous(tensor) && !tensor->view_src &&
+ use_adreno_kernels(backend_ctx, tensor);
+#else
+ GGML_UNUSED(backend_ctx);
+ GGML_UNUSED(tensor);
+ return false;
+#endif
+}
+
+static bool ggml_cl_iq2_xxs_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) {
+#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS)
+ if (const char * env = getenv("GGML_OPENCL_IQ2_XXS_NOSHUFFLE")) {
+ if (env[0] == '0') {
+ return false;
+ }
+ }
+ return tensor->type == GGML_TYPE_IQ2_XXS &&
+ backend_ctx->gpu_family == ADRENO &&
+ ggml_is_contiguous(tensor) && !tensor->view_src &&
+ use_adreno_kernels(backend_ctx, tensor);
+#else
+ GGML_UNUSED(backend_ctx);
+ GGML_UNUSED(tensor);
+ return false;
+#endif
+}
+
inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
// The moe weight repack kernels *_trans4_ns alias a private ushort8 through a uchar*.
// Certain compilers (found with some A7x and A6x) miscompiles this, corrupting the weights.
@@ -10094,6 +10680,24 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
} else if (op->src[0]->type == GGML_TYPE_Q4_0) {
// Non-contig src0 routes through on-device dequant-to-f16.
return op->src[1]->type == GGML_TYPE_F32;
+ } else if (op->src[0]->type == GGML_TYPE_IQ2_XXS) {
+ const ggml_tensor * weight = op->src[0]->view_src;
+ if (weight && ggml_cl_iq2_xxs_use_noshuffle(backend_ctx, weight)) {
+ return false;
+ }
+ return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]);
+ } else if (op->src[0]->type == GGML_TYPE_IQ2_XS) {
+ const ggml_tensor * weight = op->src[0]->view_src;
+ if (weight && ggml_cl_iq2_xs_use_noshuffle(backend_ctx, weight)) {
+ return false;
+ }
+ return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]);
+ } else if (op->src[0]->type == GGML_TYPE_IQ2_S) {
+ const ggml_tensor * weight = op->src[0]->view_src;
+ if (weight && ggml_cl_iq2_s_use_noshuffle(backend_ctx, weight)) {
+ return false;
+ }
+ return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]);
} else if (op->src[0]->type == GGML_TYPE_Q3_K) {
const ggml_tensor * weight = op->src[0]->view_src;
if (weight && ggml_cl_q3_k_use_noshuffle(backend_ctx, weight)) {
@@ -10489,6 +11093,15 @@ struct ggml_backend_opencl_buffer_context {
for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K) {
delete e;
}
+ for (ggml_tensor_extra_cl_iq2_s * e : temp_tensor_extras_iq2_s) {
+ delete e;
+ }
+ for (ggml_tensor_extra_cl_iq2_xs * e : temp_tensor_extras_iq2_xs) {
+ delete e;
+ }
+ for (ggml_tensor_extra_cl_iq2_xxs * e : temp_tensor_extras_iq2_xxs) {
+ delete e;
+ }
for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) {
delete e;
}
@@ -10498,6 +11111,15 @@ struct ggml_backend_opencl_buffer_context {
for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K_in_use) {
delete e;
}
+ for (ggml_tensor_extra_cl_iq2_s * e : temp_tensor_extras_iq2_s_in_use) {
+ delete e;
+ }
+ for (ggml_tensor_extra_cl_iq2_xs * e : temp_tensor_extras_iq2_xs_in_use) {
+ delete e;
+ }
+ for (ggml_tensor_extra_cl_iq2_xxs * e : temp_tensor_extras_iq2_xxs_in_use) {
+ delete e;
+ }
for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K) {
delete e;
}
@@ -10701,6 +11323,51 @@ struct ggml_backend_opencl_buffer_context {
return extra;
}
+ ggml_tensor_extra_cl_iq2_s * ggml_opencl_alloc_temp_tensor_extra_iq2_s() {
+ ggml_tensor_extra_cl_iq2_s * extra;
+ if (temp_tensor_extras_iq2_s.empty()) {
+ extra = new ggml_tensor_extra_cl_iq2_s();
+ } else {
+ extra = temp_tensor_extras_iq2_s.back();
+ temp_tensor_extras_iq2_s.pop_back();
+ }
+
+ temp_tensor_extras_iq2_s_in_use.push_back(extra);
+
+ extra->reset();
+ return extra;
+ }
+
+ ggml_tensor_extra_cl_iq2_xs * ggml_opencl_alloc_temp_tensor_extra_iq2_xs() {
+ ggml_tensor_extra_cl_iq2_xs * extra;
+ if (temp_tensor_extras_iq2_xs.empty()) {
+ extra = new ggml_tensor_extra_cl_iq2_xs();
+ } else {
+ extra = temp_tensor_extras_iq2_xs.back();
+ temp_tensor_extras_iq2_xs.pop_back();
+ }
+
+ temp_tensor_extras_iq2_xs_in_use.push_back(extra);
+
+ extra->reset();
+ return extra;
+ }
+
+ ggml_tensor_extra_cl_iq2_xxs * ggml_opencl_alloc_temp_tensor_extra_iq2_xxs() {
+ ggml_tensor_extra_cl_iq2_xxs * extra;
+ if (temp_tensor_extras_iq2_xxs.empty()) {
+ extra = new ggml_tensor_extra_cl_iq2_xxs();
+ } else {
+ extra = temp_tensor_extras_iq2_xxs.back();
+ temp_tensor_extras_iq2_xxs.pop_back();
+ }
+
+ temp_tensor_extras_iq2_xxs_in_use.push_back(extra);
+
+ extra->reset();
+ return extra;
+ }
+
ggml_tensor_extra_cl_q6_K * ggml_opencl_alloc_temp_tensor_extra_q6_K() {
ggml_tensor_extra_cl_q6_K * extra;
if (temp_tensor_extras_q6_K.empty()) {
@@ -10787,6 +11454,21 @@ struct ggml_backend_opencl_buffer_context {
}
temp_tensor_extras_q3_K_in_use.clear();
+ for (ggml_tensor_extra_cl_iq2_s * e : temp_tensor_extras_iq2_s_in_use) {
+ temp_tensor_extras_iq2_s.push_back(e);
+ }
+ temp_tensor_extras_iq2_s_in_use.clear();
+
+ for (ggml_tensor_extra_cl_iq2_xs * e : temp_tensor_extras_iq2_xs_in_use) {
+ temp_tensor_extras_iq2_xs.push_back(e);
+ }
+ temp_tensor_extras_iq2_xs_in_use.clear();
+
+ for (ggml_tensor_extra_cl_iq2_xxs * e : temp_tensor_extras_iq2_xxs_in_use) {
+ temp_tensor_extras_iq2_xxs.push_back(e);
+ }
+ temp_tensor_extras_iq2_xxs_in_use.clear();
+
q8_0_soa_tensors.clear();
q4_0_soa_tensors.clear();
}
@@ -10820,8 +11502,14 @@ struct ggml_backend_opencl_buffer_context {
std::vector<ggml_tensor_extra_cl_q5_K *> temp_tensor_extras_q5_K_in_use;
std::vector<ggml_tensor_extra_cl_q2_K *> temp_tensor_extras_q2_K;
std::vector<ggml_tensor_extra_cl_q3_K *> temp_tensor_extras_q3_K;
+ std::vector<ggml_tensor_extra_cl_iq2_s *> temp_tensor_extras_iq2_s;
+ std::vector<ggml_tensor_extra_cl_iq2_xs *> temp_tensor_extras_iq2_xs;
+ std::vector<ggml_tensor_extra_cl_iq2_xxs *> temp_tensor_extras_iq2_xxs;
std::vector<ggml_tensor_extra_cl_q2_K *> temp_tensor_extras_q2_K_in_use;
std::vector<ggml_tensor_extra_cl_q3_K *> temp_tensor_extras_q3_K_in_use;
+ std::vector<ggml_tensor_extra_cl_iq2_s *> temp_tensor_extras_iq2_s_in_use;
+ std::vector<ggml_tensor_extra_cl_iq2_xs *> temp_tensor_extras_iq2_xs_in_use;
+ std::vector<ggml_tensor_extra_cl_iq2_xxs *> temp_tensor_extras_iq2_xxs_in_use;
std::vector<ggml_tensor_extra_cl_q6_K *> temp_tensor_extras_q6_K;
std::vector<ggml_tensor_extra_cl_q6_K *> temp_tensor_extras_q6_K_in_use;
@@ -11163,18 +11851,24 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
return;
}
- if (tensor->type == GGML_TYPE_Q1_0) {
+ if (tensor->type == GGML_TYPE_IQ2_S) {
+ GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor));
ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;
- GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized");
+ GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized");
- // Allocate the new extra and create aliases from the original.
ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;
- ggml_tensor_extra_cl_q1_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q1_0();
+ ggml_tensor_extra_cl_iq2_s * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq2_s();
+ extra->noshuffle = ggml_cl_iq2_s_use_noshuffle(backend_ctx, tensor);
- // q1_0 block = ggml_half d + (QK1_0/8) quant bytes = 2 + 16 = 18 bytes
- size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t);
- size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8);
- GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size");
+ const size_t blck = (size_t)ggml_blck_size(tensor->type);
+ const size_t n_blk = ggml_nelements(tensor) / blck;
+
+ const size_t size_qs = n_blk * 32;
+ const size_t size_sg = n_blk * 32;
+ const size_t size_qh = n_blk * 8;
+ const size_t size_sc = n_blk * 8;
+ const size_t size_d = n_blk * 2;
+ GGML_ASSERT(size_qs + size_sg + size_qh + size_sc + size_d == ggml_nbytes(tensor) && "Incorrect tensor size");
cl_int err;
cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,
@@ -11184,79 +11878,308 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
queue, data_device, CL_TRUE, 0,
ggml_nbytes(tensor), data, 0, NULL, NULL));
- // The original tensor memory is divided into scales and quants, i.e.,
- // we first store scales, then quants.
cl_buffer_region region;
-
- // Create subbuffer for scales.
region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment);
- region.size = size_d;
- extra->d = clCreateSubBuffer(
- extra_orig->data_device, CL_MEM_READ_WRITE,
- CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
- CL_CHECK(err);
- auto previous_origin = region.origin;
+ region.size = size_qs;
+ CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
- // Create subbuffer for quants.
- region.origin = align_to(previous_origin + size_d, backend_ctx->alignment);
- region.size = size_q;
- extra->q = clCreateSubBuffer(
- extra_orig->data_device, CL_MEM_READ_WRITE,
- CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
- CL_CHECK(err);
+ region.origin = align_to(region.origin + size_qs, backend_ctx->alignment);
+ region.size = size_sg;
+ CL_CHECK((extra->sg = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
- cl_kernel kernel = backend_ctx->kernel_convert_block_q1_0;
+ region.origin = align_to(region.origin + size_sg, backend_ctx->alignment);
+ region.size = size_qh;
+ CL_CHECK((extra->qh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
- CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));
- CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q));
- CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d));
+ region.origin = align_to(region.origin + size_qh, backend_ctx->alignment);
+ region.size = size_sc;
+ CL_CHECK((extra->sc = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
- size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};
- size_t local_work_size[] = {64, 1, 1};
+ region.origin = align_to(region.origin + size_sc, backend_ctx->alignment);
+ region.size = size_d;
+ CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+
+ cl_kernel kernel = backend_ctx->kernel_convert_block_iq2_s;
+ cl_ulong nb_arg = (cl_ulong)n_blk;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sg));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->qh));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->sc));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra->d));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb_arg));
+
+ size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 };
+ size_t local_work_size[] = { 64, 1, 1 };
cl_event evt;
CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));
CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clReleaseEvent(evt));
CL_CHECK(clReleaseMemObject(data_device));
- tensor->extra = extra;
-
- // q is uint32 (32 sign bits each); d is one half per 128-block.
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
- if (enable_adreno_trans_weight(backend_ctx, tensor)) {
- int M = tensor->ne[1]; // ne01
- int K = tensor->ne[0]; // ne00
-
- GGML_ASSERT(K % 128 == 0);
- GGML_ASSERT(M % 4 == 0);
- GGML_ASSERT(tensor->ne[2] == 1);
- GGML_ASSERT(tensor->ne[3] == 1);
+ if (extra->noshuffle) {
+ const int m = tensor->ne[1];
+ const int k = tensor->ne[0];
+ transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/8, m, true, true);
+ transpose_2d_as_8b(backend_ctx, extra->sg, extra->sg, size_sg, k/8, m, true, true);
+ transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, k/32, m, true, true);
+ transpose_2d_as_8b(backend_ctx, extra->sc, extra->sc, size_sc, k/32, m, true, true);
+ transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, extra->d, size_d, k/(int)blck, m, true, true);
+ }
+#endif
- transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/32, M);
- transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/128, M);
- } // end transpose
-#endif // GGML_OPENCL_USE_ADRENO_KERNELS
+ extra->size_qs = size_qs;
+ extra->size_sg = size_sg;
+ extra->size_qh = size_qh;
+ extra->size_sc = size_sc;
+ extra->size_d = size_d;
+ tensor->extra = extra;
return;
}
- // We separate the quantized bits and scale from block_q4_0 by using an
- // additional kernel, where each thread handles a block. We first read the
- // original weights into a temporary buffer, then create two separate
- // buffers for quantized bits and scales, which are then populated by the
- // conversion kernel.
- if (tensor->type == GGML_TYPE_Q4_0) {
- // Views can't SoA-ify here — parent owns the layout (see q8_0 guard).
- // A view of an AoS tensor (the KV cache, written by a state restore)
- // is written in place.
- if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) {
- if (!ggml_cl_is_q4_0_soa(tensor)) {
- ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra;
- CL_CHECK(clEnqueueWriteBuffer(
- queue, extra_aos->data_device, CL_TRUE,
- extra_aos->offset + tensor->view_offs + offset,
- size, data, 0, NULL, NULL));
- }
- return;
+
+ if (tensor->type == GGML_TYPE_IQ2_XS) {
+ GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor));
+ ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;
+ GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized");
+
+ ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;
+ ggml_tensor_extra_cl_iq2_xs * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq2_xs();
+ extra->noshuffle = ggml_cl_iq2_xs_use_noshuffle(backend_ctx, tensor);
+
+ const size_t blck = (size_t)ggml_blck_size(tensor->type);
+ const size_t n_blk = ggml_nelements(tensor) / blck;
+
+ const size_t size_qs = n_blk * 64;
+ const size_t size_sc = n_blk * 8;
+ const size_t size_d = n_blk * 2;
+ GGML_ASSERT(size_qs + size_sc + size_d == ggml_nbytes(tensor) && "Incorrect tensor size");
+
+ cl_int err;
+ cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,
+ ggml_nbytes(tensor), NULL, &err);
+ CL_CHECK(err);
+ CL_CHECK(clEnqueueWriteBuffer(
+ queue, data_device, CL_TRUE, 0,
+ ggml_nbytes(tensor), data, 0, NULL, NULL));
+
+ cl_buffer_region region;
+ region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment);
+ region.size = size_qs;
+ CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+
+ region.origin = align_to(region.origin + size_qs, backend_ctx->alignment);
+ region.size = size_sc;
+ CL_CHECK((extra->sc = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+
+ region.origin = align_to(region.origin + size_sc, backend_ctx->alignment);
+ region.size = size_d;
+ CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+
+ cl_kernel kernel = backend_ctx->kernel_convert_block_iq2_xs;
+ cl_ulong nb_arg = (cl_ulong)n_blk;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sc));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg));
+
+ size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 };
+ size_t local_work_size[] = { 64, 1, 1 };
+
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clReleaseEvent(evt));
+ CL_CHECK(clReleaseMemObject(data_device));
+
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ if (extra->noshuffle) {
+ const int m = tensor->ne[1];
+ const int k = tensor->ne[0];
+ transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->qs, extra->qs, size_qs, k/8, m, true, true);
+ transpose_2d_as_8b(backend_ctx, extra->sc, extra->sc, size_sc, k/32, m, true, true);
+ transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, extra->d, size_d, k/(int)blck, m, true, true);
+ }
+#endif
+
+ extra->size_qs = size_qs;
+ extra->size_sc = size_sc;
+ extra->size_d = size_d;
+
+ tensor->extra = extra;
+ return;
+ }
+
+ if (tensor->type == GGML_TYPE_IQ2_XXS) {
+ GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor));
+ ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;
+ GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized");
+
+ ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;
+ ggml_tensor_extra_cl_iq2_xxs * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq2_xxs();
+ extra->noshuffle = ggml_cl_iq2_xxs_use_noshuffle(backend_ctx, tensor);
+
+ const size_t blck = (size_t)ggml_blck_size(tensor->type);
+ const size_t n_blk = ggml_nelements(tensor) / blck;
+
+ const size_t size_qs = n_blk * 32;
+ const size_t size_sas = n_blk * 32;
+ const size_t size_d = n_blk * 2;
+ GGML_ASSERT(size_qs + size_sas + size_d == ggml_nbytes(tensor) && "Incorrect tensor size");
+
+ cl_int err;
+ cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,
+ ggml_nbytes(tensor), NULL, &err);
+ CL_CHECK(err);
+ CL_CHECK(clEnqueueWriteBuffer(
+ queue, data_device, CL_TRUE, 0,
+ ggml_nbytes(tensor), data, 0, NULL, NULL));
+
+ cl_buffer_region region;
+ region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment);
+ region.size = size_qs;
+ CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+
+ region.origin = align_to(region.origin + size_qs, backend_ctx->alignment);
+ region.size = size_sas;
+ CL_CHECK((extra->sas = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+
+ region.origin = align_to(region.origin + size_sas, backend_ctx->alignment);
+ region.size = size_d;
+ CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
+
+ cl_kernel kernel = backend_ctx->kernel_convert_block_iq2_xxs;
+ cl_ulong nb_arg = (cl_ulong)n_blk;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sas));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg));
+
+ size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 };
+ size_t local_work_size[] = { 64, 1, 1 };
+
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clReleaseEvent(evt));
+ CL_CHECK(clReleaseMemObject(data_device));
+
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ if (extra->noshuffle) {
+ const int m = tensor->ne[1];
+ const int k = tensor->ne[0];
+ transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/8, m, true, true);
+ transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->sas, extra->sas, size_sas, k/32, m, true, true);
+ transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, extra->d, size_d, k/(int)blck, m, true, true);
+ }
+#endif
+
+ extra->size_qs = size_qs;
+ extra->size_sas = size_sas;
+ extra->size_d = size_d;
+
+ tensor->extra = extra;
+ return;
+ }
+
+ if (tensor->type == GGML_TYPE_Q1_0) {
+ ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;
+ GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized");
+
+ // Allocate the new extra and create aliases from the original.
+ ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;
+ ggml_tensor_extra_cl_q1_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q1_0();
+
+ // q1_0 block = ggml_half d + (QK1_0/8) quant bytes = 2 + 16 = 18 bytes
+ size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t);
+ size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8);
+ GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size");
+
+ cl_int err;
+ cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,
+ ggml_nbytes(tensor), NULL, &err);
+ CL_CHECK(err);
+ CL_CHECK(clEnqueueWriteBuffer(
+ queue, data_device, CL_TRUE, 0,
+ ggml_nbytes(tensor), data, 0, NULL, NULL));
+
+ // The original tensor memory is divided into scales and quants, i.e.,
+ // we first store scales, then quants.
+ cl_buffer_region region;
+
+ // Create subbuffer for scales.
+ region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment);
+ region.size = size_d;
+ extra->d = clCreateSubBuffer(
+ extra_orig->data_device, CL_MEM_READ_WRITE,
+ CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+ auto previous_origin = region.origin;
+
+ // Create subbuffer for quants.
+ region.origin = align_to(previous_origin + size_d, backend_ctx->alignment);
+ region.size = size_q;
+ extra->q = clCreateSubBuffer(
+ extra_orig->data_device, CL_MEM_READ_WRITE,
+ CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+
+ cl_kernel kernel = backend_ctx->kernel_convert_block_q1_0;
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d));
+
+ size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};
+ size_t local_work_size[] = {64, 1, 1};
+
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clReleaseMemObject(data_device));
+
+ tensor->extra = extra;
+
+ // q is uint32 (32 sign bits each); d is one half per 128-block.
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ if (enable_adreno_trans_weight(backend_ctx, tensor)) {
+ int M = tensor->ne[1]; // ne01
+ int K = tensor->ne[0]; // ne00
+
+ GGML_ASSERT(K % 128 == 0);
+ GGML_ASSERT(M % 4 == 0);
+ GGML_ASSERT(tensor->ne[2] == 1);
+ GGML_ASSERT(tensor->ne[3] == 1);
+
+ transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/32, M);
+ transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/128, M);
+ } // end transpose
+#endif // GGML_OPENCL_USE_ADRENO_KERNELS
+
+ return;
+ }
+ // We separate the quantized bits and scale from block_q4_0 by using an
+ // additional kernel, where each thread handles a block. We first read the
+ // original weights into a temporary buffer, then create two separate
+ // buffers for quantized bits and scales, which are then populated by the
+ // conversion kernel.
+ if (tensor->type == GGML_TYPE_Q4_0) {
+ // Views can't SoA-ify here — parent owns the layout (see q8_0 guard).
+ // A view of an AoS tensor (the KV cache, written by a state restore)
+ // is written in place.
+ if (tensor->view_src != nullptr || !ggml_is_contiguous(tensor)) {
+ if (!ggml_cl_is_q4_0_soa(tensor)) {
+ ggml_tensor_extra_cl * extra_aos = (ggml_tensor_extra_cl *) tensor->extra;
+ CL_CHECK(clEnqueueWriteBuffer(
+ queue, extra_aos->data_device, CL_TRUE,
+ extra_aos->offset + tensor->view_offs + offset,
+ size, data, 0, NULL, NULL));
+ }
+ return;
}
// Tensors should have been preallocated, therefore they should
// already have ggml_tensor_extra_cl as extra.
@@ -12948,41 +13871,196 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,
return;
}
- if (tensor->type == GGML_TYPE_Q1_0) {
- ggml_tensor_extra_cl_q1_0 * extra = (ggml_tensor_extra_cl_q1_0 *)tensor->extra;
-
-#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
- if (enable_adreno_trans_weight(backend_ctx, tensor)) {
- ggml_cl_buffer buf_trans_q;
- ggml_cl_buffer buf_trans_d;
- ggml_cl_buffer buf_unpacked;
-
- int M = tensor->ne[1];
- int K = tensor->ne[0];
-
- size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t);
- size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8);
-
- buf_trans_q.allocate(backend_ctx->context, size_q);
- buf_trans_d.allocate(backend_ctx->context, size_d);
- buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor));
+ if (tensor->type == GGML_TYPE_IQ2_S) {
+ if (tensor->view_src) {
+ offset += tensor->view_offs;
+ tensor = tensor->view_src;
+ }
+ ggml_tensor_extra_cl_iq2_s * extra = (ggml_tensor_extra_cl_iq2_s *)tensor->extra;
- transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/32);
- transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/128);
+ const size_t blck = (size_t)ggml_blck_size(tensor->type);
- cl_kernel kernel = backend_ctx->kernel_restore_block_q1_0;
- CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer));
- CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer));
- CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer));
+ ggml_cl_buffer buf_unpacked;
+ buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor));
- size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};
- size_t local_work_size[] = {1, 1, 1};
+ const size_t n_blk = (size_t)ggml_nelements(tensor) / blck;
+ cl_ulong nb_arg = (cl_ulong)n_blk;
- cl_event evt;
- CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));
- CL_CHECK(clWaitForEvents(1, &evt));
- CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL));
- return;
+ cl_mem qs = extra->qs, sg = extra->sg, qh = extra->qh, sc = extra->sc, d = extra->d;
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ ggml_cl_buffer buf_qs, buf_sg, buf_qh, buf_sc, buf_d;
+ if (extra->noshuffle) {
+ buf_qs.allocate(context, extra->size_qs);
+ qs = buf_qs.buffer;
+ buf_sg.allocate(context, extra->size_sg);
+ sg = buf_sg.buffer;
+ buf_qh.allocate(context, extra->size_qh);
+ qh = buf_qh.buffer;
+ buf_sc.allocate(context, extra->size_sc);
+ sc = buf_sc.buffer;
+ buf_d.allocate(context, extra->size_d);
+ d = buf_d.buffer;
+ const int m = tensor->ne[1];
+ const int k = tensor->ne[0];
+ transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/8, true, true);
+ transpose_2d_as_8b(backend_ctx, extra->sg, sg, extra->size_sg, m, k/8, true, true);
+ transpose_2d_as_8b(backend_ctx, extra->qh, qh, extra->size_qh, m, k/32, true, true);
+ transpose_2d_as_8b(backend_ctx, extra->sc, sc, extra->size_sc, m, k/32, true, true);
+ transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, d, extra->size_d, m, k/(int)blck, true, true);
+ }
+#endif
+ cl_kernel kernel = backend_ctx->kernel_restore_block_iq2_s;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sg));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &qh));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &sc));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &d));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &buf_unpacked.buffer));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb_arg));
+
+ size_t gws[] = { (n_blk + 63) / 64 * 64 };
+ size_t lws[] = { 64 };
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clReleaseEvent(evt));
+ CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL));
+ return;
+ }
+
+ if (tensor->type == GGML_TYPE_IQ2_XS) {
+ if (tensor->view_src) {
+ offset += tensor->view_offs;
+ tensor = tensor->view_src;
+ }
+ ggml_tensor_extra_cl_iq2_xs * extra = (ggml_tensor_extra_cl_iq2_xs *)tensor->extra;
+
+ const size_t blck = (size_t)ggml_blck_size(tensor->type);
+
+ ggml_cl_buffer buf_unpacked;
+ buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor));
+
+ const size_t n_blk = (size_t)ggml_nelements(tensor) / blck;
+ cl_ulong nb_arg = (cl_ulong)n_blk;
+
+ cl_mem qs = extra->qs, sc = extra->sc, d = extra->d;
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ ggml_cl_buffer buf_qs, buf_sc, buf_d;
+ if (extra->noshuffle) {
+ buf_qs.allocate(context, extra->size_qs);
+ qs = buf_qs.buffer;
+ buf_sc.allocate(context, extra->size_sc);
+ sc = buf_sc.buffer;
+ buf_d.allocate(context, extra->size_d);
+ d = buf_d.buffer;
+ const int m = tensor->ne[1];
+ const int k = tensor->ne[0];
+ transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->qs, qs, extra->size_qs, m, k/8, true, true);
+ transpose_2d_as_8b(backend_ctx, extra->sc, sc, extra->size_sc, m, k/32, true, true);
+ transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, d, extra->size_d, m, k/(int)blck, true, true);
+ }
+#endif
+ cl_kernel kernel = backend_ctx->kernel_restore_block_iq2_xs;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sc));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &d));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg));
+
+ size_t gws[] = { (n_blk + 63) / 64 * 64 };
+ size_t lws[] = { 64 };
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clReleaseEvent(evt));
+ CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL));
+ return;
+ }
+
+ if (tensor->type == GGML_TYPE_IQ2_XXS) {
+ if (tensor->view_src) {
+ offset += tensor->view_offs;
+ tensor = tensor->view_src;
+ }
+ ggml_tensor_extra_cl_iq2_xxs * extra = (ggml_tensor_extra_cl_iq2_xxs *)tensor->extra;
+
+ const size_t blck = (size_t)ggml_blck_size(tensor->type);
+
+ ggml_cl_buffer buf_unpacked;
+ buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor));
+
+ const size_t n_blk = (size_t)ggml_nelements(tensor) / blck;
+ cl_ulong nb_arg = (cl_ulong)n_blk;
+
+ cl_mem qs = extra->qs, sas = extra->sas, d = extra->d;
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ ggml_cl_buffer buf_qs, buf_sas, buf_d;
+ if (extra->noshuffle) {
+ buf_qs.allocate(context, extra->size_qs);
+ qs = buf_qs.buffer;
+ buf_sas.allocate(context, extra->size_sas);
+ sas = buf_sas.buffer;
+ buf_d.allocate(context, extra->size_d);
+ d = buf_d.buffer;
+ const int m = tensor->ne[1];
+ const int k = tensor->ne[0];
+ transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/8, true, true);
+ transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->sas, sas, extra->size_sas, m, k/32, true, true);
+ transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, d, extra->size_d, m, k/(int)blck, true, true);
+ }
+#endif
+ cl_kernel kernel = backend_ctx->kernel_restore_block_iq2_xxs;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sas));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &d));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &nb_arg));
+
+ size_t gws[] = { (n_blk + 63) / 64 * 64 };
+ size_t lws[] = { 64 };
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clReleaseEvent(evt));
+ CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL));
+ return;
+ }
+
+ if (tensor->type == GGML_TYPE_Q1_0) {
+ ggml_tensor_extra_cl_q1_0 * extra = (ggml_tensor_extra_cl_q1_0 *)tensor->extra;
+
+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
+ if (enable_adreno_trans_weight(backend_ctx, tensor)) {
+ ggml_cl_buffer buf_trans_q;
+ ggml_cl_buffer buf_trans_d;
+ ggml_cl_buffer buf_unpacked;
+
+ int M = tensor->ne[1];
+ int K = tensor->ne[0];
+
+ size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t);
+ size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8);
+
+ buf_trans_q.allocate(backend_ctx->context, size_q);
+ buf_trans_d.allocate(backend_ctx->context, size_d);
+ buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor));
+
+ transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/32);
+ transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/128);
+
+ cl_kernel kernel = backend_ctx->kernel_restore_block_q1_0;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer));
+
+ size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};
+ size_t local_work_size[] = {1, 1, 1};
+
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL));
+ return;
}
#endif
@@ -25420,9 +26498,491 @@ static bool ggml_cl_mul_mat_q3_K_dp4a(ggml_backend_t backend, const ggml_tensor
return true;
}
-static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+static bool ggml_cl_mul_mat_iq2_s_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
+ ggml_tensor_extra_cl_iq2_s * extra0 = (ggml_tensor_extra_cl_iq2_s *)src0->extra;
+ ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
+
+ cl_ulong offset1 = extra1->offset + src1->view_offs;
+ cl_ulong offsetd = extrad->offset + dst->view_offs;
+ const int m = src0->ne[1];
+ const int n = src1->ne[1];
+ const int k = src0->ne[0];
+
+ const int tile_n = n <= 8 ? 8 : 32;
+ cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq2_s_q8_1_dp4a_narrow
+ : backend_ctx->kernel_gemm_noshuffle_iq2_s_q8_1_dp4a;
+ const char * env = getenv("GGML_OPENCL_IQ2_S_DENSE_DP4A");
+ if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') ||
+ n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
+ return false;
+ }
+
+ cl_int err;
+ cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) };
+ cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+ const size_t n_blocks = (size_t)n*(k/32);
+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k);
+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half));
+
+ cl_int nb = n_blocks;
+ cl_kernel quant = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation));
+ CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb));
+ size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 };
+ size_t q_local[] = { 64 };
+ backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst);
+
+ CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs));
+ CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sg));
+ CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->qh));
+ CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &extra0->sc));
+ CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &extra0->d));
+ CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(gemm, 7, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(gemm, 8, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &m));
+ CL_CHECK(clSetKernelArg(gemm, 10, sizeof(int), &n));
+ CL_CHECK(clSetKernelArg(gemm, 11, sizeof(int), &k));
+ size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) };
+ size_t local[] = { 64, 1, 1 };
+ backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst);
+ CL_CHECK(clReleaseMemObject(activation));
+ return true;
+}
+
+static bool ggml_cl_mul_mat_iq2_xs_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
+ ggml_tensor_extra_cl_iq2_xs * extra0 = (ggml_tensor_extra_cl_iq2_xs *)src0->extra;
+ ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
+
+ cl_ulong offset1 = extra1->offset + src1->view_offs;
+ cl_ulong offsetd = extrad->offset + dst->view_offs;
+ const int m = src0->ne[1];
+ const int n = src1->ne[1];
+ const int k = src0->ne[0];
+
+ const int tile_n = n <= 8 ? 8 : 32;
+ cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a_narrow
+ : backend_ctx->kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a;
+ const char * env = getenv("GGML_OPENCL_IQ2_XS_DENSE_DP4A");
+ if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') ||
+ n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
+ return false;
+ }
+
+ cl_int err;
+ cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) };
+ cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+ const size_t n_blocks = (size_t)n*(k/32);
+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k);
+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half));
+
+ cl_int nb = n_blocks;
+ cl_kernel quant = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation));
+ CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb));
+ size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 };
+ size_t q_local[] = { 64 };
+ backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst);
+
+ CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs));
+ CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sc));
+ CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->d));
+ CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(gemm, 7, sizeof(int), &m));
+ CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &n));
+ CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &k));
+ size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) };
+ size_t local[] = { 64, 1, 1 };
+ backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst);
+ CL_CHECK(clReleaseMemObject(activation));
+ return true;
+}
+
+static bool ggml_cl_mul_mat_iq2_xxs_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
+ ggml_tensor_extra_cl_iq2_xxs * extra0 = (ggml_tensor_extra_cl_iq2_xxs *)src0->extra;
+ ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
+
+ cl_ulong offset1 = extra1->offset + src1->view_offs;
+ cl_ulong offsetd = extrad->offset + dst->view_offs;
+ const int m = src0->ne[1];
+ const int n = src1->ne[1];
+ const int k = src0->ne[0];
+
+ const int tile_n = n <= 8 ? 8 : 32;
+ cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a_narrow
+ : backend_ctx->kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a;
+ const char * env = getenv("GGML_OPENCL_IQ2_XXS_DENSE_DP4A");
+ if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') ||
+ n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
+ return false;
+ }
+
+ cl_int err;
+ cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) };
+ cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+ const size_t n_blocks = (size_t)n*(k/32);
+ backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k);
+ backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half));
+ backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half));
+
+ cl_int nb = n_blocks;
+ cl_kernel quant = backend_ctx->kernel_quant_a_q8_1;
+ CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation));
+ CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
+ CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb));
+ size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 };
+ size_t q_local[] = { 64 };
+ backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst);
+
+ CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs));
+ CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sas));
+ CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->d));
+ CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
+ CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
+ CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(gemm, 7, sizeof(int), &m));
+ CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &n));
+ CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &k));
+ size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) };
+ size_t local[] = { 64, 1, 1 };
+ backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst);
+ CL_CHECK(clReleaseMemObject(activation));
+ return true;
+}
+
+static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
+ ggml_tensor_extra_cl_q2_K * extra0 = (ggml_tensor_extra_cl_q2_K *)src0->extra;
+ ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
+
+ cl_ulong offset1 = extra1->offset + src1->view_offs;
+ cl_ulong offsetd = extrad->offset + dst->view_offs;
+ const int m = src0->ne[1];
+ const int n = src1->ne[1];
+ const int k = src0->ne[0];
+
+ if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
+ cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_q2_k_f32;
+ const int stride_d = dst->ne[0];
+ const int stride_b = src1->nb[1]/sizeof(float);
+ CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs));
+ CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->sc));
+ CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->dm));
+ CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(gemv, 7, sizeof(int), &k));
+ CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &m));
+ CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &stride_b));
+ CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_d));
+ const size_t rows = 64*(size_t)backend_ctx->q2_k_mv_rows;
+ const size_t nsg = backend_ctx->q2_k_mv_nsg;
+ size_t global[] = { CEIL_DIV((size_t)m, rows)*64, (size_t)n*nsg, 1 };
+ size_t local[] = { 64, nsg, 1 };
+ backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst);
+ return;
+ }
+
+ if (ggml_cl_mul_mat_q2_K_dp4a(backend, src0, src1, dst)) {
+ return;
+ }
+
+ cl_context context = backend_ctx->context;
+ cl_int err;
+ cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) };
+ cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+
+ cl_image_format img_fmt = { CL_RGBA, CL_FLOAT };
+ cl_image_desc img_desc = {};
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = (size_t)k*n/4;
+ img_desc.buffer = b_sub_buf;
+ cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err);
+ CL_CHECK(err);
+
+ const int padded_n = CEIL_DIV(n, 8)*8;
+ region.origin = 0;
+ region.size = (size_t)k*padded_n*sizeof(cl_half);
+ backend_ctx->prealloc_act_trans.allocate(context, region.size);
+ cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+
+ img_fmt = { CL_RGBA, CL_HALF_FLOAT };
+ img_desc.image_width = (size_t)k*padded_n/4;
+ img_desc.buffer = b_sub_buf_trans;
+ cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err);
+ CL_CHECK(err);
+
+ const int height_b = std::max(n/4, 1);
+ const int width_b = k/4;
+ const int padded_height_b = padded_n/4;
+ cl_kernel kernel = backend_ctx->kernel_transpose_32_16;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b));
+ size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b };
+ size_t t_local[] = { 1, 16 };
+ backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst);
+
+ kernel = backend_ctx->kernel_gemm_noshuffle_q2_k_f32;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sc));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->dm));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &m));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &padded_n));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &k));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n));
+ size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 };
+ size_t local[] = { 1, 128, 1 };
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst);
+
+ CL_CHECK(clReleaseMemObject(b_img_trans));
+ CL_CHECK(clReleaseMemObject(b_sub_buf_trans));
+ CL_CHECK(clReleaseMemObject(b_img));
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+}
+
+static void ggml_cl_mul_mat_q3_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
+ ggml_tensor_extra_cl_q3_K * extra0 = (ggml_tensor_extra_cl_q3_K *)src0->extra;
+ ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
+
+ cl_ulong offset1 = extra1->offset + src1->view_offs;
+ cl_ulong offsetd = extrad->offset + dst->view_offs;
+ const int m = src0->ne[1];
+ const int n = src1->ne[1];
+ const int k = src0->ne[0];
+
+ if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
+ cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_q3_k_f32;
+ const int stride_d = dst->ne[0];
+ const int stride_b = src1->nb[1]/sizeof(float);
+ CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs));
+ CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->hm));
+ CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sc));
+ CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d));
+ CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k));
+ CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m));
+ CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b));
+ CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d));
+ const size_t rows = 64*(size_t)backend_ctx->q3_k_mv_rows;
+ const size_t nsg = backend_ctx->q3_k_mv_nsg;
+ size_t global[] = { CEIL_DIV((size_t)m, rows)*64, (size_t)n*nsg, 1 };
+ size_t local[] = { 64, nsg, 1 };
+ backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst);
+ return;
+ }
+
+ if (ggml_cl_mul_mat_q3_K_dp4a(backend, src0, src1, dst)) {
+ return;
+ }
+
+ cl_context context = backend_ctx->context;
+ cl_int err;
+ cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) };
+ cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+
+ cl_image_format img_fmt = { CL_RGBA, CL_FLOAT };
+ cl_image_desc img_desc = {};
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = (size_t)k*n/4;
+ img_desc.buffer = b_sub_buf;
+ cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err);
+ CL_CHECK(err);
+
+ const int padded_n = CEIL_DIV(n, 8)*8;
+ region.origin = 0;
+ region.size = (size_t)k*padded_n*sizeof(cl_half);
+ backend_ctx->prealloc_act_trans.allocate(context, region.size);
+ cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+
+ img_fmt = { CL_RGBA, CL_HALF_FLOAT };
+ img_desc.image_width = (size_t)k*padded_n/4;
+ img_desc.buffer = b_sub_buf_trans;
+ cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err);
+ CL_CHECK(err);
+
+ const int height_b = std::max(n/4, 1);
+ const int width_b = k/4;
+ const int padded_height_b = padded_n/4;
+ cl_kernel kernel = backend_ctx->kernel_transpose_32_16;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b));
+ size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b };
+ size_t t_local[] = { 1, 16 };
+ backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst);
+
+ kernel = backend_ctx->kernel_gemm_noshuffle_q3_k_f32;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->hm));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->sc));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->d));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &m));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &padded_n));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &k));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n));
+ size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 };
+ size_t local[] = { 1, 128, 1 };
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst);
+
+ CL_CHECK(clReleaseMemObject(b_img_trans));
+ CL_CHECK(clReleaseMemObject(b_sub_buf_trans));
+ CL_CHECK(clReleaseMemObject(b_img));
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+}
+
+static void ggml_cl_mul_mat_iq2_s_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
+ ggml_tensor_extra_cl_iq2_s * extra0 = (ggml_tensor_extra_cl_iq2_s *)src0->extra;
+ ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
+
+ cl_ulong offset1 = extra1->offset + src1->view_offs;
+ cl_ulong offsetd = extrad->offset + dst->view_offs;
+ const int m = src0->ne[1];
+ const int n = src1->ne[1];
+ const int k = src0->ne[0];
+
+ if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
+ cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq2_s_f32;
+ const int stride_d = dst->ne[0];
+ const int stride_b = src1->nb[1]/sizeof(float);
+ CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &backend_ctx->iq2_s_grid_image));
+ CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->qs));
+ CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sg));
+ CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->qh));
+ CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra0->sc));
+ CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_mem), &extra0->d));
+ CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(gemv, 8, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(gemv, 9, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &k));
+ CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &m));
+ CL_CHECK(clSetKernelArg(gemv, 12, sizeof(int), &stride_b));
+ CL_CHECK(clSetKernelArg(gemv, 13, sizeof(int), &stride_d));
+ const size_t nsg = backend_ctx->iq2_s_mv_nsg;
+ size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 };
+ size_t local[] = { 64, nsg, 1 };
+ backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst);
+ return;
+ }
+
+ if (ggml_cl_mul_mat_iq2_s_dp4a(backend, src0, src1, dst)) {
+ return;
+ }
+
+ cl_context context = backend_ctx->context;
+ cl_int err;
+ cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) };
+ cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+
+ cl_image_format img_fmt = { CL_RGBA, CL_FLOAT };
+ cl_image_desc img_desc = {};
+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
+ img_desc.image_width = (size_t)k*n/4;
+ img_desc.buffer = b_sub_buf;
+ cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err);
+ CL_CHECK(err);
+
+ const int padded_n = CEIL_DIV(n, 8)*8;
+ region.origin = 0;
+ region.size = (size_t)k*padded_n*sizeof(cl_half);
+ backend_ctx->prealloc_act_trans.allocate(context, region.size);
+ cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+
+ img_fmt = { CL_RGBA, CL_HALF_FLOAT };
+ img_desc.image_width = (size_t)k*padded_n/4;
+ img_desc.buffer = b_sub_buf_trans;
+ cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err);
+ CL_CHECK(err);
+
+ const int height_b = std::max(n/4, 1);
+ const int width_b = k/4;
+ const int padded_height_b = padded_n/4;
+ cl_kernel kernel = backend_ctx->kernel_transpose_32_16;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b));
+ size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b };
+ size_t t_local[] = { 1, 16 };
+ backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst);
+
+ kernel = backend_ctx->kernel_gemm_noshuffle_iq2_s_f32;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sg));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->qh));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->sc));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0->d));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &b_img_trans));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &m));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &padded_n));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &k));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &n));
+ size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 };
+ size_t local[] = { 1, 128, 1 };
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst);
+
+ CL_CHECK(clReleaseMemObject(b_img_trans));
+ CL_CHECK(clReleaseMemObject(b_sub_buf_trans));
+ CL_CHECK(clReleaseMemObject(b_img));
+ CL_CHECK(clReleaseMemObject(b_sub_buf));
+}
+
+static void ggml_cl_mul_mat_iq2_xs_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
- ggml_tensor_extra_cl_q2_K * extra0 = (ggml_tensor_extra_cl_q2_K *)src0->extra;
+ ggml_tensor_extra_cl_iq2_xs * extra0 = (ggml_tensor_extra_cl_iq2_xs *)src0->extra;
ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
@@ -25433,29 +26993,29 @@ static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_t
const int k = src0->ne[0];
if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
- cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_q2_k_f32;
+ cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq2_xs_f32;
const int stride_d = dst->ne[0];
const int stride_b = src1->nb[1]/sizeof(float);
- CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs));
- CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->sc));
- CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->dm));
- CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra1->data_device));
- CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_ulong), &offset1));
- CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_mem), &extrad->data_device));
- CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_ulong), &offsetd));
- CL_CHECK(clSetKernelArg(gemv, 7, sizeof(int), &k));
- CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &m));
- CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &stride_b));
- CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_d));
- const size_t rows = 64*(size_t)backend_ctx->q2_k_mv_rows;
- const size_t nsg = backend_ctx->q2_k_mv_nsg;
- size_t global[] = { CEIL_DIV((size_t)m, rows)*64, (size_t)n*nsg, 1 };
+ CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &backend_ctx->iq2_xs_grid_image));
+ CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->qs));
+ CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sc));
+ CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d));
+ CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k));
+ CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m));
+ CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b));
+ CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d));
+ const size_t nsg = backend_ctx->iq2_xs_mv_nsg;
+ size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 };
size_t local[] = { 64, nsg, 1 };
backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst);
return;
}
- if (ggml_cl_mul_mat_q2_K_dp4a(backend, src0, src1, dst)) {
+ if (ggml_cl_mul_mat_iq2_xs_dp4a(backend, src0, src1, dst)) {
return;
}
@@ -25499,17 +27059,17 @@ static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_t
size_t t_local[] = { 1, 16 };
backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst);
- kernel = backend_ctx->kernel_gemm_noshuffle_q2_k_f32;
- CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs));
- CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sc));
- CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->dm));
- CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans));
- CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
- CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
- CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &m));
- CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &padded_n));
- CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &k));
- CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n));
+ kernel = backend_ctx->kernel_gemm_noshuffle_iq2_xs_f32;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sc));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->d));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &m));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &padded_n));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &k));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n));
size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 };
size_t local[] = { 1, 128, 1 };
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst);
@@ -25520,9 +27080,9 @@ static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clReleaseMemObject(b_sub_buf));
}
-static void ggml_cl_mul_mat_q3_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+static void ggml_cl_mul_mat_iq2_xxs_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
- ggml_tensor_extra_cl_q3_K * extra0 = (ggml_tensor_extra_cl_q3_K *)src0->extra;
+ ggml_tensor_extra_cl_iq2_xxs * extra0 = (ggml_tensor_extra_cl_iq2_xxs *)src0->extra;
ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
@@ -25533,12 +27093,12 @@ static void ggml_cl_mul_mat_q3_k_f32_adreno(ggml_backend_t backend, const ggml_t
const int k = src0->ne[0];
if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
- cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_q3_k_f32;
+ cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq2_xxs_f32;
const int stride_d = dst->ne[0];
const int stride_b = src1->nb[1]/sizeof(float);
- CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs));
- CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->hm));
- CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sc));
+ CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &backend_ctx->iq2_xxs_grid_image));
+ CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->qs));
+ CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sas));
CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d));
CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device));
CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1));
@@ -25548,15 +27108,14 @@ static void ggml_cl_mul_mat_q3_k_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m));
CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b));
CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d));
- const size_t rows = 64*(size_t)backend_ctx->q3_k_mv_rows;
- const size_t nsg = backend_ctx->q3_k_mv_nsg;
- size_t global[] = { CEIL_DIV((size_t)m, rows)*64, (size_t)n*nsg, 1 };
+ const size_t nsg = backend_ctx->iq2_xxs_mv_nsg;
+ size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 };
size_t local[] = { 64, nsg, 1 };
backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst);
return;
}
- if (ggml_cl_mul_mat_q3_K_dp4a(backend, src0, src1, dst)) {
+ if (ggml_cl_mul_mat_iq2_xxs_dp4a(backend, src0, src1, dst)) {
return;
}
@@ -25600,18 +27159,17 @@ static void ggml_cl_mul_mat_q3_k_f32_adreno(ggml_backend_t backend, const ggml_t
size_t t_local[] = { 1, 16 };
backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst);
- kernel = backend_ctx->kernel_gemm_noshuffle_q3_k_f32;
+ kernel = backend_ctx->kernel_gemm_noshuffle_iq2_xxs_f32;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs));
- CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->hm));
- CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->sc));
- CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->d));
- CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans));
- CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device));
- CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd));
- CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &m));
- CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &padded_n));
- CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &k));
- CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sas));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->d));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &m));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &padded_n));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &k));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &n));
size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 };
size_t local[] = { 1, 128, 1 };
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst);
@@ -25693,6 +27251,9 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)soa0_src->extra;
ggml_tensor_extra_cl_q2_K * extra0_q2_K = (ggml_tensor_extra_cl_q2_K *)soa0_src->extra;
ggml_tensor_extra_cl_q3_K * extra0_q3_K = (ggml_tensor_extra_cl_q3_K *)soa0_src->extra;
+ ggml_tensor_extra_cl_iq2_s * extra0_iq2_s = (ggml_tensor_extra_cl_iq2_s *)soa0_src->extra;
+ ggml_tensor_extra_cl_iq2_xs * extra0_iq2_xs = (ggml_tensor_extra_cl_iq2_xs *)soa0_src->extra;
+ ggml_tensor_extra_cl_iq2_xxs * extra0_iq2_xxs = (ggml_tensor_extra_cl_iq2_xxs *)soa0_src->extra;
ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)soa0_src->extra;
ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)soa0_src->extra;
ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)soa0_src->extra;
@@ -26100,6 +27661,24 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
return;
}
+ // iq2_s x fp32
+ if (src0t == GGML_TYPE_IQ2_S && src1t == GGML_TYPE_F32 && extra0_iq2_s->noshuffle) {
+ ggml_cl_mul_mat_iq2_s_f32_adreno(backend, src0, src1, dst);
+ return;
+ }
+
+ // iq2_xs x fp32
+ if (src0t == GGML_TYPE_IQ2_XS && src1t == GGML_TYPE_F32 && extra0_iq2_xs->noshuffle) {
+ ggml_cl_mul_mat_iq2_xs_f32_adreno(backend, src0, src1, dst);
+ return;
+ }
+
+ // iq2_xxs x fp32
+ if (src0t == GGML_TYPE_IQ2_XXS && src1t == GGML_TYPE_F32 && extra0_iq2_xxs->noshuffle) {
+ ggml_cl_mul_mat_iq2_xxs_f32_adreno(backend, src0, src1, dst);
+ return;
+ }
+
// q4_k x fp32
if (src0t == GGML_TYPE_Q4_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q4_K(backend_ctx, src0)) {
ggml_cl_mul_mat_q4_k_f32_adreno(backend, src0, src1, dst);
@@ -26695,6 +28274,140 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
return;
}
+ case GGML_TYPE_IQ2_S: {
+ if (ne11 < 32) {
+ break;
+ }
+ if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {
+ break;
+ }
+
+ kernel = backend_ctx->kernel_mul_mm_iq2_s_f32_l4_lm;
+ nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128;
+ offset0 = src0->view_offs;
+
+ int batch_stride_a = ne00*ne01;
+ int batch_stride_b = ne10*ne11;
+ int batch_stride_d = ne0*ne1;
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq2_s->qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq2_s->sg));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq2_s->qh));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq2_s->sc));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_iq2_s->d));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne02));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne11));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne10));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_a));
+ CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_b));
+ CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &batch_stride_d));
+ CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r2));
+ CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r3));
+
+ size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13};
+ size_t local_work_size[] = {(size_t)nth0, 1, 1};
+
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+ return;
+ }
+ case GGML_TYPE_IQ2_XS: {
+ if (ne11 < 32) {
+ break;
+ }
+ if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {
+ break;
+ }
+
+ kernel = backend_ctx->kernel_mul_mm_iq2_xs_f32_l4_lm;
+ nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128;
+ offset0 = src0->view_offs;
+
+ int batch_stride_a = ne00*ne01;
+ int batch_stride_b = ne10*ne11;
+ int batch_stride_d = ne0*ne1;
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq2_xs->qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq2_xs->sc));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq2_xs->d));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d));
+ CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2));
+ CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3));
+
+ size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13};
+ size_t local_work_size[] = {(size_t)nth0, 1, 1};
+
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+ return;
+ }
+ case GGML_TYPE_IQ2_XXS: {
+ if (ne11 < 32) {
+ break;
+ }
+ if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {
+ break;
+ }
+
+ kernel = backend_ctx->kernel_mul_mm_iq2_xxs_f32_l4_lm;
+ nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128;
+ offset0 = src0->view_offs;
+
+ int batch_stride_a = ne00*ne01;
+ int batch_stride_b = ne10*ne11;
+ int batch_stride_d = ne0*ne1;
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq2_xxs->qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq2_xxs->sas));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq2_xxs->d));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne02));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne11));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne10));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_a));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_b));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_d));
+ CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2));
+ CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3));
+
+ size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13};
+ size_t local_work_size[] = {(size_t)nth0, 1, 1};
+
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+ return;
+ }
case GGML_TYPE_Q4_K: {
if (ne11 < 32) {
break;
@@ -27738,6 +29451,197 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
#endif // GGML_OPENCL_SOA_Q
break;
}
+ case GGML_TYPE_IQ2_S: {
+ if (backend_ctx->gpu_family == INTEL) {
+ nth0 = 16;
+ nth1 = 1;
+ ndst = 4;
+ } else if (backend_ctx->gpu_family == ADRENO) {
+ nth0 = 64;
+ nth1 = 1;
+ ndst = 4;
+ } else {
+ GGML_ASSERT(false && "TODO: Unknown GPU");
+ }
+
+#ifdef GGML_OPENCL_SOA_Q
+ kernel = backend_ctx->kernel_mul_mv_iq2_s_f32_flat;
+ offset0 = src0->view_offs;
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq2_s->qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq2_s->sg));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq2_s->qh));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq2_s->sc));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra0_iq2_s->d));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb01));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb02));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb03));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb11));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb12));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(cl_ulong), &nb13));
+ CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne0));
+ CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne1));
+ CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r2));
+ CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r3));
+#else
+ kernel = backend_ctx->kernel_mul_mv_iq2_s_f32;
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));
+#endif // GGML_OPENCL_SOA_Q
+ break;
+ }
+ case GGML_TYPE_IQ2_XS: {
+ if (backend_ctx->gpu_family == INTEL) {
+ nth0 = 16;
+ nth1 = 1;
+ ndst = 4;
+ } else if (backend_ctx->gpu_family == ADRENO) {
+ nth0 = 64;
+ nth1 = 1;
+ ndst = 4;
+ } else {
+ GGML_ASSERT(false && "TODO: Unknown GPU");
+ }
+
+#ifdef GGML_OPENCL_SOA_Q
+ kernel = backend_ctx->kernel_mul_mv_iq2_xs_f32_flat;
+ offset0 = src0->view_offs;
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq2_xs->qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq2_xs->sc));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq2_xs->d));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1));
+ CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2));
+ CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3));
+#else
+ kernel = backend_ctx->kernel_mul_mv_iq2_xs_f32;
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));
+#endif // GGML_OPENCL_SOA_Q
+ break;
+ }
+ case GGML_TYPE_IQ2_XXS: {
+ if (backend_ctx->gpu_family == INTEL) {
+ nth0 = 16;
+ nth1 = 1;
+ ndst = 4;
+ } else if (backend_ctx->gpu_family == ADRENO) {
+ nth0 = 64;
+ nth1 = 1;
+ ndst = 4;
+ } else {
+ GGML_ASSERT(false && "TODO: Unknown GPU");
+ }
+
+#ifdef GGML_OPENCL_SOA_Q
+ kernel = backend_ctx->kernel_mul_mv_iq2_xxs_f32_flat;
+ offset0 = src0->view_offs;
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq2_xxs->qs));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq2_xxs->sas));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq2_xxs->d));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb01));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb02));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb03));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne0));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne1));
+ CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &r2));
+ CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r3));
+#else
+ kernel = backend_ctx->kernel_mul_mv_iq2_xxs_f32;
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));
+#endif // GGML_OPENCL_SOA_Q
+ break;
+ }
case GGML_TYPE_Q4_K: {
#ifdef GGML_OPENCL_SOA_Q
kernel = backend_ctx->kernel_mul_mv_q4_K_f32_flat;
@@ -28045,7 +29949,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
src0t == GGML_TYPE_Q8_0 ||
src0t == GGML_TYPE_Q1_0 ||
src0t == GGML_TYPE_IQ4_NL ||
- src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_Q3_K) {
+ src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_Q3_K || src0t == GGML_TYPE_IQ2_S || src0t == GGML_TYPE_IQ2_XS || src0t == GGML_TYPE_IQ2_XXS) {
// Each SIMD group produces N_DST values in the result. Assuming each
// workgroup has N_SIMDGROUP SIMD groups, then each workgroup will
// produce N_DST*N_SIMDGROUP values in the result. Hence, the grid size
diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.cl
index fb21e8eba..94623df63 100644
--- a/ggml/src/ggml-opencl/kernels/cvt.cl
+++ b/ggml/src/ggml-opencl/kernels/cvt.cl
@@ -2926,3 +2926,132 @@ kernel void kernel_restore_block_q3_k_noshuffle(
b->hmask[ee] = (uchar)v;
}
}
+
+struct block_iq2_s {
+ half d;
+ uchar qs[QK_K/4];
+ uchar qh[QK_K/32];
+ uchar scales[QK_K/32];
+};
+
+kernel void kernel_convert_block_iq2_s(
+ global struct block_iq2_s * src0,
+ global uchar * dst_qs, // QK_K/8 per block
+ global uchar * dst_sg, // QK_K/8
+ global uchar * dst_qh, // QK_K/32
+ global uchar * dst_sc, // QK_K/32
+ global half * dst_d, // 1
+ ulong n_blk
+) {
+ const ulong i = get_global_id(0);
+ if (i >= n_blk) {
+ return;
+ }
+ global struct block_iq2_s * b = src0 + i;
+
+ dst_d[i] = b->d;
+ for (int j = 0; j < QK_K/8; ++j) { dst_qs[(QK_K/8) * i + j] = b->qs[j]; }
+ for (int j = 0; j < QK_K/8; ++j) { dst_sg[(QK_K/8) * i + j] = b->qs[QK_K/8 + j]; }
+ for (int j = 0; j < QK_K/32; ++j) { dst_qh[(QK_K/32) * i + j] = b->qh[j]; }
+ for (int j = 0; j < QK_K/32; ++j) { dst_sc[(QK_K/32) * i + j] = b->scales[j]; }
+}
+
+kernel void kernel_restore_block_iq2_s(
+ global uchar * src_qs,
+ global uchar * src_sg,
+ global uchar * src_qh,
+ global uchar * src_sc,
+ global half * src_d,
+ global struct block_iq2_s * dst,
+ ulong n_blk
+) {
+ const ulong i = get_global_id(0);
+ if (i >= n_blk) {
+ return;
+ }
+ global struct block_iq2_s * b = dst + i;
+
+ b->d = src_d[i];
+ for (int j = 0; j < QK_K/8; ++j) { b->qs[j] = src_qs[(QK_K/8) * i + j]; }
+ for (int j = 0; j < QK_K/8; ++j) { b->qs[QK_K/8 + j] = src_sg[(QK_K/8) * i + j]; }
+ for (int j = 0; j < QK_K/32; ++j) { b->qh[j] = src_qh[(QK_K/32) * i + j]; }
+ for (int j = 0; j < QK_K/32; ++j) { b->scales[j] = src_sc[(QK_K/32) * i + j]; }
+}
+
+struct block_iq2_xs {
+ half d;
+ ushort qs[QK_K/8];
+ uchar scales[QK_K/32];
+};
+
+kernel void kernel_convert_block_iq2_xs(
+ global struct block_iq2_xs * src0,
+ global ushort * dst_qs,
+ global uchar * dst_sc,
+ global half * dst_d,
+ ulong n_blocks
+) {
+ const ulong i = get_global_id(0);
+ if (i >= n_blocks) { return; }
+ dst_d[i] = src0[i].d;
+ for (int j = 0; j < 32; ++j) { dst_qs[32*i + j] = src0[i].qs[j]; }
+ for (int j = 0; j < 8; ++j) { dst_sc[8*i + j] = src0[i].scales[j]; }
+}
+
+kernel void kernel_restore_block_iq2_xs(
+ global ushort * src_qs,
+ global uchar * src_sc,
+ global half * src_d,
+ global struct block_iq2_xs * dst,
+ ulong n_blocks
+) {
+ const ulong i = get_global_id(0);
+ if (i >= n_blocks) { return; }
+ dst[i].d = src_d[i];
+ for (int j = 0; j < 32; ++j) { dst[i].qs[j] = src_qs[32*i + j]; }
+ for (int j = 0; j < 8; ++j) { dst[i].scales[j] = src_sc[8*i + j]; }
+}
+
+struct block_iq2_xxs {
+ half d;
+ ushort qs[QK_K/8];
+};
+
+kernel void kernel_convert_block_iq2_xxs(
+ global struct block_iq2_xxs * src0,
+ global uchar * dst_qs,
+ global uint * dst_sas,
+ global half * dst_d,
+ ulong n_blocks
+) {
+ const ulong i = get_global_id(0);
+ if (i >= n_blocks) { return; }
+ dst_d[i] = src0[i].d;
+ for (int j = 0; j < 8; ++j) {
+ global ushort * q = src0[i].qs + 4*j;
+ uint indices = (uint)q[0] | ((uint)q[1] << 16);
+ vstore4(as_uchar4(indices), 8*i + j, dst_qs);
+ dst_sas[8*i + j] = (uint)q[2] | ((uint)q[3] << 16);
+ }
+}
+
+kernel void kernel_restore_block_iq2_xxs(
+ global uchar * src_qs,
+ global uint * src_sas,
+ global half * src_d,
+ global struct block_iq2_xxs * dst,
+ ulong n_blocks
+) {
+ const ulong i = get_global_id(0);
+ if (i >= n_blocks) { return; }
+ dst[i].d = src_d[i];
+ for (int j = 0; j < 8; ++j) {
+ global ushort * q = dst[i].qs + 4*j;
+ uint indices = as_uint(vload4(8*i + j, src_qs));
+ uint sas = src_sas[8*i + j];
+ q[0] = (ushort)indices;
+ q[1] = (ushort)(indices >> 16);
+ q[2] = (ushort)sas;
+ q[3] = (ushort)(sas >> 16);
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_s_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_s_f32.cl
new file mode 100644
index 000000000..757a5ebd1
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_s_f32.cl
@@ -0,0 +1,93 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_qcom_reqd_sub_group_size
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+__attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+kernel void kernel_gemm_noshuffle_iq2_s_f32(
+ global const uchar * src0_qs,
+ global const uchar * src0_sg,
+ global const uchar * src0_qh,
+ global const uchar * src0_sc,
+ global const half * src0_d,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int m,
+ int n,
+ int k,
+ int n_no_padding
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+ const int gy = get_global_id(0);
+ const int row = get_global_id(1) * 4;
+ const int n_4 = n / 4;
+ if (row >= m) {
+ return;
+ }
+
+ half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
+ for (int i = 0; i < k; i += 16) {
+ const uchar4 sc = vload4(0, src0_sc + row + (i / 32) * m);
+ const half4 d = vload4(0, src0_d + row + (i / 256) * m);
+ const float4 scale_f = convert_float4(d) * 0.25f * (0.5f + convert_float4((sc >> (uchar4)(4*((i / 16) & 1))) & (uchar4)(15)));
+ const half4 scale = convert_half4(scale_f);
+
+ for (int l = 0; l < 16; l += 4) {
+ const int ki = i + l;
+ const uint4 qs = convert_uint4(vload4(0, src0_qs + row + (ki / 8) * m));
+ const uint4 qh = convert_uint4(vload4(0, src0_qh + row + (ki / 32) * m));
+ const uchar4 sg = vload4(0, src0_sg + row + (ki / 8) * m);
+ const uint4 gi = qs | (((qh >> (uint4)(2*((ki / 8) & 3))) & (uint4)(3)) << (uint4)(8));
+ const uint half_grid = (ki & 7) / 4;
+ const uint4 grid = (uint4)(iq2_s_grid[2*gi.s0 + half_grid], iq2_s_grid[2*gi.s1 + half_grid],
+ iq2_s_grid[2*gi.s2 + half_grid], iq2_s_grid[2*gi.s3 + half_grid]);
+ #pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ half8 b;
+ b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4);
+ b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
+ const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255));
+ const int4 signs = convert_int4((sg >> (uchar4)((ki & 7) + j)) & (uchar4)(1));
+ const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale;
+ c0 += b * w.s0;
+ c1 += b * w.s1;
+ c2 += b * w.s2;
+ c3 += b * w.s3;
+ }
+ }
+ }
+
+ int idx = gy * 8 * m + row;
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_s_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_s_q8_1_dp4a.cl
new file mode 100644
index 000000000..a8f1bf9d0
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_s_q8_1_dp4a.cl
@@ -0,0 +1,149 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#define QK_K 256
+
+#ifndef TILESIZE_N
+#define TILESIZE_N 32
+#endif
+
+inline uint iq2_s_pack(uint gv, uint sg, uint base) {
+ int v0 = (int)((gv >> 0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; }
+ int v1 = (int)((gv >> 8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; }
+ int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; }
+ int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; }
+ return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8)
+ | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24);
+}
+
+// Pass staged operands by value; local dp4a operands miscompile on X2.
+inline int dot4_q8a_v(uint4 qw, uint4 a) {
+ int r = 0;
+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r);
+ return r;
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_iq2_s_q8_1_dp4a(
+ __global const uchar * src0_qs,
+ __global const uchar * src0_sg,
+ __global const uchar * src0_qh,
+ __global const uchar * src0_sc,
+ __global const half * src0_d,
+ __global const uint * src1_qa,
+ __global const half * src1_da,
+ __global float * dst,
+ ulong offsetd,
+ int m,
+ int n_no_padding,
+ int k
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0);
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0;
+
+ const uint k_u = (uint)k >> 2;
+ const uint k_b = (uint)k >> 5;
+
+ __local uint4 sh_qa4[TILESIZE_N][2];
+ __local half sh_d[TILESIZE_N];
+
+#define IQ2_S_GRID(i) iq2_s_grid[(i)]
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+ const uint ib = sub >> 3;
+
+ const uint qhv = (uint)src0_qh[rrow + sub * (uint)m];
+ const uint scv = (uint)src0_sc[rrow + sub * (uint)m];
+ const float dv = (float)src0_d[rrow + ib * (uint)m];
+ const float dl0 = dv * (0.5f + (float)( scv & 0xFu)) * 0.25f;
+ const float dl1 = dv * (0.5f + (float)( scv >> 4) ) * 0.25f;
+
+ const uint gb = rrow + (step >> 3) * (uint)m;
+
+ uint4 qlo, qhi;
+ {
+ const uint gi0 = (uint)src0_qs[gb + 0u * (uint)m] | ((( qhv & 3u) << 8));
+ const uint gi1 = (uint)src0_qs[gb + 1u * (uint)m] | ((((qhv >> 2) & 3u) << 8));
+ const uint gi2 = (uint)src0_qs[gb + 2u * (uint)m] | ((((qhv >> 4) & 3u) << 8));
+ const uint gi3 = (uint)src0_qs[gb + 3u * (uint)m] | ((((qhv >> 6) & 3u) << 8));
+ const uint s0 = (uint)src0_sg[gb + 0u * (uint)m];
+ const uint s1 = (uint)src0_sg[gb + 1u * (uint)m];
+ const uint s2 = (uint)src0_sg[gb + 2u * (uint)m];
+ const uint s3 = (uint)src0_sg[gb + 3u * (uint)m];
+ qlo.s0 = iq2_s_pack(IQ2_S_GRID(2u*gi0 + 0u), s0, 0u);
+ qlo.s1 = iq2_s_pack(IQ2_S_GRID(2u*gi0 + 1u), s0, 4u);
+ qlo.s2 = iq2_s_pack(IQ2_S_GRID(2u*gi1 + 0u), s1, 0u);
+ qlo.s3 = iq2_s_pack(IQ2_S_GRID(2u*gi1 + 1u), s1, 4u);
+ qhi.s0 = iq2_s_pack(IQ2_S_GRID(2u*gi2 + 0u), s2, 0u);
+ qhi.s1 = iq2_s_pack(IQ2_S_GRID(2u*gi2 + 1u), s2, 4u);
+ qhi.s2 = iq2_s_pack(IQ2_S_GRID(2u*gi3 + 0u), s3, 0u);
+ qhi.s3 = iq2_s_pack(IQ2_S_GRID(2u*gi3 + 1u), s3, 4u);
+ }
+
+ for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
+ const uint t = idx >> 1;
+ const uint v = idx & 1;
+ const uint c = col_base + t;
+ sh_qa4[t][v] = (c < (uint)n_no_padding)
+ ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
+ : (uint4)(0u);
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+#define IQ2_S_COL(b) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) \
+ + dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1])))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+ rf.s0 = IQ2_S_COL(b+0); rf.s1 = IQ2_S_COL(b+1);
+ rf.s2 = IQ2_S_COL(b+2); rf.s3 = IQ2_S_COL(b+3);
+ acc[g] += LD4(sh_d, b) * rf;
+ }
+#undef IQ2_S_COL
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+#undef NGROUPS
+#undef IQ2_S_GRID
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xs_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xs_f32.cl
new file mode 100644
index 000000000..f4a075063
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xs_f32.cl
@@ -0,0 +1,91 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_qcom_reqd_sub_group_size
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+__attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+kernel void kernel_gemm_noshuffle_iq2_xs_f32(
+ global const ushort * src0_qs,
+ global const uchar * src0_sc,
+ global const half * src0_d,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int m,
+ int n,
+ int k,
+ int n_no_padding
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+ const int gy = get_global_id(0);
+ const int row = get_global_id(1) * 4;
+ const int n_4 = n / 4;
+ if (row >= m) {
+ return;
+ }
+
+ half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
+ for (int i = 0; i < k; i += 16) {
+ const uchar4 sc = vload4(0, src0_sc + row + (i / 32) * m);
+ const half4 d = vload4(0, src0_d + row + (i / 256) * m);
+ const float4 scale_f = convert_float4(d) * 0.25f * (0.5f + convert_float4((sc >> (uchar4)(4*((i / 16) & 1))) & (uchar4)(15)));
+ const half4 scale = convert_half4(scale_f);
+
+ for (int l = 0; l < 16; l += 4) {
+ const int ki = i + l;
+ const uint4 qs = convert_uint4(vload4(0, src0_qs + row + (ki / 8) * m));
+ const uint4 code = qs >> (uint4)(9);
+ const uint4 sg = code | ((popcount(code) & (uint4)(1)) << (uint4)(7));
+ const uint4 gi = qs & (uint4)(511);
+ const uint half_grid = (ki & 7) / 4;
+ const uint4 grid = (uint4)(iq2_xs_grid[2*gi.s0 + half_grid], iq2_xs_grid[2*gi.s1 + half_grid],
+ iq2_xs_grid[2*gi.s2 + half_grid], iq2_xs_grid[2*gi.s3 + half_grid]);
+ #pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ half8 b;
+ b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4);
+ b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
+ const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255));
+ const int4 signs = convert_int4((sg >> (uint4)((ki & 7) + j)) & (uint4)(1));
+ const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale;
+ c0 += b * w.s0;
+ c1 += b * w.s1;
+ c2 += b * w.s2;
+ c3 += b * w.s3;
+ }
+ }
+ }
+
+ int idx = gy * 8 * m + row;
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xs_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xs_q8_1_dp4a.cl
new file mode 100644
index 000000000..c322803b9
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xs_q8_1_dp4a.cl
@@ -0,0 +1,150 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#define QK_K 256
+
+#ifndef TILESIZE_N
+#define TILESIZE_N 32
+#endif
+
+inline uint iq2_xs_pack(uint gv, uint sg, uint base) {
+ int v0 = (int)((gv >> 0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; }
+ int v1 = (int)((gv >> 8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; }
+ int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; }
+ int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; }
+ return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8)
+ | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24);
+}
+
+// Pass staged operands by value; local dp4a operands miscompile on X2.
+inline int dot4_q8a_v(uint4 qw, uint4 a) {
+ int r = 0;
+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r);
+ return r;
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_iq2_xs_q8_1_dp4a(
+ __global const ushort * src0_qs,
+ __global const uchar * src0_sc,
+ __global const half * src0_d,
+ __global const uint * src1_qa,
+ __global const half * src1_da,
+ __global float * dst,
+ ulong offsetd,
+ int m,
+ int n_no_padding,
+ int k
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0);
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0;
+
+ const uint k_u = (uint)k >> 2;
+ const uint k_b = (uint)k >> 5;
+
+ __local uint4 sh_qa4[TILESIZE_N][2];
+ __local half sh_d[TILESIZE_N];
+
+#define IQ2_XS_GRID(i) iq2_xs_grid[(i)]
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+ const uint ib = sub >> 3;
+
+ const uint scv = (uint)src0_sc[rrow + sub * (uint)m];
+ const float dv = (float)src0_d[rrow + ib * (uint)m];
+ const float dl0 = dv * (0.5f + (float)( scv & 0xFu)) * 0.25f;
+ const float dl1 = dv * (0.5f + (float)( scv >> 4) ) * 0.25f;
+
+ const uint gb = rrow + (step >> 3) * (uint)m;
+
+ uint4 qlo, qhi;
+ {
+ const uint q0 = (uint)src0_qs[gb + 0u * (uint)m];
+ const uint q1 = (uint)src0_qs[gb + 1u * (uint)m];
+ const uint q2 = (uint)src0_qs[gb + 2u * (uint)m];
+ const uint q3 = (uint)src0_qs[gb + 3u * (uint)m];
+ const uint gi0 = (q0 & 511u) << 1;
+ const uint gi1 = (q1 & 511u) << 1;
+ const uint gi2 = (q2 & 511u) << 1;
+ const uint gi3 = (q3 & 511u) << 1;
+ const uint s0 = iq2_xs_signs(q0 >> 9);
+ const uint s1 = iq2_xs_signs(q1 >> 9);
+ const uint s2 = iq2_xs_signs(q2 >> 9);
+ const uint s3 = iq2_xs_signs(q3 >> 9);
+ qlo.s0 = iq2_xs_pack(IQ2_XS_GRID(gi0 + 0u), s0, 0u);
+ qlo.s1 = iq2_xs_pack(IQ2_XS_GRID(gi0 + 1u), s0, 4u);
+ qlo.s2 = iq2_xs_pack(IQ2_XS_GRID(gi1 + 0u), s1, 0u);
+ qlo.s3 = iq2_xs_pack(IQ2_XS_GRID(gi1 + 1u), s1, 4u);
+ qhi.s0 = iq2_xs_pack(IQ2_XS_GRID(gi2 + 0u), s2, 0u);
+ qhi.s1 = iq2_xs_pack(IQ2_XS_GRID(gi2 + 1u), s2, 4u);
+ qhi.s2 = iq2_xs_pack(IQ2_XS_GRID(gi3 + 0u), s3, 0u);
+ qhi.s3 = iq2_xs_pack(IQ2_XS_GRID(gi3 + 1u), s3, 4u);
+ }
+
+ for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
+ const uint t = idx >> 1;
+ const uint v = idx & 1;
+ const uint c = col_base + t;
+ sh_qa4[t][v] = (c < (uint)n_no_padding)
+ ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
+ : (uint4)(0u);
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+#define IQ2_XS_COL(b) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) \
+ + dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1])))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+ rf.s0 = IQ2_XS_COL(b+0); rf.s1 = IQ2_XS_COL(b+1);
+ rf.s2 = IQ2_XS_COL(b+2); rf.s3 = IQ2_XS_COL(b+3);
+ acc[g] += LD4(sh_d, b) * rf;
+ }
+#undef IQ2_XS_COL
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+#undef NGROUPS
+#undef IQ2_XS_GRID
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xxs_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xxs_f32.cl
new file mode 100644
index 000000000..5d6cc0dea
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xxs_f32.cl
@@ -0,0 +1,90 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_qcom_reqd_sub_group_size
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+__attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+kernel void kernel_gemm_noshuffle_iq2_xxs_f32(
+ global const uchar * src0_qs,
+ global const uint * src0_sas,
+ global const half * src0_d,
+ read_only image1d_buffer_t src1,
+ global float * dst,
+ ulong offsetd,
+ int m,
+ int n,
+ int k,
+ int n_no_padding
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+ const int gy = get_global_id(0);
+ const int row = get_global_id(1) * 4;
+ const int n_4 = n / 4;
+ if (row >= m) {
+ return;
+ }
+
+ half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
+ for (int i = 0; i < k; i += 16) {
+ const uint4 sas = vload4(0, src0_sas + row + (i / 32) * m);
+ const half4 d = vload4(0, src0_d + row + (i / 256) * m);
+ const float4 scale_f = convert_float4(d) * 0.25f * (0.5f + convert_float4(sas >> (uint4)(28)));
+ const half4 scale = convert_half4(scale_f);
+
+ for (int l = 0; l < 16; l += 4) {
+ const int ki = i + l;
+ const uint4 gi = convert_uint4(vload4(0, src0_qs + row + (ki / 8) * m));
+ const uint4 code = (sas >> (uint4)(7*((ki / 8) & 3))) & (uint4)(127);
+ const uint4 sg = code | ((popcount(code) & (uint4)(1)) << (uint4)(7));
+ const uint half_grid = (ki & 7) / 4;
+ const uint4 grid = (uint4)(iq2_xxs_grid[2*gi.s0 + half_grid], iq2_xxs_grid[2*gi.s1 + half_grid],
+ iq2_xxs_grid[2*gi.s2 + half_grid], iq2_xxs_grid[2*gi.s3 + half_grid]);
+ #pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ half8 b;
+ b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4);
+ b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
+ const int4 magnitude = convert_int4((grid >> (uint4)(8*j)) & (uint4)(255));
+ const int4 signs = convert_int4((sg >> (uint4)((ki & 7) + j)) & (uint4)(1));
+ const half4 w = convert_half4(magnitude * ((int4)(1) - 2*signs)) * scale;
+ c0 += b * w.s0;
+ c1 += b * w.s1;
+ c2 += b * w.s2;
+ c3 += b * w.s3;
+ }
+ }
+ }
+
+ int idx = gy * 8 * m + row;
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
+ idx += m;
+ }
+ if (idx + 3 < m * n_no_padding) {
+ vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xxs_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xxs_q8_1_dp4a.cl
new file mode 100644
index 000000000..60cd268f3
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq2_xxs_q8_1_dp4a.cl
@@ -0,0 +1,145 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#ifdef cl_khr_integer_dot_product
+#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
+#endif
+
+#define QK_K 256
+
+#ifndef TILESIZE_N
+#define TILESIZE_N 32
+#endif
+
+inline uint iq2_xxs_pack(uint gv, uint sg, uint base) {
+ int v0 = (int)((gv >> 0) & 0xFF); if (sg & (1u << (base + 0))) { v0 = -v0; }
+ int v1 = (int)((gv >> 8) & 0xFF); if (sg & (1u << (base + 1))) { v1 = -v1; }
+ int v2 = (int)((gv >> 16) & 0xFF); if (sg & (1u << (base + 2))) { v2 = -v2; }
+ int v3 = (int)((gv >> 24) & 0xFF); if (sg & (1u << (base + 3))) { v3 = -v3; }
+ return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8)
+ | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24);
+}
+
+// Pass staged operands by value; local dp4a operands miscompile on X2.
+inline int dot4_q8a_v(uint4 qw, uint4 a) {
+ int r = 0;
+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r);
+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r);
+ return r;
+}
+
+__attribute__((qcom_wave_pair_mode(1)))
+kernel void kernel_gemm_noshuffle_iq2_xxs_q8_1_dp4a(
+ __global const uchar * src0_qs,
+ __global const uint * src0_sas,
+ __global const half * src0_d,
+ __global const uint * src1_qa,
+ __global const half * src1_da,
+ __global float * dst,
+ ulong offsetd,
+ int m,
+ int n_no_padding,
+ int k
+) {
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint lid = get_local_id(0);
+ const uint block_id_m = get_global_id(1);
+ const uint block_id_n = get_global_id(2);
+
+ const uint row = block_id_m * 64 + lid;
+ const uint col_base = block_id_n * TILESIZE_N;
+ const bool row_valid = row < (uint)m;
+ const uint rrow = row_valid ? row : 0;
+
+ const uint k_u = (uint)k >> 2;
+ const uint k_b = (uint)k >> 5;
+
+ __local uint4 sh_qa4[TILESIZE_N][2];
+ __local half sh_d[TILESIZE_N];
+
+#define IQ2_XXS_GRID(i) iq2_xxs_grid[(i)]
+
+#define NGROUPS (TILESIZE_N / 4)
+ float4 acc[NGROUPS];
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
+
+ for (uint step = 0; step < (uint)k; step += 32) {
+ const uint sub = step >> 5;
+ const uint ib = sub >> 3;
+
+ const uint aux = src0_sas[rrow + sub * (uint)m];
+ const float dv = (float)src0_d[rrow + ib * (uint)m];
+ const float dl = dv * (0.5f + (float)(aux >> 28)) * 0.25f;
+
+ const uint gb = rrow + (step >> 3) * (uint)m;
+
+ uint4 qlo, qhi;
+ {
+ const uint gi0 = (uint)src0_qs[gb + 0u * (uint)m] << 1;
+ const uint gi1 = (uint)src0_qs[gb + 1u * (uint)m] << 1;
+ const uint gi2 = (uint)src0_qs[gb + 2u * (uint)m] << 1;
+ const uint gi3 = (uint)src0_qs[gb + 3u * (uint)m] << 1;
+ const uint s0 = iq2_xxs_signs((aux >> 0) & 127u);
+ const uint s1 = iq2_xxs_signs((aux >> 7) & 127u);
+ const uint s2 = iq2_xxs_signs((aux >> 14) & 127u);
+ const uint s3 = iq2_xxs_signs((aux >> 21) & 127u);
+ qlo.s0 = iq2_xxs_pack(IQ2_XXS_GRID(gi0 + 0u), s0, 0u);
+ qlo.s1 = iq2_xxs_pack(IQ2_XXS_GRID(gi0 + 1u), s0, 4u);
+ qlo.s2 = iq2_xxs_pack(IQ2_XXS_GRID(gi1 + 0u), s1, 0u);
+ qlo.s3 = iq2_xxs_pack(IQ2_XXS_GRID(gi1 + 1u), s1, 4u);
+ qhi.s0 = iq2_xxs_pack(IQ2_XXS_GRID(gi2 + 0u), s2, 0u);
+ qhi.s1 = iq2_xxs_pack(IQ2_XXS_GRID(gi2 + 1u), s2, 4u);
+ qhi.s2 = iq2_xxs_pack(IQ2_XXS_GRID(gi3 + 0u), s3, 0u);
+ qhi.s3 = iq2_xxs_pack(IQ2_XXS_GRID(gi3 + 1u), s3, 4u);
+ }
+
+ for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
+ const uint t = idx >> 1;
+ const uint v = idx & 1;
+ const uint c = col_base + t;
+ sh_qa4[t][v] = (c < (uint)n_no_padding)
+ ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
+ : (uint4)(0u);
+ }
+ if (lid < TILESIZE_N) {
+ const uint c = col_base + lid;
+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
+#define IQ2_XXS_COL(b) (dl * (float)(dot4_q8a_v(qlo, (uint4)(sh_qa4[b][0])) \
+ + dot4_q8a_v(qhi, (uint4)(sh_qa4[b][1]))))
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const int b = g * 4;
+ float4 rf;
+ rf.s0 = IQ2_XXS_COL(b+0); rf.s1 = IQ2_XXS_COL(b+1);
+ rf.s2 = IQ2_XXS_COL(b+2); rf.s3 = IQ2_XXS_COL(b+3);
+ acc[g] += LD4(sh_d, b) * rf;
+ }
+#undef IQ2_XXS_COL
+#undef LD4
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ if (!row_valid) {
+ return;
+ }
+
+ #pragma unroll
+ for (int g = 0; g < NGROUPS; ++g) {
+ const uint b = (uint)(g * 4);
+ const float4 a = acc[g];
+ const uint c0 = col_base + b;
+ if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
+ if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
+ if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
+ if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
+ }
+#undef NGROUPS
+#undef IQ2_XXS_GRID
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_s_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_s_f32.cl
new file mode 100644
index 000000000..25de1cb91
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_s_f32.cl
@@ -0,0 +1,122 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#define QK_K 256
+
+#ifndef IQ2_S_MV_NSG
+#define IQ2_S_MV_NSG 8
+#endif
+
+inline float4 iq2_s_vals(uint gv, uint sg, uint base) {
+
+ const uint s = sg >> base;
+ const uint4 sgn = (uint4)(s << 31, s << 30, s << 29, s << 28) & 0x80000000u;
+ return as_float4(as_uint4(convert_float4(as_uchar4(gv))) ^ sgn);
+}
+
+kernel void kernel_iq2_s_grid_export(global uint * out) {
+ out[get_global_id(0)] = iq2_s_grid[get_global_id(0)];
+}
+
+kernel void kernel_gemv_noshuffle_iq2_s_f32(
+ read_only image1d_buffer_t grid_img,
+ global const uchar * src0_qs,
+ global const uchar * src0_sg,
+ global const uchar * src0_qh,
+ global const uchar * src0_sc,
+ global const half * src0_d,
+ global const float * src1,
+ ulong offset1,
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ int ne10,
+ int ne0
+) {
+ src1 = (global const float *)((global const char *)src1 + offset1);
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint m = (uint)ne01;
+ const uint K = (uint)ne00;
+ const uint nsb = K / QK_K;
+
+ const uint lid = get_local_id(0);
+ const uint sgi = get_local_id(1);
+ const uint col = get_group_id(1);
+
+ global const float * y = src1 + (ulong)col * (uint)ne10;
+
+#define IQ2_S_GRID(i) read_imageui(grid_img, (int)(i)).x
+
+ const uint mh = m >> 1;
+ const uint j = get_group_id(0) * 64u + lid;
+ const uint row = j << 1;
+
+ float sumf = 0.f, sumf1 = 0.f;
+
+ if (j < mh) {
+ global const ushort * qsu = (global const ushort *)src0_qs;
+ global const ushort * sgu = (global const ushort *)src0_sg;
+ global const ushort * qhu = (global const ushort *)src0_qh;
+ global const ushort * scu = (global const ushort *)src0_sc;
+
+ for (uint ib = sgi; ib < nsb; ib += IQ2_S_MV_NSG) {
+ const half2 dh = vload2(j + ib * mh, src0_d);
+
+ float acc0 = 0.f, acc1 = 0.f;
+ for (uint sb = 0; sb < 8u; ++sb) {
+ const uint sub = ib * 8u + sb;
+ const uint qhv = (uint)qhu[j + sub * mh];
+ const uint scv = (uint)scu[j + sub * mh];
+ const uint gb = j + (sub * 4u) * mh;
+
+ for (uint h = 0; h < 2u; ++h) {
+ const uint n0 = (h == 0u) ? ( scv & 0xFu) : (( scv >> 4) & 0xFu);
+ const uint n1 = (h == 0u) ? ((scv >> 8) & 0xFu) : (((scv >> 8) >> 4) & 0xFu);
+
+ float a0 = 0.f, a1 = 0.f;
+ for (uint t = 0; t < 2u; ++t) {
+ const uint l = 2u*h + t;
+ const uint qsv = (uint)qsu[gb + l * mh];
+ const uint sgv = (uint)sgu[gb + l * mh];
+ const uint gi0 = ( qsv & 0xFFu) | ((((qhv ) >> (2u*l)) & 3u) << 8);
+ const uint gi1 = ((qsv >> 8) & 0xFFu) | ((((qhv >> 8) >> (2u*l)) & 3u) << 8);
+ const uint s0 = sgv & 0xFFu;
+ const uint s1 = (sgv >> 8) & 0xFFu;
+
+ const uint grp = (ib * 64u + sb * 8u) + l * 2u;
+ const float4 y0 = vload4(grp + 0u, y);
+ const float4 y1 = vload4(grp + 1u, y);
+ a0 += dot(y0, iq2_s_vals(IQ2_S_GRID(2u*gi0 + 0u), s0, 0u));
+ a0 += dot(y1, iq2_s_vals(IQ2_S_GRID(2u*gi0 + 1u), s0, 4u));
+ a1 += dot(y0, iq2_s_vals(IQ2_S_GRID(2u*gi1 + 0u), s1, 0u));
+ a1 += dot(y1, iq2_s_vals(IQ2_S_GRID(2u*gi1 + 1u), s1, 4u));
+ }
+ acc0 += (0.5f + (float)n0) * a0;
+ acc1 += (0.5f + (float)n1) * a1;
+ }
+ }
+ sumf += (float)dh.s0 * 0.25f * acc0;
+ sumf1 += (float)dh.s1 * 0.25f * acc1;
+ }
+ }
+
+#if IQ2_S_MV_NSG > 1
+ __local float2 part[IQ2_S_MV_NSG][64];
+ part[sgi][lid] = (float2)(sumf, sumf1);
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (sgi != 0) {
+ return;
+ }
+ for (uint s = 1; s < IQ2_S_MV_NSG; ++s) {
+ const float2 p = part[s][lid];
+ sumf += p.s0;
+ sumf1 += p.s1;
+ }
+#endif
+
+ if (j < mh) {
+ vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
+ }
+#undef IQ2_S_GRID
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_xs_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_xs_f32.cl
new file mode 100644
index 000000000..52d626ec8
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_xs_f32.cl
@@ -0,0 +1,128 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#define QK_K 256
+
+#ifndef IQ2_XS_MV_NSG
+#define IQ2_XS_MV_NSG 4
+#endif
+
+inline float4 iq2_xs_vals(uint gv, uint sgv, uint base) {
+ const uint s = sgv >> base;
+ const uint4 sgn = (uint4)(s << 31, s << 30, s << 29, s << 28) & 0x80000000u;
+ return as_float4(as_uint4(convert_float4(as_uchar4(gv))) ^ sgn);
+}
+
+kernel void kernel_gemv_noshuffle_iq2_xs_f32(
+ __read_only image1d_buffer_t grid_img,
+ global const ushort * src0_qs,
+ global const uchar * src0_sc,
+ global const half * src0_d,
+ global const float * src1,
+ ulong offset1,
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ int ne10,
+ int ne0
+) {
+ src1 = (global const float *)((global const char *)src1 + offset1);
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint m = (uint)ne01;
+ const uint K = (uint)ne00;
+ const uint nsb = K / QK_K;
+
+ const uint lid = get_local_id(0);
+ const uint sgi = get_local_id(1);
+ const uint col = get_group_id(1);
+
+ global const float * y = src1 + (ulong)col * (uint)ne10;
+
+#define IQ2_XS_GRID(i) (read_imageui(grid_img, (int)(i)).x)
+
+ const uint mh = m >> 1;
+ const uint j = get_group_id(0) * 64u + lid;
+ const uint row = j << 1;
+
+ float sumf = 0.f;
+ float sumf1 = 0.f;
+
+ if (j < mh) {
+ global const uint * qsu = (global const uint *)src0_qs;
+ global const ushort * scu = (global const ushort *)src0_sc;
+
+ for (uint ib = sgi; ib < nsb; ib += IQ2_XS_MV_NSG) {
+ const half2 dh = vload2(j + ib * mh, src0_d);
+ const float d0 = (float)dh.s0;
+ const float d1 = (float)dh.s1;
+
+ float acc0 = 0.f, acc1 = 0.f;
+ for (uint sb = 0; sb < 8u; ++sb) {
+ const uint sub = ib * 8u + sb;
+ const uint scv = (uint)scu[j + sub * mh];
+
+ const float lo0 = 0.5f + (float)( scv & 0xFu);
+ const float hi0 = 0.5f + (float)((scv >> 4) & 0xFu);
+ const float lo1 = 0.5f + (float)((scv >> 8) & 0xFu);
+ const float hi1 = 0.5f + (float)((scv >> 12) & 0xFu);
+
+ const uint grp = sub * 4u;
+ const uint qsb = j + grp * mh;
+
+ float alo0 = 0.f, ahi0 = 0.f, alo1 = 0.f, ahi1 = 0.f;
+ for (uint u = 0; u < 4u; ++u) {
+ const uint qsv = qsu[qsb + u * mh];
+ const uint q0 = qsv & 0xFFFFu;
+ const uint q1 = qsv >> 16;
+
+ const uint g0 = (q0 & 511u) << 1;
+ const uint g1 = (q1 & 511u) << 1;
+ const uint sg0 = iq2_xs_signs(q0 >> 9);
+ const uint sg1 = iq2_xs_signs(q1 >> 9);
+
+ const float4 yl = vload4(sub * 8u + u * 2u + 0u, y);
+ const float4 yh = vload4(sub * 8u + u * 2u + 1u, y);
+
+ const float a0 = dot(yl, iq2_xs_vals(IQ2_XS_GRID(g0 ), sg0, 0u))
+ + dot(yh, iq2_xs_vals(IQ2_XS_GRID(g0 + 1), sg0, 4u));
+ const float a1 = dot(yl, iq2_xs_vals(IQ2_XS_GRID(g1 ), sg1, 0u))
+ + dot(yh, iq2_xs_vals(IQ2_XS_GRID(g1 + 1), sg1, 4u));
+
+ if (u < 2u) { alo0 += a0; alo1 += a1; }
+ else { ahi0 += a0; ahi1 += a1; }
+ }
+ acc0 += lo0 * alo0 + hi0 * ahi0;
+ acc1 += lo1 * alo1 + hi1 * ahi1;
+ }
+ sumf += d0 * 0.25f * acc0;
+ sumf1 += d1 * 0.25f * acc1;
+ }
+ }
+
+#if IQ2_XS_MV_NSG > 1
+ __local float2 part[IQ2_XS_MV_NSG][64];
+ part[sgi][lid] = (float2)(sumf, sumf1);
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (sgi != 0) {
+ return;
+ }
+ for (uint s = 1; s < IQ2_XS_MV_NSG; ++s) {
+ const float2 p = part[s][lid];
+ sumf += p.s0;
+ sumf1 += p.s1;
+ }
+#endif
+
+ if (j < mh) {
+ vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
+ }
+#undef IQ2_XS_GRID
+}
+
+kernel void kernel_iq2_xs_grid_export(global uint * out) {
+ const uint i = get_global_id(0);
+ if (i < 1024u) {
+ out[i] = iq2_xs_grid[i];
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_xxs_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_xxs_f32.cl
new file mode 100644
index 000000000..405a8144f
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq2_xxs_f32.cl
@@ -0,0 +1,117 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#define QK_K 256
+
+#ifndef IQ2_XXS_MV_NSG
+#define IQ2_XXS_MV_NSG 4
+#endif
+
+inline float4 iq2_xxs_vals(uint gv, uint sgv, uint base) {
+ const uint s = sgv >> base;
+ const uint4 sgn = (uint4)(s << 31, s << 30, s << 29, s << 28) & 0x80000000u;
+ return as_float4(as_uint4(convert_float4(as_uchar4(gv))) ^ sgn);
+}
+
+kernel void kernel_gemv_noshuffle_iq2_xxs_f32(
+ __read_only image1d_buffer_t grid_img,
+ global const uchar * src0_qs,
+ global const uint * src0_sas,
+ global const half * src0_d,
+ global const float * src1,
+ ulong offset1,
+ global float * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ int ne10,
+ int ne0
+) {
+ src1 = (global const float *)((global const char *)src1 + offset1);
+ dst = (global float *)((global char *)dst + offsetd);
+
+ const uint m = (uint)ne01;
+ const uint K = (uint)ne00;
+ const uint nsb = K / QK_K;
+
+ const uint lid = get_local_id(0);
+ const uint sgi = get_local_id(1);
+ const uint col = get_group_id(1);
+
+ global const float * y = src1 + (ulong)col * (uint)ne10;
+
+#define IQ2_XXS_GRID(i) (read_imageui(grid_img, (int)(i)).x)
+
+ const uint mh = m >> 1;
+ const uint j = get_group_id(0) * 64u + lid;
+ const uint row = j << 1;
+
+ float sumf = 0.f;
+ float sumf1 = 0.f;
+
+ if (j < mh) {
+ global const ushort * qsu = (global const ushort *)src0_qs;
+
+ for (uint ib = sgi; ib < nsb; ib += IQ2_XXS_MV_NSG) {
+ const half2 dh = vload2(j + ib * mh, src0_d);
+ const float d0 = (float)dh.s0;
+ const float d1 = (float)dh.s1;
+
+ float acc0 = 0.f, acc1 = 0.f;
+ for (uint sb = 0; sb < 8u; ++sb) {
+ const uint sub = ib * 8u + sb;
+ const uint2 aux = vload2(j + sub * mh, src0_sas);
+
+ const uint grp = sub * 4u;
+ const uint qsb = j + grp * mh;
+
+ float a0 = 0.f, a1 = 0.f;
+ for (uint u = 0; u < 4u; ++u) {
+ const uint qsv = (uint)qsu[qsb + u * mh];
+ const uint sg0 = iq2_xxs_signs((aux.s0 >> (7u * u)) & 127u);
+ const uint sg1 = iq2_xxs_signs((aux.s1 >> (7u * u)) & 127u);
+
+ const uint g0 = ( qsv & 0xFFu) << 1;
+ const uint g1 = ((qsv >> 8) & 0xFFu) << 1;
+
+ const float4 yl = vload4(sub * 8u + u * 2u + 0u, y);
+ const float4 yh = vload4(sub * 8u + u * 2u + 1u, y);
+
+ a0 += dot(yl, iq2_xxs_vals(IQ2_XXS_GRID(g0 ), sg0, 0u))
+ + dot(yh, iq2_xxs_vals(IQ2_XXS_GRID(g0 + 1), sg0, 4u));
+ a1 += dot(yl, iq2_xxs_vals(IQ2_XXS_GRID(g1 ), sg1, 0u))
+ + dot(yh, iq2_xxs_vals(IQ2_XXS_GRID(g1 + 1), sg1, 4u));
+ }
+ acc0 += (0.5f + (float)(aux.s0 >> 28)) * a0;
+ acc1 += (0.5f + (float)(aux.s1 >> 28)) * a1;
+ }
+ sumf += d0 * 0.25f * acc0;
+ sumf1 += d1 * 0.25f * acc1;
+ }
+ }
+
+#if IQ2_XXS_MV_NSG > 1
+ __local float2 part[IQ2_XXS_MV_NSG][64];
+ part[sgi][lid] = (float2)(sumf, sumf1);
+ barrier(CLK_LOCAL_MEM_FENCE);
+ if (sgi != 0) {
+ return;
+ }
+ for (uint s = 1; s < IQ2_XXS_MV_NSG; ++s) {
+ const float2 p = part[s][lid];
+ sumf += p.s0;
+ sumf1 += p.s1;
+ }
+#endif
+
+ if (j < mh) {
+ vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row);
+ }
+#undef IQ2_XXS_GRID
+}
+
+kernel void kernel_iq2_xxs_grid_export(global uint * out) {
+ const uint i = get_global_id(0);
+ if (i < 512u) {
+ out[i] = iq2_xxs_grid[i];
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/iq2_s_grid.cl b/ggml/src/ggml-opencl/kernels/iq2_s_grid.cl
new file mode 100644
index 000000000..e06665746
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/iq2_s_grid.cl
@@ -0,0 +1,259 @@
+// IQ2_S codebook, stored as pairs of four packed magnitudes.
+constant uint iq2_s_grid[2048] = {
+ 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808,
+ 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808,
+ 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808,
+ 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808,
+ 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808,
+ 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x192b192b, 0x08080808,
+ 0x192b2b19, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808,
+ 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, 0x2b191908, 0x08080808, 0x2b2b0808, 0x08080808,
+ 0x2b2b1919, 0x08080808, 0x2b2b2b2b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819,
+ 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, 0x0819082b, 0x08080819,
+ 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x082b0819, 0x08080819, 0x082b1908, 0x08080819,
+ 0x19080808, 0x08080819, 0x1908082b, 0x08080819, 0x19081919, 0x08080819, 0x19082b08, 0x08080819,
+ 0x19190819, 0x08080819, 0x19191908, 0x08080819, 0x1919192b, 0x08080819, 0x19192b19, 0x08080819,
+ 0x192b0808, 0x08080819, 0x192b1919, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819,
+ 0x2b081908, 0x08080819, 0x2b190808, 0x08080819, 0x2b19082b, 0x08080819, 0x2b191919, 0x08080819,
+ 0x2b2b0819, 0x08080819, 0x2b2b1908, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b,
+ 0x08081919, 0x0808082b, 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b,
+ 0x082b0808, 0x0808082b, 0x082b2b2b, 0x0808082b, 0x19080819, 0x0808082b, 0x19081908, 0x0808082b,
+ 0x1908192b, 0x0808082b, 0x19082b19, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b,
+ 0x2b080808, 0x0808082b, 0x2b081919, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x2b191908, 0x0808082b,
+ 0x2b2b082b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x0808192b, 0x08081908,
+ 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, 0x08191919, 0x08081908,
+ 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, 0x082b192b, 0x08081908,
+ 0x082b2b19, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908,
+ 0x19082b08, 0x08081908, 0x19082b2b, 0x08081908, 0x19190819, 0x08081908, 0x19191908, 0x08081908,
+ 0x1919192b, 0x08081908, 0x19192b19, 0x08081908, 0x192b0808, 0x08081908, 0x192b082b, 0x08081908,
+ 0x192b1919, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b08192b, 0x08081908,
+ 0x2b082b19, 0x08081908, 0x2b190808, 0x08081908, 0x2b191919, 0x08081908, 0x2b192b08, 0x08081908,
+ 0x2b2b0819, 0x08081908, 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919,
+ 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08082b2b, 0x08081919, 0x08190819, 0x08081919,
+ 0x08191908, 0x08081919, 0x0819192b, 0x08081919, 0x08192b19, 0x08081919, 0x082b0808, 0x08081919,
+ 0x082b1919, 0x08081919, 0x082b2b08, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919,
+ 0x1908192b, 0x08081919, 0x19082b19, 0x08081919, 0x19190808, 0x08081919, 0x1919082b, 0x08081919,
+ 0x19191919, 0x08081919, 0x19192b08, 0x08081919, 0x192b0819, 0x08081919, 0x192b1908, 0x08081919,
+ 0x2b080808, 0x08081919, 0x2b08082b, 0x08081919, 0x2b081919, 0x08081919, 0x2b082b08, 0x08081919,
+ 0x2b190819, 0x08081919, 0x2b191908, 0x08081919, 0x2b2b0808, 0x08081919, 0x08080819, 0x0808192b,
+ 0x08081908, 0x0808192b, 0x0808192b, 0x0808192b, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b,
+ 0x08191919, 0x0808192b, 0x19080808, 0x0808192b, 0x19081919, 0x0808192b, 0x19082b08, 0x0808192b,
+ 0x19190819, 0x0808192b, 0x19191908, 0x0808192b, 0x192b0808, 0x0808192b, 0x2b080819, 0x0808192b,
+ 0x2b081908, 0x0808192b, 0x2b190808, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08,
+ 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08190819, 0x08082b08, 0x08191908, 0x08082b08,
+ 0x0819192b, 0x08082b08, 0x08192b19, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08,
+ 0x082b2b2b, 0x08082b08, 0x19080819, 0x08082b08, 0x19081908, 0x08082b08, 0x1908192b, 0x08082b08,
+ 0x19082b19, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x19191919, 0x08082b08,
+ 0x19192b08, 0x08082b08, 0x192b0819, 0x08082b08, 0x192b1908, 0x08082b08, 0x2b080808, 0x08082b08,
+ 0x2b081919, 0x08082b08, 0x2b191908, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19,
+ 0x08081908, 0x08082b19, 0x08190808, 0x08082b19, 0x0819082b, 0x08082b19, 0x08191919, 0x08082b19,
+ 0x08192b08, 0x08082b19, 0x082b0819, 0x08082b19, 0x19080808, 0x08082b19, 0x19081919, 0x08082b19,
+ 0x19082b08, 0x08082b19, 0x19190819, 0x08082b19, 0x19191908, 0x08082b19, 0x192b0808, 0x08082b19,
+ 0x2b080819, 0x08082b19, 0x2b190808, 0x08082b19, 0x08080808, 0x08082b2b, 0x08190819, 0x08082b2b,
+ 0x08191908, 0x08082b2b, 0x082b082b, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x082b2b2b, 0x08082b2b,
+ 0x19190808, 0x08082b2b, 0x2b192b19, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808,
+ 0x0808192b, 0x08190808, 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808,
+ 0x08191919, 0x08190808, 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808,
+ 0x082b192b, 0x08190808, 0x19080808, 0x08190808, 0x1908082b, 0x08190808, 0x19081919, 0x08190808,
+ 0x19082b08, 0x08190808, 0x19190819, 0x08190808, 0x19191908, 0x08190808, 0x1919192b, 0x08190808,
+ 0x19192b19, 0x08190808, 0x192b0808, 0x08190808, 0x192b082b, 0x08190808, 0x192b1919, 0x08190808,
+ 0x192b2b08, 0x08190808, 0x2b080819, 0x08190808, 0x2b081908, 0x08190808, 0x2b08192b, 0x08190808,
+ 0x2b190808, 0x08190808, 0x2b191919, 0x08190808, 0x2b192b08, 0x08190808, 0x2b2b0819, 0x08190808,
+ 0x2b2b1908, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, 0x08081919, 0x08190819,
+ 0x08082b08, 0x08190819, 0x08082b2b, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819,
+ 0x0819192b, 0x08190819, 0x08192b19, 0x08190819, 0x082b0808, 0x08190819, 0x082b082b, 0x08190819,
+ 0x082b1919, 0x08190819, 0x082b2b08, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819,
+ 0x1908192b, 0x08190819, 0x19082b19, 0x08190819, 0x19190808, 0x08190819, 0x1919082b, 0x08190819,
+ 0x19191919, 0x08190819, 0x19192b08, 0x08190819, 0x192b0819, 0x08190819, 0x192b1908, 0x08190819,
+ 0x2b080808, 0x08190819, 0x2b08082b, 0x08190819, 0x2b081919, 0x08190819, 0x2b082b08, 0x08190819,
+ 0x2b190819, 0x08190819, 0x2b191908, 0x08190819, 0x08080819, 0x0819082b, 0x08081908, 0x0819082b,
+ 0x08082b19, 0x0819082b, 0x08190808, 0x0819082b, 0x08191919, 0x0819082b, 0x082b0819, 0x0819082b,
+ 0x082b1908, 0x0819082b, 0x19080808, 0x0819082b, 0x19081919, 0x0819082b, 0x19190819, 0x0819082b,
+ 0x19191908, 0x0819082b, 0x2b080819, 0x0819082b, 0x2b081908, 0x0819082b, 0x2b190808, 0x0819082b,
+ 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, 0x08082b08, 0x08191908,
+ 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x0819192b, 0x08191908, 0x08192b19, 0x08191908,
+ 0x082b0808, 0x08191908, 0x082b1919, 0x08191908, 0x082b2b08, 0x08191908, 0x19080819, 0x08191908,
+ 0x19081908, 0x08191908, 0x1908192b, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908,
+ 0x1919082b, 0x08191908, 0x19191919, 0x08191908, 0x19192b08, 0x08191908, 0x192b0819, 0x08191908,
+ 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x2b08082b, 0x08191908, 0x2b081919, 0x08191908,
+ 0x2b082b08, 0x08191908, 0x2b190819, 0x08191908, 0x2b191908, 0x08191908, 0x2b2b0808, 0x08191908,
+ 0x08080819, 0x08191919, 0x08081908, 0x08191919, 0x0808192b, 0x08191919, 0x08082b19, 0x08191919,
+ 0x08190808, 0x08191919, 0x0819082b, 0x08191919, 0x08191919, 0x08191919, 0x08192b08, 0x08191919,
+ 0x082b0819, 0x08191919, 0x082b1908, 0x08191919, 0x19080808, 0x08191919, 0x1908082b, 0x08191919,
+ 0x19081919, 0x08191919, 0x19082b08, 0x08191919, 0x19190819, 0x08191919, 0x19191908, 0x08191919,
+ 0x192b0808, 0x08191919, 0x2b080819, 0x08191919, 0x2b081908, 0x08191919, 0x2b190808, 0x08191919,
+ 0x08080808, 0x0819192b, 0x08081919, 0x0819192b, 0x08082b08, 0x0819192b, 0x08190819, 0x0819192b,
+ 0x08191908, 0x0819192b, 0x082b0808, 0x0819192b, 0x19080819, 0x0819192b, 0x19081908, 0x0819192b,
+ 0x19190808, 0x0819192b, 0x2b080808, 0x0819192b, 0x2b2b2b2b, 0x0819192b, 0x08080819, 0x08192b08,
+ 0x08081908, 0x08192b08, 0x0808192b, 0x08192b08, 0x08082b19, 0x08192b08, 0x08190808, 0x08192b08,
+ 0x08191919, 0x08192b08, 0x08192b08, 0x08192b08, 0x082b0819, 0x08192b08, 0x19080808, 0x08192b08,
+ 0x1908082b, 0x08192b08, 0x19081919, 0x08192b08, 0x19082b08, 0x08192b08, 0x19190819, 0x08192b08,
+ 0x19191908, 0x08192b08, 0x192b0808, 0x08192b08, 0x2b080819, 0x08192b08, 0x2b081908, 0x08192b08,
+ 0x08080808, 0x08192b19, 0x0808082b, 0x08192b19, 0x08081919, 0x08192b19, 0x08082b08, 0x08192b19,
+ 0x08190819, 0x08192b19, 0x08191908, 0x08192b19, 0x082b0808, 0x08192b19, 0x19080819, 0x08192b19,
+ 0x19081908, 0x08192b19, 0x19190808, 0x08192b19, 0x192b2b19, 0x08192b19, 0x2b2b082b, 0x08192b19,
+ 0x08081908, 0x08192b2b, 0x08190808, 0x08192b2b, 0x19080808, 0x08192b2b, 0x1919192b, 0x08192b2b,
+ 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, 0x08082b08, 0x082b0808,
+ 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, 0x0819192b, 0x082b0808, 0x08192b19, 0x082b0808,
+ 0x082b0808, 0x082b0808, 0x082b1919, 0x082b0808, 0x082b2b2b, 0x082b0808, 0x19080819, 0x082b0808,
+ 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, 0x1919082b, 0x082b0808, 0x19191919, 0x082b0808,
+ 0x192b1908, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b082b2b, 0x082b0808, 0x2b191908, 0x082b0808,
+ 0x2b2b2b2b, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, 0x08190808, 0x082b0819,
+ 0x0819082b, 0x082b0819, 0x08191919, 0x082b0819, 0x082b0819, 0x082b0819, 0x19080808, 0x082b0819,
+ 0x1908082b, 0x082b0819, 0x19081919, 0x082b0819, 0x19190819, 0x082b0819, 0x19191908, 0x082b0819,
+ 0x192b0808, 0x082b0819, 0x2b080819, 0x082b0819, 0x2b081908, 0x082b0819, 0x2b190808, 0x082b0819,
+ 0x08080808, 0x082b082b, 0x08082b2b, 0x082b082b, 0x082b082b, 0x082b082b, 0x082b2b08, 0x082b082b,
+ 0x082b2b2b, 0x082b082b, 0x19081908, 0x082b082b, 0x19190808, 0x082b082b, 0x2b082b08, 0x082b082b,
+ 0x2b082b2b, 0x082b082b, 0x2b2b2b08, 0x082b082b, 0x08080819, 0x082b1908, 0x08081908, 0x082b1908,
+ 0x0808192b, 0x082b1908, 0x08082b19, 0x082b1908, 0x08190808, 0x082b1908, 0x08191919, 0x082b1908,
+ 0x08192b08, 0x082b1908, 0x082b0819, 0x082b1908, 0x082b1908, 0x082b1908, 0x19080808, 0x082b1908,
+ 0x1908082b, 0x082b1908, 0x19081919, 0x082b1908, 0x19082b08, 0x082b1908, 0x19190819, 0x082b1908,
+ 0x19191908, 0x082b1908, 0x192b0808, 0x082b1908, 0x2b080819, 0x082b1908, 0x2b081908, 0x082b1908,
+ 0x2b190808, 0x082b1908, 0x08080808, 0x082b1919, 0x08081919, 0x082b1919, 0x08082b08, 0x082b1919,
+ 0x08190819, 0x082b1919, 0x08191908, 0x082b1919, 0x082b0808, 0x082b1919, 0x19080819, 0x082b1919,
+ 0x19081908, 0x082b1919, 0x19190808, 0x082b1919, 0x192b192b, 0x082b1919, 0x2b080808, 0x082b1919,
+ 0x08080819, 0x082b192b, 0x08081908, 0x082b192b, 0x08190808, 0x082b192b, 0x19080808, 0x082b192b,
+ 0x19192b19, 0x082b192b, 0x08080808, 0x082b2b08, 0x08081919, 0x082b2b08, 0x08190819, 0x082b2b08,
+ 0x08191908, 0x082b2b08, 0x19080819, 0x082b2b08, 0x19081908, 0x082b2b08, 0x19190808, 0x082b2b08,
+ 0x2b082b2b, 0x082b2b08, 0x2b2b2b2b, 0x082b2b08, 0x08080819, 0x082b2b19, 0x08081908, 0x082b2b19,
+ 0x08190808, 0x082b2b19, 0x2b191919, 0x082b2b19, 0x08082b2b, 0x082b2b2b, 0x082b082b, 0x082b2b2b,
+ 0x192b1908, 0x082b2b2b, 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808,
+ 0x08081908, 0x19080808, 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808,
+ 0x0819082b, 0x19080808, 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x08192b2b, 0x19080808,
+ 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x082b192b, 0x19080808, 0x19080808, 0x19080808,
+ 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, 0x19082b2b, 0x19080808,
+ 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x1919192b, 0x19080808, 0x19192b19, 0x19080808,
+ 0x192b0808, 0x19080808, 0x192b082b, 0x19080808, 0x192b1919, 0x19080808, 0x2b080819, 0x19080808,
+ 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, 0x2b191919, 0x19080808, 0x2b192b08, 0x19080808,
+ 0x2b2b0819, 0x19080808, 0x2b2b1908, 0x19080808, 0x08080808, 0x19080819, 0x0808082b, 0x19080819,
+ 0x08081919, 0x19080819, 0x08082b08, 0x19080819, 0x08190819, 0x19080819, 0x08191908, 0x19080819,
+ 0x0819192b, 0x19080819, 0x08192b19, 0x19080819, 0x082b0808, 0x19080819, 0x082b082b, 0x19080819,
+ 0x082b1919, 0x19080819, 0x19080819, 0x19080819, 0x19081908, 0x19080819, 0x1908192b, 0x19080819,
+ 0x19082b19, 0x19080819, 0x19190808, 0x19080819, 0x1919082b, 0x19080819, 0x19191919, 0x19080819,
+ 0x19192b08, 0x19080819, 0x192b0819, 0x19080819, 0x192b1908, 0x19080819, 0x2b080808, 0x19080819,
+ 0x2b08082b, 0x19080819, 0x2b081919, 0x19080819, 0x2b082b08, 0x19080819, 0x2b190819, 0x19080819,
+ 0x2b191908, 0x19080819, 0x2b2b0808, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b,
+ 0x08190808, 0x1908082b, 0x0819082b, 0x1908082b, 0x08191919, 0x1908082b, 0x08192b08, 0x1908082b,
+ 0x082b1908, 0x1908082b, 0x19080808, 0x1908082b, 0x19081919, 0x1908082b, 0x19082b08, 0x1908082b,
+ 0x19190819, 0x1908082b, 0x19191908, 0x1908082b, 0x192b0808, 0x1908082b, 0x2b080819, 0x1908082b,
+ 0x2b081908, 0x1908082b, 0x08080808, 0x19081908, 0x0808082b, 0x19081908, 0x08081919, 0x19081908,
+ 0x08082b08, 0x19081908, 0x08082b2b, 0x19081908, 0x08190819, 0x19081908, 0x08191908, 0x19081908,
+ 0x0819192b, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x082b082b, 0x19081908,
+ 0x082b1919, 0x19081908, 0x082b2b08, 0x19081908, 0x19080819, 0x19081908, 0x19081908, 0x19081908,
+ 0x1908192b, 0x19081908, 0x19082b19, 0x19081908, 0x19190808, 0x19081908, 0x1919082b, 0x19081908,
+ 0x19191919, 0x19081908, 0x19192b08, 0x19081908, 0x192b0819, 0x19081908, 0x192b1908, 0x19081908,
+ 0x2b080808, 0x19081908, 0x2b08082b, 0x19081908, 0x2b081919, 0x19081908, 0x2b082b08, 0x19081908,
+ 0x2b190819, 0x19081908, 0x2b191908, 0x19081908, 0x2b2b0808, 0x19081908, 0x08080819, 0x19081919,
+ 0x08081908, 0x19081919, 0x0808192b, 0x19081919, 0x08082b19, 0x19081919, 0x08190808, 0x19081919,
+ 0x0819082b, 0x19081919, 0x08191919, 0x19081919, 0x08192b08, 0x19081919, 0x082b0819, 0x19081919,
+ 0x082b1908, 0x19081919, 0x19080808, 0x19081919, 0x1908082b, 0x19081919, 0x19081919, 0x19081919,
+ 0x19082b08, 0x19081919, 0x19190819, 0x19081919, 0x19191908, 0x19081919, 0x192b0808, 0x19081919,
+ 0x192b2b2b, 0x19081919, 0x2b080819, 0x19081919, 0x2b081908, 0x19081919, 0x2b190808, 0x19081919,
+ 0x08080808, 0x1908192b, 0x0808082b, 0x1908192b, 0x08081919, 0x1908192b, 0x08082b08, 0x1908192b,
+ 0x08190819, 0x1908192b, 0x08191908, 0x1908192b, 0x082b0808, 0x1908192b, 0x19080819, 0x1908192b,
+ 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x2b080808, 0x1908192b, 0x2b2b1919, 0x1908192b,
+ 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, 0x08082b19, 0x19082b08, 0x08190808, 0x19082b08,
+ 0x0819082b, 0x19082b08, 0x08191919, 0x19082b08, 0x08192b08, 0x19082b08, 0x082b0819, 0x19082b08,
+ 0x082b1908, 0x19082b08, 0x19080808, 0x19082b08, 0x1908082b, 0x19082b08, 0x19081919, 0x19082b08,
+ 0x19082b08, 0x19082b08, 0x19190819, 0x19082b08, 0x19191908, 0x19082b08, 0x192b0808, 0x19082b08,
+ 0x2b081908, 0x19082b08, 0x2b190808, 0x19082b08, 0x08080808, 0x19082b19, 0x0808082b, 0x19082b19,
+ 0x08081919, 0x19082b19, 0x08082b08, 0x19082b19, 0x08190819, 0x19082b19, 0x08191908, 0x19082b19,
+ 0x082b0808, 0x19082b19, 0x19080819, 0x19082b19, 0x19081908, 0x19082b19, 0x19190808, 0x19082b19,
+ 0x2b080808, 0x19082b19, 0x2b19192b, 0x19082b19, 0x08080819, 0x19082b2b, 0x08081908, 0x19082b2b,
+ 0x08190808, 0x19082b2b, 0x19080808, 0x19082b2b, 0x08080808, 0x19190808, 0x0808082b, 0x19190808,
+ 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, 0x08191908, 0x19190808,
+ 0x0819192b, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x082b082b, 0x19190808,
+ 0x082b1919, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, 0x19081908, 0x19190808,
+ 0x1908192b, 0x19190808, 0x19082b19, 0x19190808, 0x19190808, 0x19190808, 0x1919082b, 0x19190808,
+ 0x19191919, 0x19190808, 0x19192b08, 0x19190808, 0x192b0819, 0x19190808, 0x192b1908, 0x19190808,
+ 0x2b080808, 0x19190808, 0x2b08082b, 0x19190808, 0x2b081919, 0x19190808, 0x2b082b08, 0x19190808,
+ 0x2b190819, 0x19190808, 0x2b191908, 0x19190808, 0x08080819, 0x19190819, 0x08081908, 0x19190819,
+ 0x0808192b, 0x19190819, 0x08082b19, 0x19190819, 0x08190808, 0x19190819, 0x0819082b, 0x19190819,
+ 0x08191919, 0x19190819, 0x08192b08, 0x19190819, 0x082b0819, 0x19190819, 0x082b1908, 0x19190819,
+ 0x19080808, 0x19190819, 0x1908082b, 0x19190819, 0x19081919, 0x19190819, 0x19082b08, 0x19190819,
+ 0x19190819, 0x19190819, 0x19191908, 0x19190819, 0x192b0808, 0x19190819, 0x2b080819, 0x19190819,
+ 0x2b081908, 0x19190819, 0x2b190808, 0x19190819, 0x08080808, 0x1919082b, 0x08081919, 0x1919082b,
+ 0x08082b08, 0x1919082b, 0x08190819, 0x1919082b, 0x08191908, 0x1919082b, 0x082b0808, 0x1919082b,
+ 0x19080819, 0x1919082b, 0x19081908, 0x1919082b, 0x19190808, 0x1919082b, 0x192b2b19, 0x1919082b,
+ 0x2b080808, 0x1919082b, 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x0808192b, 0x19191908,
+ 0x08082b19, 0x19191908, 0x08190808, 0x19191908, 0x0819082b, 0x19191908, 0x08191919, 0x19191908,
+ 0x08192b08, 0x19191908, 0x082b0819, 0x19191908, 0x082b1908, 0x19191908, 0x19080808, 0x19191908,
+ 0x1908082b, 0x19191908, 0x19081919, 0x19191908, 0x19082b08, 0x19191908, 0x19190819, 0x19191908,
+ 0x19191908, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b081908, 0x19191908,
+ 0x2b190808, 0x19191908, 0x08080808, 0x19191919, 0x0808082b, 0x19191919, 0x08081919, 0x19191919,
+ 0x08082b08, 0x19191919, 0x08190819, 0x19191919, 0x08191908, 0x19191919, 0x082b0808, 0x19191919,
+ 0x19080819, 0x19191919, 0x19081908, 0x19191919, 0x19190808, 0x19191919, 0x2b080808, 0x19191919,
+ 0x08080819, 0x1919192b, 0x08081908, 0x1919192b, 0x08190808, 0x1919192b, 0x082b192b, 0x1919192b,
+ 0x19080808, 0x1919192b, 0x08080808, 0x19192b08, 0x0808082b, 0x19192b08, 0x08081919, 0x19192b08,
+ 0x08082b08, 0x19192b08, 0x08190819, 0x19192b08, 0x08191908, 0x19192b08, 0x082b0808, 0x19192b08,
+ 0x19080819, 0x19192b08, 0x19081908, 0x19192b08, 0x19190808, 0x19192b08, 0x19192b2b, 0x19192b08,
+ 0x2b080808, 0x19192b08, 0x08080819, 0x19192b19, 0x08081908, 0x19192b19, 0x08190808, 0x19192b19,
+ 0x19080808, 0x19192b19, 0x08080808, 0x19192b2b, 0x08192b19, 0x19192b2b, 0x2b081919, 0x19192b2b,
+ 0x2b2b2b08, 0x19192b2b, 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x0808192b, 0x192b0808,
+ 0x08190808, 0x192b0808, 0x0819082b, 0x192b0808, 0x08191919, 0x192b0808, 0x08192b08, 0x192b0808,
+ 0x082b0819, 0x192b0808, 0x082b1908, 0x192b0808, 0x19080808, 0x192b0808, 0x19081919, 0x192b0808,
+ 0x19082b08, 0x192b0808, 0x19190819, 0x192b0808, 0x19191908, 0x192b0808, 0x192b0808, 0x192b0808,
+ 0x2b081908, 0x192b0808, 0x2b190808, 0x192b0808, 0x08080808, 0x192b0819, 0x0808082b, 0x192b0819,
+ 0x08081919, 0x192b0819, 0x08082b08, 0x192b0819, 0x08190819, 0x192b0819, 0x08191908, 0x192b0819,
+ 0x082b0808, 0x192b0819, 0x19080819, 0x192b0819, 0x19081908, 0x192b0819, 0x19190808, 0x192b0819,
+ 0x2b080808, 0x192b0819, 0x2b192b19, 0x192b0819, 0x08081908, 0x192b082b, 0x08190808, 0x192b082b,
+ 0x19080808, 0x192b082b, 0x1919192b, 0x192b082b, 0x2b2b0819, 0x192b082b, 0x08080808, 0x192b1908,
+ 0x08081919, 0x192b1908, 0x08082b08, 0x192b1908, 0x08190819, 0x192b1908, 0x08191908, 0x192b1908,
+ 0x082b0808, 0x192b1908, 0x19080819, 0x192b1908, 0x19081908, 0x192b1908, 0x19190808, 0x192b1908,
+ 0x2b080808, 0x192b1908, 0x08080819, 0x192b1919, 0x08081908, 0x192b1919, 0x08190808, 0x192b1919,
+ 0x19080808, 0x192b1919, 0x19082b2b, 0x192b1919, 0x192b2b08, 0x192b1919, 0x2b19082b, 0x192b1919,
+ 0x08080808, 0x192b192b, 0x2b191908, 0x192b192b, 0x08080819, 0x192b2b08, 0x08081908, 0x192b2b08,
+ 0x08190808, 0x192b2b08, 0x192b1919, 0x192b2b08, 0x2b192b08, 0x192b2b08, 0x08080808, 0x192b2b19,
+ 0x082b2b2b, 0x192b2b19, 0x1908082b, 0x192b2b2b, 0x2b2b0819, 0x192b2b2b, 0x08080808, 0x2b080808,
+ 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, 0x08190819, 0x2b080808,
+ 0x08191908, 0x2b080808, 0x08192b19, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b1919, 0x2b080808,
+ 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x1919082b, 0x2b080808,
+ 0x19191919, 0x2b080808, 0x19192b08, 0x2b080808, 0x192b0819, 0x2b080808, 0x2b080808, 0x2b080808,
+ 0x2b081919, 0x2b080808, 0x2b190819, 0x2b080808, 0x2b191908, 0x2b080808, 0x08080819, 0x2b080819,
+ 0x08081908, 0x2b080819, 0x08082b19, 0x2b080819, 0x08190808, 0x2b080819, 0x0819082b, 0x2b080819,
+ 0x08191919, 0x2b080819, 0x08192b08, 0x2b080819, 0x082b0819, 0x2b080819, 0x082b1908, 0x2b080819,
+ 0x19080808, 0x2b080819, 0x1908082b, 0x2b080819, 0x19081919, 0x2b080819, 0x19082b08, 0x2b080819,
+ 0x19190819, 0x2b080819, 0x19191908, 0x2b080819, 0x2b080819, 0x2b080819, 0x2b081908, 0x2b080819,
+ 0x2b190808, 0x2b080819, 0x2b2b2b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x08081919, 0x2b08082b,
+ 0x08082b2b, 0x2b08082b, 0x08190819, 0x2b08082b, 0x08191908, 0x2b08082b, 0x19080819, 0x2b08082b,
+ 0x19081908, 0x2b08082b, 0x19190808, 0x2b08082b, 0x08080819, 0x2b081908, 0x08081908, 0x2b081908,
+ 0x0808192b, 0x2b081908, 0x08082b19, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908,
+ 0x08191919, 0x2b081908, 0x08192b08, 0x2b081908, 0x082b0819, 0x2b081908, 0x19080808, 0x2b081908,
+ 0x1908082b, 0x2b081908, 0x19081919, 0x2b081908, 0x19082b08, 0x2b081908, 0x19190819, 0x2b081908,
+ 0x19191908, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b080819, 0x2b081908, 0x2b081908, 0x2b081908,
+ 0x2b190808, 0x2b081908, 0x08080808, 0x2b081919, 0x0808082b, 0x2b081919, 0x08081919, 0x2b081919,
+ 0x08082b08, 0x2b081919, 0x08190819, 0x2b081919, 0x08191908, 0x2b081919, 0x082b0808, 0x2b081919,
+ 0x19080819, 0x2b081919, 0x19081908, 0x2b081919, 0x19190808, 0x2b081919, 0x2b080808, 0x2b081919,
+ 0x2b082b2b, 0x2b081919, 0x08080819, 0x2b08192b, 0x08081908, 0x2b08192b, 0x08190808, 0x2b08192b,
+ 0x082b2b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08081919, 0x2b082b08,
+ 0x08190819, 0x2b082b08, 0x08191908, 0x2b082b08, 0x19080819, 0x2b082b08, 0x19081908, 0x2b082b08,
+ 0x19190808, 0x2b082b08, 0x2b2b082b, 0x2b082b08, 0x08080819, 0x2b082b19, 0x08081908, 0x2b082b19,
+ 0x19080808, 0x2b082b19, 0x192b1919, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x19192b08, 0x2b082b2b,
+ 0x19192b2b, 0x2b082b2b, 0x2b08082b, 0x2b082b2b, 0x2b2b082b, 0x2b082b2b, 0x08080819, 0x2b190808,
+ 0x08081908, 0x2b190808, 0x08082b19, 0x2b190808, 0x08190808, 0x2b190808, 0x0819082b, 0x2b190808,
+ 0x08191919, 0x2b190808, 0x08192b08, 0x2b190808, 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808,
+ 0x1908082b, 0x2b190808, 0x19081919, 0x2b190808, 0x19082b08, 0x2b190808, 0x19190819, 0x2b190808,
+ 0x19191908, 0x2b190808, 0x192b0808, 0x2b190808, 0x2b080819, 0x2b190808, 0x2b081908, 0x2b190808,
+ 0x2b190808, 0x2b190808, 0x08080808, 0x2b190819, 0x08081919, 0x2b190819, 0x08190819, 0x2b190819,
+ 0x08191908, 0x2b190819, 0x19080819, 0x2b190819, 0x19081908, 0x2b190819, 0x19190808, 0x2b190819,
+ 0x19192b2b, 0x2b190819, 0x08080819, 0x2b19082b, 0x08081908, 0x2b19082b, 0x08190808, 0x2b19082b,
+ 0x19080808, 0x2b19082b, 0x2b2b192b, 0x2b19082b, 0x08080808, 0x2b191908, 0x0808082b, 0x2b191908,
+ 0x08081919, 0x2b191908, 0x08082b08, 0x2b191908, 0x08190819, 0x2b191908, 0x08191908, 0x2b191908,
+ 0x082b0808, 0x2b191908, 0x19080819, 0x2b191908, 0x19081908, 0x2b191908, 0x19190808, 0x2b191908,
+ 0x2b080808, 0x2b191908, 0x2b19192b, 0x2b191908, 0x08080819, 0x2b191919, 0x08081908, 0x2b191919,
+ 0x08190808, 0x2b191919, 0x19080808, 0x2b191919, 0x2b192b08, 0x2b191919, 0x2b2b0819, 0x2b191919,
+ 0x08080808, 0x2b19192b, 0x1908192b, 0x2b19192b, 0x192b1908, 0x2b19192b, 0x08080819, 0x2b192b08,
+ 0x08081908, 0x2b192b08, 0x08190808, 0x2b192b08, 0x082b192b, 0x2b192b08, 0x19080808, 0x2b192b08,
+ 0x2b2b2b19, 0x2b192b08, 0x08080808, 0x2b192b19, 0x19082b19, 0x2b192b19, 0x1919082b, 0x2b192b19,
+ 0x2b190808, 0x2b192b2b, 0x08080808, 0x2b2b0808, 0x08081919, 0x2b2b0808, 0x08082b2b, 0x2b2b0808,
+ 0x08191908, 0x2b2b0808, 0x082b082b, 0x2b2b0808, 0x082b2b2b, 0x2b2b0808, 0x19080819, 0x2b2b0808,
+ 0x19081908, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b2b082b, 0x2b2b0808, 0x2b2b2b2b, 0x2b2b0808,
+ 0x19080808, 0x2b2b0819, 0x192b1919, 0x2b2b0819, 0x0808082b, 0x2b2b082b, 0x08082b2b, 0x2b2b082b,
+ 0x082b082b, 0x2b2b082b, 0x082b2b08, 0x2b2b082b, 0x082b2b2b, 0x2b2b082b, 0x2b08082b, 0x2b2b082b,
+ 0x2b082b08, 0x2b2b082b, 0x2b082b2b, 0x2b2b082b, 0x2b2b2b08, 0x2b2b082b, 0x08080819, 0x2b2b1908,
+ 0x08081908, 0x2b2b1908, 0x08190808, 0x2b2b1908, 0x19080808, 0x2b2b1908, 0x2b082b19, 0x2b2b1908,
+ 0x2b2b1908, 0x2b2b1908, 0x08080808, 0x2b2b1919, 0x08192b19, 0x2b2b1919, 0x19190819, 0x2b2b192b,
+ 0x08082b2b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b082b, 0x2b2b2b08, 0x19191908, 0x2b2b2b19,
+ 0x2b08192b, 0x2b2b2b19, 0x08082b08, 0x2b2b2b2b, 0x08082b2b, 0x2b2b2b2b, 0x082b0808, 0x2b2b2b2b,
+ 0x082b082b, 0x2b2b2b2b, 0x082b2b08, 0x2b2b2b2b, 0x2b082b08, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b
+};
diff --git a/ggml/src/ggml-opencl/kernels/iq2_xs_grid.cl b/ggml/src/ggml-opencl/kernels/iq2_xs_grid.cl
new file mode 100644
index 000000000..0893713a9
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/iq2_xs_grid.cl
@@ -0,0 +1,135 @@
+// IQ2_XS codebook, stored as pairs of four packed magnitudes.
+constant uint iq2_xs_grid[1024] = {
+ 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808,
+ 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808,
+ 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808,
+ 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808,
+ 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808,
+ 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x2b080808, 0x08080808,
+ 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808,
+ 0x2b191908, 0x08080808, 0x2b192b19, 0x08080808, 0x2b2b0808, 0x08080808, 0x08080819, 0x08080819,
+ 0x08081908, 0x08080819, 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819,
+ 0x0819082b, 0x08080819, 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x08192b2b, 0x08080819,
+ 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, 0x19080808, 0x08080819, 0x1908082b, 0x08080819,
+ 0x19081919, 0x08080819, 0x19082b08, 0x08080819, 0x19190819, 0x08080819, 0x19191908, 0x08080819,
+ 0x192b0808, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, 0x2b081908, 0x08080819,
+ 0x2b190808, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x08081919, 0x0808082b,
+ 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, 0x082b0808, 0x0808082b,
+ 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b,
+ 0x2b080808, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908,
+ 0x0808192b, 0x08081908, 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908,
+ 0x08191919, 0x08081908, 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908,
+ 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, 0x19082b08, 0x08081908,
+ 0x19190819, 0x08081908, 0x19191908, 0x08081908, 0x1919192b, 0x08081908, 0x192b0808, 0x08081908,
+ 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, 0x08080808, 0x08081919,
+ 0x0808082b, 0x08081919, 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08190819, 0x08081919,
+ 0x08191908, 0x08081919, 0x082b0808, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919,
+ 0x19190808, 0x08081919, 0x192b0819, 0x08081919, 0x2b080808, 0x08081919, 0x08080819, 0x0808192b,
+ 0x08081908, 0x0808192b, 0x08190808, 0x0808192b, 0x082b192b, 0x0808192b, 0x19080808, 0x0808192b,
+ 0x1908082b, 0x0808192b, 0x2b081908, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08,
+ 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08082b2b, 0x08082b08, 0x08190819, 0x08082b08,
+ 0x08191908, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, 0x19080819, 0x08082b08,
+ 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x19192b08, 0x08082b08, 0x2b080808, 0x08082b08,
+ 0x2b2b0808, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, 0x08081908, 0x08082b19,
+ 0x08190808, 0x08082b19, 0x19080808, 0x08082b19, 0x2b080819, 0x08082b19, 0x2b082b19, 0x08082b19,
+ 0x08080808, 0x08082b2b, 0x082b0808, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x2b19192b, 0x08082b2b,
+ 0x2b2b0808, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x0808192b, 0x08190808,
+ 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, 0x08191919, 0x08190808,
+ 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, 0x19080808, 0x08190808,
+ 0x1908082b, 0x08190808, 0x19081919, 0x08190808, 0x19082b08, 0x08190808, 0x19190819, 0x08190808,
+ 0x19191908, 0x08190808, 0x192b0808, 0x08190808, 0x192b2b2b, 0x08190808, 0x2b080819, 0x08190808,
+ 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819,
+ 0x08081919, 0x08190819, 0x08082b08, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819,
+ 0x082b0808, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, 0x19190808, 0x08190819,
+ 0x2b080808, 0x08190819, 0x2b191908, 0x08190819, 0x2b19192b, 0x08190819, 0x08080819, 0x0819082b,
+ 0x08081908, 0x0819082b, 0x0808192b, 0x0819082b, 0x08190808, 0x0819082b, 0x19080808, 0x0819082b,
+ 0x192b0808, 0x0819082b, 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908,
+ 0x08082b08, 0x08191908, 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x082b0808, 0x08191908,
+ 0x19080819, 0x08191908, 0x19081908, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908,
+ 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x08080819, 0x08191919, 0x08081908, 0x08191919,
+ 0x08190808, 0x08191919, 0x19080808, 0x08191919, 0x08080808, 0x0819192b, 0x08191908, 0x0819192b,
+ 0x19082b19, 0x0819192b, 0x08080819, 0x08192b08, 0x08081908, 0x08192b08, 0x08190808, 0x08192b08,
+ 0x0819082b, 0x08192b08, 0x19080808, 0x08192b08, 0x19191908, 0x08192b08, 0x2b08192b, 0x08192b08,
+ 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x192b192b, 0x08192b19, 0x19190819, 0x08192b2b,
+ 0x2b2b2b19, 0x08192b2b, 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808,
+ 0x08082b08, 0x082b0808, 0x08082b2b, 0x082b0808, 0x08190819, 0x082b0808, 0x08191908, 0x082b0808,
+ 0x082b0808, 0x082b0808, 0x19080819, 0x082b0808, 0x19081908, 0x082b0808, 0x19190808, 0x082b0808,
+ 0x2b080808, 0x082b0808, 0x2b2b0808, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819,
+ 0x08190808, 0x082b0819, 0x19080808, 0x082b0819, 0x19082b08, 0x082b0819, 0x192b1919, 0x082b0819,
+ 0x08080808, 0x082b082b, 0x082b082b, 0x082b082b, 0x2b080808, 0x082b082b, 0x2b2b2b08, 0x082b082b,
+ 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x082b2b19, 0x082b1908,
+ 0x19080808, 0x082b1908, 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x1919082b, 0x082b1919,
+ 0x2b192b19, 0x082b1919, 0x08080819, 0x082b192b, 0x08192b2b, 0x082b192b, 0x2b2b192b, 0x082b192b,
+ 0x08080808, 0x082b2b08, 0x08082b08, 0x082b2b08, 0x08082b2b, 0x082b2b08, 0x082b0808, 0x082b2b08,
+ 0x19191919, 0x082b2b08, 0x2b082b08, 0x082b2b08, 0x2b2b082b, 0x082b2b08, 0x192b2b08, 0x082b2b19,
+ 0x2b190808, 0x082b2b19, 0x08082b08, 0x082b2b2b, 0x082b0808, 0x082b2b2b, 0x2b08082b, 0x082b2b2b,
+ 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, 0x08081908, 0x19080808,
+ 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, 0x0819082b, 0x19080808,
+ 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x082b0819, 0x19080808, 0x082b1908, 0x19080808,
+ 0x19080808, 0x19080808, 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808,
+ 0x19082b2b, 0x19080808, 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x192b0808, 0x19080808,
+ 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, 0x2b190808, 0x19080808,
+ 0x08080808, 0x19080819, 0x0808082b, 0x19080819, 0x08081919, 0x19080819, 0x08082b08, 0x19080819,
+ 0x08190819, 0x19080819, 0x08191908, 0x19080819, 0x082b0808, 0x19080819, 0x19080819, 0x19080819,
+ 0x19081908, 0x19080819, 0x19190808, 0x19080819, 0x2b080808, 0x19080819, 0x2b081919, 0x19080819,
+ 0x2b2b082b, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, 0x08190808, 0x1908082b,
+ 0x0819082b, 0x1908082b, 0x082b2b19, 0x1908082b, 0x19080808, 0x1908082b, 0x08080808, 0x19081908,
+ 0x0808082b, 0x19081908, 0x08081919, 0x19081908, 0x08082b08, 0x19081908, 0x08190819, 0x19081908,
+ 0x08191908, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x19080819, 0x19081908,
+ 0x19081908, 0x19081908, 0x19190808, 0x19081908, 0x2b080808, 0x19081908, 0x2b191908, 0x19081908,
+ 0x08080819, 0x19081919, 0x08081908, 0x19081919, 0x08190808, 0x19081919, 0x082b1908, 0x19081919,
+ 0x19080808, 0x19081919, 0x2b192b2b, 0x19081919, 0x08080808, 0x1908192b, 0x08082b2b, 0x1908192b,
+ 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x08080819, 0x19082b08, 0x08081908, 0x19082b08,
+ 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, 0x19191908, 0x19082b08,
+ 0x192b082b, 0x19082b08, 0x08080808, 0x19082b19, 0x08190819, 0x19082b19, 0x19081908, 0x19082b19,
+ 0x19190808, 0x19082b19, 0x192b2b19, 0x19082b19, 0x08081908, 0x19082b2b, 0x08080808, 0x19190808,
+ 0x0808082b, 0x19190808, 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808,
+ 0x08191908, 0x19190808, 0x082b0808, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808,
+ 0x19081908, 0x19190808, 0x19190808, 0x19190808, 0x2b080808, 0x19190808, 0x08080819, 0x19190819,
+ 0x08081908, 0x19190819, 0x08190808, 0x19190819, 0x08191919, 0x19190819, 0x19080808, 0x19190819,
+ 0x1908082b, 0x19190819, 0x08080808, 0x1919082b, 0x19081908, 0x1919082b, 0x2b2b2b2b, 0x1919082b,
+ 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x08190808, 0x19191908, 0x082b0819, 0x19191908,
+ 0x19080808, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b2b0819, 0x19191908,
+ 0x08080808, 0x19191919, 0x08082b08, 0x19191919, 0x2b080808, 0x19191919, 0x2b082b08, 0x19191919,
+ 0x082b0819, 0x1919192b, 0x192b2b08, 0x1919192b, 0x2b2b0819, 0x1919192b, 0x08080808, 0x19192b08,
+ 0x08191908, 0x19192b08, 0x19080819, 0x19192b08, 0x19190808, 0x19192b08, 0x2b192b19, 0x19192b08,
+ 0x08192b2b, 0x19192b19, 0x19080808, 0x19192b19, 0x1908082b, 0x19192b19, 0x2b081919, 0x19192b2b,
+ 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808,
+ 0x19191908, 0x192b0808, 0x192b082b, 0x192b0808, 0x2b08192b, 0x192b0808, 0x2b2b2b19, 0x192b0808,
+ 0x08080808, 0x192b0819, 0x082b1908, 0x192b082b, 0x19082b2b, 0x192b082b, 0x2b19082b, 0x192b082b,
+ 0x08080808, 0x192b1908, 0x0819192b, 0x192b1908, 0x08190808, 0x192b1919, 0x19080808, 0x192b1919,
+ 0x19081919, 0x192b1919, 0x2b2b1908, 0x192b1919, 0x08080819, 0x192b2b08, 0x192b2b2b, 0x192b2b08,
+ 0x082b1919, 0x192b2b19, 0x0808192b, 0x192b2b2b, 0x19191908, 0x192b2b2b, 0x192b082b, 0x192b2b2b,
+ 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808,
+ 0x08190819, 0x2b080808, 0x08191908, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b2b2b, 0x2b080808,
+ 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x2b080808, 0x2b080808,
+ 0x2b08082b, 0x2b080808, 0x2b2b2b08, 0x2b080808, 0x2b2b2b2b, 0x2b080808, 0x08080819, 0x2b080819,
+ 0x08081908, 0x2b080819, 0x0808192b, 0x2b080819, 0x08190808, 0x2b080819, 0x19080808, 0x2b080819,
+ 0x19190819, 0x2b080819, 0x19192b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x082b0808, 0x2b08082b,
+ 0x2b080808, 0x2b08082b, 0x2b08082b, 0x2b08082b, 0x2b2b0808, 0x2b08082b, 0x2b2b2b08, 0x2b08082b,
+ 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908,
+ 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b082b19, 0x2b081908,
+ 0x08080808, 0x2b081919, 0x19081908, 0x2b081919, 0x2b2b1919, 0x2b081919, 0x08192b08, 0x2b08192b,
+ 0x192b2b2b, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08082b08, 0x2b082b08, 0x082b1919, 0x2b082b08,
+ 0x19192b2b, 0x2b082b08, 0x2b080808, 0x2b082b08, 0x2b08082b, 0x2b082b08, 0x2b2b2b08, 0x2b082b08,
+ 0x0808192b, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x2b080808, 0x2b082b2b, 0x2b082b08, 0x2b082b2b,
+ 0x2b19192b, 0x2b082b2b, 0x2b2b2b08, 0x2b082b2b, 0x08080819, 0x2b190808, 0x08081908, 0x2b190808,
+ 0x08190808, 0x2b190808, 0x19080808, 0x2b190808, 0x1919192b, 0x2b190808, 0x2b081908, 0x2b190808,
+ 0x08080808, 0x2b190819, 0x082b082b, 0x2b190819, 0x192b1908, 0x2b190819, 0x1919192b, 0x2b19082b,
+ 0x2b082b19, 0x2b19082b, 0x08080808, 0x2b191908, 0x08081919, 0x2b191908, 0x19081908, 0x2b191908,
+ 0x19190808, 0x2b191908, 0x19192b08, 0x2b191908, 0x082b2b19, 0x2b191919, 0x2b190808, 0x2b191919,
+ 0x2b19082b, 0x2b191919, 0x19080819, 0x2b19192b, 0x19190819, 0x2b192b08, 0x2b2b192b, 0x2b192b08,
+ 0x19082b19, 0x2b192b19, 0x08191919, 0x2b192b2b, 0x192b0808, 0x2b192b2b, 0x08080808, 0x2b2b0808,
+ 0x0808082b, 0x2b2b0808, 0x08082b08, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, 0x082b0808, 0x2b2b0808,
+ 0x082b2b2b, 0x2b2b0808, 0x2b2b0808, 0x2b2b0808, 0x19190819, 0x2b2b0819, 0x19192b19, 0x2b2b0819,
+ 0x2b2b192b, 0x2b2b0819, 0x08080808, 0x2b2b082b, 0x0808082b, 0x2b2b082b, 0x08082b08, 0x2b2b082b,
+ 0x082b2b2b, 0x2b2b082b, 0x2b080808, 0x2b2b082b, 0x2b2b0808, 0x2b2b082b, 0x19080808, 0x2b2b1908,
+ 0x2b191919, 0x2b2b1908, 0x192b1919, 0x2b2b192b, 0x2b192b08, 0x2b2b192b, 0x08082b2b, 0x2b2b2b08,
+ 0x082b0808, 0x2b2b2b08, 0x082b082b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b0808, 0x2b2b2b08,
+ 0x2b2b2b08, 0x2b2b2b08, 0x08081908, 0x2b2b2b19, 0x2b081908, 0x2b2b2b19, 0x2b08192b, 0x2b2b2b19,
+ 0x082b2b08, 0x2b2b2b2b, 0x082b2b2b, 0x2b2b2b2b, 0x2b190819, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b
+};
+
+inline uint iq2_xs_signs(uint code) {
+ return code | ((popcount(code) & 1u) << 7);
+}
diff --git a/ggml/src/ggml-opencl/kernels/iq2_xxs_grid.cl b/ggml/src/ggml-opencl/kernels/iq2_xxs_grid.cl
new file mode 100644
index 000000000..794e097ef
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/iq2_xxs_grid.cl
@@ -0,0 +1,71 @@
+// IQ2_XXS codebook, stored as pairs of four packed magnitudes.
+constant uint iq2_xxs_grid[512] = {
+ 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808,
+ 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x082b0808, 0x08080808,
+ 0x082b082b, 0x08080808, 0x082b2b08, 0x08080808, 0x082b2b2b, 0x08080808, 0x19080819, 0x08080808,
+ 0x19081908, 0x08080808, 0x19190808, 0x08080808, 0x19192b08, 0x08080808, 0x192b0819, 0x08080808,
+ 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b082b2b, 0x08080808,
+ 0x2b2b082b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819, 0x08190808, 0x08080819,
+ 0x08191919, 0x08080819, 0x19080808, 0x08080819, 0x2b081908, 0x08080819, 0x2b192b08, 0x08080819,
+ 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x082b082b, 0x0808082b, 0x2b08082b, 0x0808082b,
+ 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x08190808, 0x08081908, 0x082b0819, 0x08081908,
+ 0x082b1908, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19082b08, 0x08081908,
+ 0x192b0808, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908,
+ 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919, 0x08082b08, 0x08081919,
+ 0x082b0808, 0x08081919, 0x1908192b, 0x08081919, 0x192b2b19, 0x08081919, 0x2b080808, 0x08081919,
+ 0x2b190819, 0x08081919, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b, 0x19080808, 0x0808192b,
+ 0x2b081908, 0x0808192b, 0x2b2b1908, 0x0808192b, 0x08080808, 0x08082b08, 0x08081919, 0x08082b08,
+ 0x08082b08, 0x08082b08, 0x08191908, 0x08082b08, 0x082b2b08, 0x08082b08, 0x19080819, 0x08082b08,
+ 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x2b082b08, 0x08082b08,
+ 0x08081908, 0x08082b19, 0x19080808, 0x08082b19, 0x0808082b, 0x08082b2b, 0x08191908, 0x08082b2b,
+ 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x08190808, 0x08190808, 0x082b0819, 0x08190808,
+ 0x19080808, 0x08190808, 0x192b0808, 0x08190808, 0x2b081908, 0x08190808, 0x2b190808, 0x08190808,
+ 0x2b191919, 0x08190808, 0x08080808, 0x08190819, 0x08082b08, 0x08190819, 0x082b0808, 0x08190819,
+ 0x19190808, 0x08190819, 0x19192b2b, 0x08190819, 0x2b080808, 0x08190819, 0x082b1908, 0x0819082b,
+ 0x19081919, 0x0819082b, 0x08080808, 0x08191908, 0x08082b08, 0x08191908, 0x082b0808, 0x08191908,
+ 0x082b1919, 0x08191908, 0x19082b19, 0x08191908, 0x2b080808, 0x08191908, 0x08192b08, 0x08191919,
+ 0x192b082b, 0x08191919, 0x08080808, 0x0819192b, 0x0819192b, 0x0819192b, 0x08080819, 0x08192b08,
+ 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, 0x19080808, 0x08192b08, 0x2b080819, 0x08192b08,
+ 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x2b2b0808, 0x08192b19, 0x19190819, 0x08192b2b,
+ 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08082b2b, 0x082b0808, 0x19081908, 0x082b0808,
+ 0x192b0819, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b08082b, 0x082b0808, 0x082b2b19, 0x082b0819,
+ 0x19082b08, 0x082b0819, 0x08080808, 0x082b082b, 0x0808082b, 0x082b082b, 0x08080819, 0x082b1908,
+ 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x19080808, 0x082b1908, 0x1919192b, 0x082b1908,
+ 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x192b1908, 0x082b1919, 0x2b190808, 0x082b192b,
+ 0x08082b08, 0x082b2b08, 0x082b0808, 0x082b2b08, 0x2b191908, 0x082b2b08, 0x19081908, 0x082b2b2b,
+ 0x08080819, 0x19080808, 0x08081908, 0x19080808, 0x08190808, 0x19080808, 0x08192b08, 0x19080808,
+ 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x19080808, 0x19080808, 0x19082b08, 0x19080808,
+ 0x1919192b, 0x19080808, 0x192b0808, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808,
+ 0x2b190808, 0x19080808, 0x08080808, 0x19080819, 0x082b0808, 0x19080819, 0x192b0819, 0x19080819,
+ 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, 0x08080819, 0x1908082b, 0x08190808, 0x1908082b,
+ 0x19082b08, 0x1908082b, 0x1919192b, 0x1908082b, 0x192b2b08, 0x1908082b, 0x08080808, 0x19081908,
+ 0x08082b08, 0x19081908, 0x082b0808, 0x19081908, 0x2b080808, 0x19081908, 0x2b192b19, 0x19081908,
+ 0x0819082b, 0x19081919, 0x082b1908, 0x19081919, 0x08080808, 0x1908192b, 0x08080819, 0x19082b08,
+ 0x08081908, 0x19082b08, 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08,
+ 0x08080808, 0x19082b19, 0x19192b08, 0x19082b19, 0x192b0819, 0x19082b19, 0x2b08082b, 0x19082b19,
+ 0x19081919, 0x19082b2b, 0x2b190808, 0x19082b2b, 0x08080808, 0x19190808, 0x08082b08, 0x19190808,
+ 0x08190819, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x2b080808, 0x19190808,
+ 0x2b082b08, 0x19190808, 0x08081908, 0x19190819, 0x1908082b, 0x19190819, 0x2b2b1908, 0x19190819,
+ 0x2b190819, 0x1919082b, 0x2b190808, 0x19191908, 0x2b19082b, 0x19191908, 0x08082b2b, 0x19191919,
+ 0x08080819, 0x1919192b, 0x19191908, 0x1919192b, 0x08080808, 0x19192b08, 0x08190819, 0x19192b08,
+ 0x08192b19, 0x19192b08, 0x192b1908, 0x19192b08, 0x19080808, 0x19192b19, 0x08082b08, 0x19192b2b,
+ 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, 0x192b2b08, 0x192b0808,
+ 0x08080808, 0x192b0819, 0x19191919, 0x192b0819, 0x08192b08, 0x192b082b, 0x192b0808, 0x192b082b,
+ 0x08080808, 0x192b1908, 0x08081919, 0x192b1908, 0x08190808, 0x192b1919, 0x0819082b, 0x192b1919,
+ 0x2b081908, 0x192b1919, 0x1908082b, 0x192b2b08, 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808,
+ 0x08082b2b, 0x2b080808, 0x19080819, 0x2b080808, 0x2b08082b, 0x2b080808, 0x08081908, 0x2b080819,
+ 0x08192b08, 0x2b080819, 0x19080808, 0x2b080819, 0x08190819, 0x2b08082b, 0x08080819, 0x2b081908,
+ 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x08191919, 0x2b081908, 0x19080808, 0x2b081908,
+ 0x192b0808, 0x2b081908, 0x08080808, 0x2b081919, 0x1908192b, 0x2b081919, 0x2b191908, 0x2b081919,
+ 0x08082b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x192b0808, 0x2b08192b, 0x0808082b, 0x2b082b08,
+ 0x08081908, 0x2b082b19, 0x08190819, 0x2b082b2b, 0x08081908, 0x2b190808, 0x08190808, 0x2b190808,
+ 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808, 0x2b2b0819, 0x2b190808, 0x0819192b, 0x2b190819,
+ 0x2b080808, 0x2b190819, 0x19081919, 0x2b19082b, 0x08080808, 0x2b191908, 0x082b082b, 0x2b191908,
+ 0x19081908, 0x2b191908, 0x19190819, 0x2b191919, 0x2b080819, 0x2b192b08, 0x082b0808, 0x2b192b19,
+ 0x0808082b, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b081919, 0x2b2b0808, 0x08082b19, 0x2b2b0819,
+ 0x08080808, 0x2b2b082b, 0x08192b08, 0x2b2b1908, 0x19190808, 0x2b2b2b08, 0x08081908, 0x2b2b2b19
+};
+
+inline uint iq2_xxs_signs(uint code) {
+ return code | ((popcount(code) & 1u) << 7);
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq2_s_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq2_s_f32_l4_lm.cl
new file mode 100644
index 000000000..f33f500a4
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq2_s_f32_l4_lm.cl
@@ -0,0 +1,191 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_required_subgroup_size
+#define INTEL_GPU 1
+#endif
+
+#define QK_K 256
+
+struct block_iq2_s {
+ half d;
+ uchar qs[QK_K/4];
+ uchar qh[QK_K/32];
+ uchar scales[QK_K/32];
+};
+
+#define LOAD_VEC_A 4
+#define LOAD_VEC_B 4
+
+#define BM 64
+#define BN 64
+#define BK 32
+#ifdef INTEL_GPU
+#define TM 8
+#define TN 8
+#else
+#define TM 4
+#define TN 8
+#endif
+
+kernel void kernel_mul_mm_iq2_s_f32_l4_lm(
+ global uchar * src0_qs,
+ global uchar * src0_sg,
+ global uchar * src0_qh,
+ global uchar * src0_sc,
+ global half * src0_d,
+ ulong offset0,
+ global float4 * src1,
+ ulong offset1,
+ global float * dst,
+ ulong offsetd,
+
+ int ne00,
+ int ne01,
+ int ne02,
+ int ne11,
+ int ne12,
+
+ int stride_a,
+ int stride_b,
+ int stride_d,
+
+ int batch_stride_a,
+ int batch_stride_b,
+ int batch_stride_d,
+
+ int r2,
+ int r3
+) {
+ src1 = (global float4*)((global char*)src1 + offset1);
+ dst = (global float *)((global char*)dst + offsetd);
+
+ local float buf_a[BM * BK];
+ local float buf_b[BN * BK];
+
+ const int batch_idx = get_global_id(2);
+
+ const int i13 = batch_idx / ne12;
+ const int i12 = batch_idx % ne12;
+
+ const int i03 = i13 / r3;
+ const int i02 = i12 / r2;
+
+ const int batch_idx_a = i03 * ne02 + i02;
+
+ const int ir = get_group_id(0);
+ const int ic = get_group_id(1);
+
+ const int tid = get_local_id(0);
+ const int th_r = tid % (BM / TM);
+ const int th_c = tid / (BM / TM);
+
+ const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
+ const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
+ const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
+ const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
+
+ const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
+ const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
+
+ // pos_a counts elements, not blocks - one iq2_s super block holds QK_K of them
+ int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
+ int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
+
+ float sums[TM * TN];
+ float cache_a[TM];
+ float cache_b[TN];
+
+ for (int i = 0; i < TM * TN; i++) {
+ sums[i] = 0.0f;
+ }
+
+ for (int block = 0; block < ne00; block += BK) {
+ for (int l = 0; l < BM; l += loadstride_a) {
+ if (ir*BM + loadc_a + l < ne01) {
+ int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
+ int ib = idx / QK_K;
+ ib += offset0/sizeof(struct block_iq2_s);
+ int e = idx % QK_K;
+
+ int ib32 = e >> 5;
+ int rem = e & 31;
+ int lg = rem >> 3;
+ int j0 = rem & 7;
+
+ uchar scb = src0_sc[8*ib + ib32];
+ float db = (float)src0_d[ib] * (0.5f + (float)((lg < 2) ? (scb & 0xf) : (scb >> 4))) * 0.25f;
+
+ uint gi = (uint)src0_qs[32*ib + 4*ib32 + lg] | (((uint)src0_qh[8*ib + ib32] << (8-2*lg)) & 0x300);
+ uchar sg = src0_sg[32*ib + 4*ib32 + lg];
+ uint g = iq2_s_grid[2*gi + (j0 >> 2)];
+
+ float4 v1;
+ v1.s0 = db * (float)((g >> 0) & 0xFF) * ((sg & (1 << (j0+0))) ? -1.f : 1.f);
+ v1.s1 = db * (float)((g >> 8) & 0xFF) * ((sg & (1 << (j0+1))) ? -1.f : 1.f);
+ v1.s2 = db * (float)((g >> 16) & 0xFF) * ((sg & (1 << (j0+2))) ? -1.f : 1.f);
+ v1.s3 = db * (float)((g >> 24) & 0xFF) * ((sg & (1 << (j0+3))) ? -1.f : 1.f);
+
+ buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
+ buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
+ buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
+ buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
+ } else {
+ buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
+ buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
+ buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
+ buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
+ }
+ }
+
+ for (int l = 0; l < BN; l += loadstride_b) {
+ if (ic*BN + loadc_b + l < ne11) {
+ int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
+ buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
+ buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
+ buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
+ buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
+ } else {
+ buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
+ buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
+ buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
+ buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
+ }
+ }
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ pos_a += BK;
+ pos_b += BK / LOAD_VEC_B;
+
+ for (int i = 0; i < BK; i++) {
+ for (int j = 0; j < TM; j++) {
+ cache_a[j] = buf_a[(i) * BM + th_r * TM + j];
+ }
+
+ for (int j = 0; j < TN; j++) {
+ cache_b[j] = buf_b[(i) * BN + th_c * TN + j];
+ }
+
+ for (int cc = 0; cc < TN; cc++) {
+ for (int cr = 0; cr < TM; cr++) {
+ const int sums_idx = cc*TM + cr;
+ sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);
+ }
+ }
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ const int dr = ir * BM + th_r * TM;
+ const int dc = ic * BN + th_c * TN;
+
+ const ulong offsets = batch_idx * batch_stride_d;
+
+ for (int cc = 0; cc < TN; cc++) {
+ for (int cr = 0; cr < TM; cr++) {
+ if (dr + cr < ne01 && dc + cc < ne11) {
+ dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];
+ }
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq2_xs_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq2_xs_f32_l4_lm.cl
new file mode 100644
index 000000000..cdff38bc4
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq2_xs_f32_l4_lm.cl
@@ -0,0 +1,189 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_required_subgroup_size
+#define INTEL_GPU 1
+#endif
+
+#define QK_K 256
+
+struct block_iq2_xs {
+ half d;
+ ushort qs[QK_K/8];
+ uchar scales[QK_K/32];
+};
+
+#define LOAD_VEC_A 4
+#define LOAD_VEC_B 4
+
+#define BM 64
+#define BN 64
+#define BK 32
+#ifdef INTEL_GPU
+#define TM 8
+#define TN 8
+#else
+#define TM 4
+#define TN 8
+#endif
+
+kernel void kernel_mul_mm_iq2_xs_f32_l4_lm(
+ global ushort * src0_qs,
+ global uchar * src0_sc,
+ global half * src0_d,
+ ulong offset0,
+ global float4 * src1,
+ ulong offset1,
+ global float * dst,
+ ulong offsetd,
+
+ int ne00,
+ int ne01,
+ int ne02,
+ int ne11,
+ int ne12,
+
+ int stride_a,
+ int stride_b,
+ int stride_d,
+
+ int batch_stride_a,
+ int batch_stride_b,
+ int batch_stride_d,
+
+ int r2,
+ int r3
+) {
+ src1 = (global float4*)((global char*)src1 + offset1);
+ dst = (global float *)((global char*)dst + offsetd);
+
+ local float buf_a[BM * BK];
+ local float buf_b[BN * BK];
+
+ const int batch_idx = get_global_id(2);
+
+ const int i13 = batch_idx / ne12;
+ const int i12 = batch_idx % ne12;
+
+ const int i03 = i13 / r3;
+ const int i02 = i12 / r2;
+
+ const int batch_idx_a = i03 * ne02 + i02;
+
+ const int ir = get_group_id(0);
+ const int ic = get_group_id(1);
+
+ const int tid = get_local_id(0);
+ const int th_r = tid % (BM / TM);
+ const int th_c = tid / (BM / TM);
+
+ const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
+ const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
+ const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
+ const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
+
+ const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
+ const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
+
+ // pos_a counts elements, not blocks - one iq2_xs super block holds QK_K of them
+ int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
+ int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
+
+ float sums[TM * TN];
+ float cache_a[TM];
+ float cache_b[TN];
+
+ for (int i = 0; i < TM * TN; i++) {
+ sums[i] = 0.0f;
+ }
+
+ for (int block = 0; block < ne00; block += BK) {
+ for (int l = 0; l < BM; l += loadstride_a) {
+ if (ir*BM + loadc_a + l < ne01) {
+ int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
+ int ib = idx / QK_K;
+ ib += offset0/sizeof(struct block_iq2_xs);
+ int e = idx % QK_K;
+
+ int ib32 = e >> 5;
+ int rem = e & 31;
+ int lg = rem >> 3;
+ int j0 = rem & 7;
+
+ uchar scb = src0_sc[8*ib + ib32];
+ float db = (float)src0_d[ib] * (0.5f + (float)((lg < 2) ? (scb & 0xf) : (scb >> 4))) * 0.25f;
+
+ uint q = src0_qs[32*ib + 4*ib32 + lg];
+ uint gi = q & 511u;
+ uint sg = iq2_xs_signs(q >> 9);
+ uint g = iq2_xs_grid[2*gi + (j0 >> 2)];
+
+ float4 v1;
+ v1.s0 = db * (float)((g >> 0) & 0xFF) * ((sg & (1 << (j0+0))) ? -1.f : 1.f);
+ v1.s1 = db * (float)((g >> 8) & 0xFF) * ((sg & (1 << (j0+1))) ? -1.f : 1.f);
+ v1.s2 = db * (float)((g >> 16) & 0xFF) * ((sg & (1 << (j0+2))) ? -1.f : 1.f);
+ v1.s3 = db * (float)((g >> 24) & 0xFF) * ((sg & (1 << (j0+3))) ? -1.f : 1.f);
+
+ buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
+ buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
+ buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
+ buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
+ } else {
+ buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
+ buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
+ buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
+ buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
+ }
+ }
+
+ for (int l = 0; l < BN; l += loadstride_b) {
+ if (ic*BN + loadc_b + l < ne11) {
+ int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
+ buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
+ buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
+ buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
+ buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
+ } else {
+ buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
+ buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
+ buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
+ buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
+ }
+ }
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ pos_a += BK;
+ pos_b += BK / LOAD_VEC_B;
+
+ for (int i = 0; i < BK; i++) {
+ for (int j = 0; j < TM; j++) {
+ cache_a[j] = buf_a[(i) * BM + th_r * TM + j];
+ }
+
+ for (int j = 0; j < TN; j++) {
+ cache_b[j] = buf_b[(i) * BN + th_c * TN + j];
+ }
+
+ for (int cc = 0; cc < TN; cc++) {
+ for (int cr = 0; cr < TM; cr++) {
+ const int sums_idx = cc*TM + cr;
+ sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);
+ }
+ }
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ const int dr = ir * BM + th_r * TM;
+ const int dc = ic * BN + th_c * TN;
+
+ const ulong offsets = batch_idx * batch_stride_d;
+
+ for (int cc = 0; cc < TN; cc++) {
+ for (int cr = 0; cr < TM; cr++) {
+ if (dr + cr < ne01 && dc + cc < ne11) {
+ dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];
+ }
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq2_xxs_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq2_xxs_f32_l4_lm.cl
new file mode 100644
index 000000000..7a44038b8
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq2_xxs_f32_l4_lm.cl
@@ -0,0 +1,187 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_required_subgroup_size
+#define INTEL_GPU 1
+#endif
+
+#define QK_K 256
+
+struct block_iq2_xxs {
+ half d;
+ ushort qs[QK_K/8];
+};
+
+#define LOAD_VEC_A 4
+#define LOAD_VEC_B 4
+
+#define BM 64
+#define BN 64
+#define BK 32
+#ifdef INTEL_GPU
+#define TM 8
+#define TN 8
+#else
+#define TM 4
+#define TN 8
+#endif
+
+kernel void kernel_mul_mm_iq2_xxs_f32_l4_lm(
+ global uchar * src0_qs,
+ global uint * src0_sas,
+ global half * src0_d,
+ ulong offset0,
+ global float4 * src1,
+ ulong offset1,
+ global float * dst,
+ ulong offsetd,
+
+ int ne00,
+ int ne01,
+ int ne02,
+ int ne11,
+ int ne12,
+
+ int stride_a,
+ int stride_b,
+ int stride_d,
+
+ int batch_stride_a,
+ int batch_stride_b,
+ int batch_stride_d,
+
+ int r2,
+ int r3
+) {
+ src1 = (global float4*)((global char*)src1 + offset1);
+ dst = (global float *)((global char*)dst + offsetd);
+
+ local float buf_a[BM * BK];
+ local float buf_b[BN * BK];
+
+ const int batch_idx = get_global_id(2);
+
+ const int i13 = batch_idx / ne12;
+ const int i12 = batch_idx % ne12;
+
+ const int i03 = i13 / r3;
+ const int i02 = i12 / r2;
+
+ const int batch_idx_a = i03 * ne02 + i02;
+
+ const int ir = get_group_id(0);
+ const int ic = get_group_id(1);
+
+ const int tid = get_local_id(0);
+ const int th_r = tid % (BM / TM);
+ const int th_c = tid / (BM / TM);
+
+ const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
+ const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
+ const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
+ const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
+
+ const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
+ const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
+
+ // pos_a counts elements, not blocks - one iq2_xxs super block holds QK_K of them
+ int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
+ int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
+
+ float sums[TM * TN];
+ float cache_a[TM];
+ float cache_b[TN];
+
+ for (int i = 0; i < TM * TN; i++) {
+ sums[i] = 0.0f;
+ }
+
+ for (int block = 0; block < ne00; block += BK) {
+ for (int l = 0; l < BM; l += loadstride_a) {
+ if (ir*BM + loadc_a + l < ne01) {
+ int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
+ int ib = idx / QK_K;
+ ib += offset0/sizeof(struct block_iq2_xxs);
+ int e = idx % QK_K;
+
+ int ib32 = e >> 5;
+ int rem = e & 31;
+ int lg = rem >> 3;
+ int j0 = rem & 7;
+
+ uint aux = src0_sas[8*ib + ib32];
+ float db = (float)src0_d[ib] * (0.5f + (float)(aux >> 28)) * 0.25f;
+
+ uint gi = src0_qs[32*ib + 4*ib32 + lg];
+ uint sg = iq2_xxs_signs((aux >> (7*lg)) & 127u);
+ uint g = iq2_xxs_grid[2*gi + (j0 >> 2)];
+
+ float4 v1;
+ v1.s0 = db * (float)((g >> 0) & 0xFF) * ((sg & (1 << (j0+0))) ? -1.f : 1.f);
+ v1.s1 = db * (float)((g >> 8) & 0xFF) * ((sg & (1 << (j0+1))) ? -1.f : 1.f);
+ v1.s2 = db * (float)((g >> 16) & 0xFF) * ((sg & (1 << (j0+2))) ? -1.f : 1.f);
+ v1.s3 = db * (float)((g >> 24) & 0xFF) * ((sg & (1 << (j0+3))) ? -1.f : 1.f);
+
+ buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
+ buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
+ buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
+ buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
+ } else {
+ buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
+ buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
+ buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
+ buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
+ }
+ }
+
+ for (int l = 0; l < BN; l += loadstride_b) {
+ if (ic*BN + loadc_b + l < ne11) {
+ int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
+ buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
+ buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
+ buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
+ buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
+ } else {
+ buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
+ buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
+ buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
+ buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
+ }
+ }
+
+ barrier(CLK_LOCAL_MEM_FENCE);
+
+ pos_a += BK;
+ pos_b += BK / LOAD_VEC_B;
+
+ for (int i = 0; i < BK; i++) {
+ for (int j = 0; j < TM; j++) {
+ cache_a[j] = buf_a[(i) * BM + th_r * TM + j];
+ }
+
+ for (int j = 0; j < TN; j++) {
+ cache_b[j] = buf_b[(i) * BN + th_c * TN + j];
+ }
+
+ for (int cc = 0; cc < TN; cc++) {
+ for (int cr = 0; cr < TM; cr++) {
+ const int sums_idx = cc*TM + cr;
+ sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);
+ }
+ }
+ }
+ barrier(CLK_LOCAL_MEM_FENCE);
+ }
+
+ const int dr = ir * BM + th_r * TM;
+ const int dc = ic * BN + th_c * TN;
+
+ const ulong offsets = batch_idx * batch_stride_d;
+
+ for (int cc = 0; cc < TN; cc++) {
+ for (int cr = 0; cr < TM; cr++) {
+ if (dr + cr < ne01 && dc + cc < ne11) {
+ dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];
+ }
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq2_s_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_s_f32.cl
new file mode 100644
index 000000000..43650972b
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_s_f32.cl
@@ -0,0 +1,147 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_required_subgroup_size
+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
+#define INTEL_GPU 1
+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
+#elif defined(cl_qcom_reqd_sub_group_size)
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+
+#define QK_K 256
+
+// qs holds 32 grid low bytes then 32 sign bytes
+typedef struct {
+ half d;
+ uchar qs[QK_K/4];
+ uchar qh[QK_K/32];
+ uchar scales[QK_K/32];
+} block_iq2_s;
+
+#undef N_DST
+#undef N_SIMDGROUP
+#undef N_SIMDWIDTH
+
+#ifdef INTEL_GPU
+#define N_DST 4
+#define N_SIMDGROUP 1
+#define N_SIMDWIDTH 16
+#elif defined (ADRENO_GPU)
+#define N_DST 4
+#define N_SIMDGROUP 1
+#define N_SIMDWIDTH 64
+#endif
+
+#undef BLOCK_STRIDE
+// 8 threads cover one super block, one 32 element sub block each
+#define BLOCK_STRIDE (N_SIMDWIDTH/8)
+
+#ifdef INTEL_GPU
+REQD_SUBGROUP_SIZE_16
+#elif defined (ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mv_iq2_s_f32(
+ global char * src0,
+ ulong offset0,
+ global char * src1,
+ ulong offset1,
+ global char * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne12,
+ ulong nb11,
+ ulong nb12,
+ ulong nb13,
+ int ne0,
+ int ne1,
+ int r2,
+ int r3
+) {
+ src0 = src0 + offset0;
+ src1 = src1 + offset1;
+ dst = dst + offsetd;
+
+ int ix = get_sub_group_local_id()/8; // super block index
+ int it = get_sub_group_local_id()%8; // sub block inside the super block
+
+ int nb = ne00/QK_K;
+
+ int r0 = get_group_id(0);
+ int r1 = get_group_id(1);
+ int im = get_group_id(2);
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
+
+ int i12 = im%ne12;
+ int i13 = im/ne12;
+
+ ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
+ ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;
+
+ global block_iq2_s * x = (global block_iq2_s *) (src0 + offset_src0);
+ global float * y = (global float *) (src1 + offset_src1);
+
+ float yl[32];
+ float sumf[N_DST] = {0.f};
+ float all_sum;
+
+ global float * y4 = y + ix * QK_K + 32 * it;
+
+ for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
+ for (int i = 0; i < 32; ++i) {
+ yl[i] = y4[i];
+ }
+
+ global char * xrow = (global char *)(x + ib);
+
+ // only ne01 output rows exist; reading past them can pull in an inf scale
+ for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
+ global block_iq2_s * xb = (global block_iq2_s *)(xrow + row*nb01);
+
+ uchar scb = xb->scales[it];
+ float dlo = (float)xb->d * (0.5f + (float)(scb & 0xf)) * 0.25f;
+ float dhi = (float)xb->d * (0.5f + (float)(scb >> 4)) * 0.25f;
+
+ uchar qhb = xb->qh[it];
+ global uchar * qsb = xb->qs + 4*it;
+ global uchar * sgb = xb->qs + 32 + 4*it;
+
+ float alo = 0.f, ahi = 0.f;
+ for (int l = 0; l < 4; ++l) {
+ uint gi = (uint)qsb[l] | (((uint)qhb << (8-2*l)) & 0x300);
+ uchar sg = sgb[l];
+ uint lo = iq2_s_grid[2*gi+0];
+ uint hi = iq2_s_grid[2*gi+1];
+ float a = 0.f;
+ for (int j = 0; j < 4; ++j) {
+ a += yl[8*l+j+0] * (float)((lo >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f);
+ a += yl[8*l+j+4] * (float)((hi >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f);
+ }
+ if (l < 2) alo += a; else ahi += a;
+ }
+
+ sumf[row] += dlo * alo + dhi * ahi;
+ }
+
+ y4 += BLOCK_STRIDE * QK_K;
+ }
+
+ global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = sub_group_reduce_add(sumf[row]);
+ if (first_row + row < ne01) {
+ if (get_sub_group_local_id() == 0) {
+ dst_f32[first_row + row] = all_sum;
+ }
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq2_s_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_s_f32_flat.cl
new file mode 100644
index 000000000..94d3ffd6f
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_s_f32_flat.cl
@@ -0,0 +1,148 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_required_subgroup_size
+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
+#define INTEL_GPU 1
+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
+#elif defined(cl_qcom_reqd_sub_group_size)
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+
+#define QK_K 256
+
+// qs holds 32 grid low bytes then 32 sign bytes
+typedef struct {
+ half d;
+ uchar qs[QK_K/4];
+ uchar qh[QK_K/32];
+ uchar scales[QK_K/32];
+} block_iq2_s;
+
+#undef N_DST
+#undef N_SIMDGROUP
+#undef N_SIMDWIDTH
+
+#ifdef INTEL_GPU
+#define N_DST 4
+#define N_SIMDGROUP 1
+#define N_SIMDWIDTH 16
+#elif defined (ADRENO_GPU)
+#define N_DST 4
+#define N_SIMDGROUP 1
+#define N_SIMDWIDTH 64
+#endif
+
+#undef BLOCK_STRIDE
+// 8 threads cover one super block, one 32 element sub block each
+#define BLOCK_STRIDE (N_SIMDWIDTH/8)
+
+#ifdef INTEL_GPU
+REQD_SUBGROUP_SIZE_16
+#elif defined (ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mv_iq2_s_f32_flat(
+ global uchar * src0_qs,
+ global uchar * src0_sg,
+ global uchar * src0_qh,
+ global uchar * src0_sc,
+ global half * src0_d,
+ ulong offset0,
+ global char * src1,
+ ulong offset1,
+ global char * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne12,
+ ulong nb11,
+ ulong nb12,
+ ulong nb13,
+ int ne0,
+ int ne1,
+ int r2,
+ int r3
+) {
+ src1 = src1 + offset1;
+ dst = dst + offsetd;
+
+ int ix = get_sub_group_local_id()/8; // super block index
+ int it = get_sub_group_local_id()%8; // sub block inside the super block
+
+ int nb = ne00/QK_K;
+
+ int r0 = get_group_id(0);
+ int r1 = get_group_id(1);
+ int im = get_group_id(2);
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
+
+ int i12 = im%ne12;
+ int i13 = im/ne12;
+
+ ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq2_s);
+ ulong row_stride = nb01/sizeof(block_iq2_s);
+ ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;
+
+ global float * y = (global float *) (src1 + offset_src1);
+
+ float yl[32];
+ float sumf[N_DST] = {0.f};
+ float all_sum;
+
+ global float * y4 = y + ix * QK_K + 32 * it;
+
+ for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
+ for (int i = 0; i < 32; ++i) {
+ yl[i] = y4[i];
+ }
+
+ // only ne01 output rows exist; reading past them can pull in an inf scale
+ for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
+ const ulong b = offset_src0 + ib + row*row_stride;
+
+ uchar scb = src0_sc[8*b + it];
+ float dlo = (float)src0_d[b] * (0.5f + (float)(scb & 0xf)) * 0.25f;
+ float dhi = (float)src0_d[b] * (0.5f + (float)(scb >> 4)) * 0.25f;
+
+ uchar qhb = src0_qh[8*b + it];
+ global uchar * qsb = src0_qs + 32*b + 4*it;
+ global uchar * sgb = src0_sg + 32*b + 4*it;
+
+ float alo = 0.f, ahi = 0.f;
+ for (int l = 0; l < 4; ++l) {
+ uint gi = (uint)qsb[l] | (((uint)qhb << (8-2*l)) & 0x300);
+ uchar sg = sgb[l];
+ uint lo = iq2_s_grid[2*gi+0];
+ uint hi = iq2_s_grid[2*gi+1];
+ float a = 0.f;
+ for (int j = 0; j < 4; ++j) {
+ a += yl[8*l+j+0] * (float)((lo >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f);
+ a += yl[8*l+j+4] * (float)((hi >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f);
+ }
+ if (l < 2) alo += a; else ahi += a;
+ }
+
+ sumf[row] += dlo * alo + dhi * ahi;
+ }
+
+ y4 += BLOCK_STRIDE * QK_K;
+ }
+
+ global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = sub_group_reduce_add(sumf[row]);
+ if (first_row + row < ne01) {
+ if (get_sub_group_local_id() == 0) {
+ dst_f32[first_row + row] = all_sum;
+ }
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xs_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xs_f32.cl
new file mode 100644
index 000000000..264e8e8e2
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xs_f32.cl
@@ -0,0 +1,143 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_required_subgroup_size
+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
+#define INTEL_GPU 1
+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
+#elif defined(cl_qcom_reqd_sub_group_size)
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+
+#define QK_K 256
+
+typedef struct {
+ half d;
+ ushort qs[QK_K/8];
+ uchar scales[QK_K/32];
+} block_iq2_xs;
+
+#undef N_DST
+#undef N_SIMDGROUP
+#undef N_SIMDWIDTH
+
+#ifdef INTEL_GPU
+#define N_DST 4
+#define N_SIMDGROUP 1
+#define N_SIMDWIDTH 16
+#elif defined (ADRENO_GPU)
+#define N_DST 4
+#define N_SIMDGROUP 1
+#define N_SIMDWIDTH 64
+#endif
+
+#undef BLOCK_STRIDE
+// 8 threads cover one super block, one 32 element sub block each
+#define BLOCK_STRIDE (N_SIMDWIDTH/8)
+
+#ifdef INTEL_GPU
+REQD_SUBGROUP_SIZE_16
+#elif defined (ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mv_iq2_xs_f32(
+ global char * src0,
+ ulong offset0,
+ global char * src1,
+ ulong offset1,
+ global char * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne12,
+ ulong nb11,
+ ulong nb12,
+ ulong nb13,
+ int ne0,
+ int ne1,
+ int r2,
+ int r3
+) {
+ src0 = src0 + offset0;
+ src1 = src1 + offset1;
+ dst = dst + offsetd;
+
+ int ix = get_sub_group_local_id()/8; // super block index
+ int it = get_sub_group_local_id()%8; // sub block inside the super block
+
+ int nb = ne00/QK_K;
+
+ int r0 = get_group_id(0);
+ int r1 = get_group_id(1);
+ int im = get_group_id(2);
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
+
+ int i12 = im%ne12;
+ int i13 = im/ne12;
+
+ ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
+ ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;
+
+ global block_iq2_xs * x = (global block_iq2_xs *) (src0 + offset_src0);
+ global float * y = (global float *) (src1 + offset_src1);
+
+ float yl[32];
+ float sumf[N_DST] = {0.f};
+ float all_sum;
+
+ global float * y4 = y + ix * QK_K + 32 * it;
+
+ for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
+ for (int i = 0; i < 32; ++i) {
+ yl[i] = y4[i];
+ }
+
+ global char * xrow = (global char *)(x + ib);
+
+ // only ne01 output rows exist; reading past them can pull in an inf scale
+ for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
+ global block_iq2_xs * xb = (global block_iq2_xs *)(xrow + row*nb01);
+
+ uchar scb = xb->scales[it];
+ float dlo = (float)xb->d * (0.5f + (float)(scb & 0xf)) * 0.25f;
+ float dhi = (float)xb->d * (0.5f + (float)(scb >> 4)) * 0.25f;
+
+ global ushort * q16 = xb->qs + 4*it;
+
+ float alo = 0.f, ahi = 0.f;
+ for (int l = 0; l < 4; ++l) {
+ ushort q = q16[l];
+ uchar sg = iq2_xs_signs(q >> 9);
+ uint lo = iq2_xs_grid[2*(q & 511)+0];
+ uint hi = iq2_xs_grid[2*(q & 511)+1];
+ float a = 0.f;
+ for (int j = 0; j < 4; ++j) {
+ a += yl[8*l+j+0] * (float)((lo >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f);
+ a += yl[8*l+j+4] * (float)((hi >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f);
+ }
+ if (l < 2) alo += a; else ahi += a;
+ }
+
+ sumf[row] += dlo * alo + dhi * ahi;
+ }
+
+ y4 += BLOCK_STRIDE * QK_K;
+ }
+
+ global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = sub_group_reduce_add(sumf[row]);
+ if (first_row + row < ne01) {
+ if (get_sub_group_local_id() == 0) {
+ dst_f32[first_row + row] = all_sum;
+ }
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xs_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xs_f32_flat.cl
new file mode 100644
index 000000000..a699a0e9f
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xs_f32_flat.cl
@@ -0,0 +1,142 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_required_subgroup_size
+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
+#define INTEL_GPU 1
+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
+#elif defined(cl_qcom_reqd_sub_group_size)
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+
+#define QK_K 256
+
+typedef struct {
+ half d;
+ ushort qs[QK_K/8];
+ uchar scales[QK_K/32];
+} block_iq2_xs;
+
+#undef N_DST
+#undef N_SIMDGROUP
+#undef N_SIMDWIDTH
+
+#ifdef INTEL_GPU
+#define N_DST 4
+#define N_SIMDGROUP 1
+#define N_SIMDWIDTH 16
+#elif defined (ADRENO_GPU)
+#define N_DST 4
+#define N_SIMDGROUP 1
+#define N_SIMDWIDTH 64
+#endif
+
+#undef BLOCK_STRIDE
+// 8 threads cover one super block, one 32 element sub block each
+#define BLOCK_STRIDE (N_SIMDWIDTH/8)
+
+#ifdef INTEL_GPU
+REQD_SUBGROUP_SIZE_16
+#elif defined (ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mv_iq2_xs_f32_flat(
+ global ushort * src0_qs,
+ global uchar * src0_sc,
+ global half * src0_d,
+ ulong offset0,
+ global char * src1,
+ ulong offset1,
+ global char * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne12,
+ ulong nb11,
+ ulong nb12,
+ ulong nb13,
+ int ne0,
+ int ne1,
+ int r2,
+ int r3
+) {
+ src1 = src1 + offset1;
+ dst = dst + offsetd;
+
+ int ix = get_sub_group_local_id()/8; // super block index
+ int it = get_sub_group_local_id()%8; // sub block inside the super block
+
+ int nb = ne00/QK_K;
+
+ int r0 = get_group_id(0);
+ int r1 = get_group_id(1);
+ int im = get_group_id(2);
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
+
+ int i12 = im%ne12;
+ int i13 = im/ne12;
+
+ ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq2_xs);
+ ulong row_stride = nb01/sizeof(block_iq2_xs);
+ ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;
+
+ global float * y = (global float *) (src1 + offset_src1);
+
+ float yl[32];
+ float sumf[N_DST] = {0.f};
+ float all_sum;
+
+ global float * y4 = y + ix * QK_K + 32 * it;
+
+ for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
+ for (int i = 0; i < 32; ++i) {
+ yl[i] = y4[i];
+ }
+
+ // only ne01 output rows exist; reading past them can pull in an inf scale
+ for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
+ const ulong b = offset_src0 + ib + row*row_stride;
+
+ uchar scb = src0_sc[8*b + it];
+ float dlo = (float)src0_d[b] * (0.5f + (float)(scb & 0xf)) * 0.25f;
+ float dhi = (float)src0_d[b] * (0.5f + (float)(scb >> 4)) * 0.25f;
+
+ global ushort * q16 = src0_qs + 32*b + 4*it;
+
+ float alo = 0.f, ahi = 0.f;
+ for (int l = 0; l < 4; ++l) {
+ ushort q = q16[l];
+ uchar sg = iq2_xs_signs(q >> 9);
+ uint lo = iq2_xs_grid[2*(q & 511)+0];
+ uint hi = iq2_xs_grid[2*(q & 511)+1];
+ float a = 0.f;
+ for (int j = 0; j < 4; ++j) {
+ a += yl[8*l+j+0] * (float)((lo >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f);
+ a += yl[8*l+j+4] * (float)((hi >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f);
+ }
+ if (l < 2) alo += a; else ahi += a;
+ }
+
+ sumf[row] += dlo * alo + dhi * ahi;
+ }
+
+ y4 += BLOCK_STRIDE * QK_K;
+ }
+
+ global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = sub_group_reduce_add(sumf[row]);
+ if (first_row + row < ne01) {
+ if (get_sub_group_local_id() == 0) {
+ dst_f32[first_row + row] = all_sum;
+ }
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xxs_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xxs_f32.cl
new file mode 100644
index 000000000..7d4595086
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xxs_f32.cl
@@ -0,0 +1,140 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_required_subgroup_size
+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
+#define INTEL_GPU 1
+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
+#elif defined(cl_qcom_reqd_sub_group_size)
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+
+#define QK_K 256
+
+typedef struct {
+ half d;
+ ushort qs[QK_K/8];
+} block_iq2_xxs;
+
+#undef N_DST
+#undef N_SIMDGROUP
+#undef N_SIMDWIDTH
+
+#ifdef INTEL_GPU
+#define N_DST 4
+#define N_SIMDGROUP 1
+#define N_SIMDWIDTH 16
+#elif defined (ADRENO_GPU)
+#define N_DST 4
+#define N_SIMDGROUP 1
+#define N_SIMDWIDTH 64
+#endif
+
+#undef BLOCK_STRIDE
+// 8 threads cover one super block, one 32 element sub block each
+#define BLOCK_STRIDE (N_SIMDWIDTH/8)
+
+#ifdef INTEL_GPU
+REQD_SUBGROUP_SIZE_16
+#elif defined (ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mv_iq2_xxs_f32(
+ global char * src0,
+ ulong offset0,
+ global char * src1,
+ ulong offset1,
+ global char * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne12,
+ ulong nb11,
+ ulong nb12,
+ ulong nb13,
+ int ne0,
+ int ne1,
+ int r2,
+ int r3
+) {
+ src0 = src0 + offset0;
+ src1 = src1 + offset1;
+ dst = dst + offsetd;
+
+ int ix = get_sub_group_local_id()/8; // super block index
+ int it = get_sub_group_local_id()%8; // sub block inside the super block
+
+ int nb = ne00/QK_K;
+
+ int r0 = get_group_id(0);
+ int r1 = get_group_id(1);
+ int im = get_group_id(2);
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
+
+ int i12 = im%ne12;
+ int i13 = im/ne12;
+
+ ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
+ ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;
+
+ global block_iq2_xxs * x = (global block_iq2_xxs *) (src0 + offset_src0);
+ global float * y = (global float *) (src1 + offset_src1);
+
+ float yl[32];
+ float sumf[N_DST] = {0.f};
+ float all_sum;
+
+ global float * y4 = y + ix * QK_K + 32 * it;
+
+ for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
+ for (int i = 0; i < 32; ++i) {
+ yl[i] = y4[i];
+ }
+
+ global char * xrow = (global char *)(x + ib);
+
+ // only ne01 output rows exist; reading past them can pull in an inf scale
+ for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
+ global block_iq2_xxs * xb = (global block_iq2_xxs *)(xrow + row*nb01);
+
+ global ushort * q16 = xb->qs + 4*it;
+ uint a0 = (uint)q16[0] | ((uint)q16[1] << 16);
+ uint a1 = (uint)q16[2] | ((uint)q16[3] << 16);
+
+ float db = (float)xb->d * (0.5f + (float)(a1 >> 28)) * 0.25f;
+
+ float acc = 0.f;
+ for (int l = 0; l < 4; ++l) {
+ uint gi = (a0 >> (8*l)) & 0xFF;
+ uchar sg = iq2_xxs_signs((a1 >> (7*l)) & 127);
+ uint lo = iq2_xxs_grid[2*gi+0];
+ uint hi = iq2_xxs_grid[2*gi+1];
+ for (int j = 0; j < 4; ++j) {
+ acc += yl[8*l+j+0] * (float)((lo >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f);
+ acc += yl[8*l+j+4] * (float)((hi >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f);
+ }
+ }
+
+ sumf[row] += db * acc;
+ }
+
+ y4 += BLOCK_STRIDE * QK_K;
+ }
+
+ global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = sub_group_reduce_add(sumf[row]);
+ if (first_row + row < ne01) {
+ if (get_sub_group_local_id() == 0) {
+ dst_f32[first_row + row] = all_sum;
+ }
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xxs_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xxs_f32_flat.cl
new file mode 100644
index 000000000..4d9b98416
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq2_xxs_f32_flat.cl
@@ -0,0 +1,137 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_required_subgroup_size
+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
+#define INTEL_GPU 1
+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
+#elif defined(cl_qcom_reqd_sub_group_size)
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+
+#define QK_K 256
+
+typedef struct {
+ half d;
+ ushort qs[QK_K/8];
+} block_iq2_xxs;
+
+#undef N_DST
+#undef N_SIMDGROUP
+#undef N_SIMDWIDTH
+
+#ifdef INTEL_GPU
+#define N_DST 4
+#define N_SIMDGROUP 1
+#define N_SIMDWIDTH 16
+#elif defined (ADRENO_GPU)
+#define N_DST 4
+#define N_SIMDGROUP 1
+#define N_SIMDWIDTH 64
+#endif
+
+#undef BLOCK_STRIDE
+// 8 threads cover one super block, one 32 element sub block each
+#define BLOCK_STRIDE (N_SIMDWIDTH/8)
+
+#ifdef INTEL_GPU
+REQD_SUBGROUP_SIZE_16
+#elif defined (ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mv_iq2_xxs_f32_flat(
+ global uchar * src0_qs,
+ global uint * src0_sas,
+ global half * src0_d,
+ ulong offset0,
+ global char * src1,
+ ulong offset1,
+ global char * dst,
+ ulong offsetd,
+ int ne00,
+ int ne01,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne12,
+ ulong nb11,
+ ulong nb12,
+ ulong nb13,
+ int ne0,
+ int ne1,
+ int r2,
+ int r3
+) {
+ src1 = src1 + offset1;
+ dst = dst + offsetd;
+
+ int ix = get_sub_group_local_id()/8; // super block index
+ int it = get_sub_group_local_id()%8; // sub block inside the super block
+
+ int nb = ne00/QK_K;
+
+ int r0 = get_group_id(0);
+ int r1 = get_group_id(1);
+ int im = get_group_id(2);
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
+
+ int i12 = im%ne12;
+ int i13 = im/ne12;
+
+ ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq2_xxs);
+ ulong row_stride = nb01/sizeof(block_iq2_xxs);
+ ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;
+
+ global float * y = (global float *) (src1 + offset_src1);
+
+ float yl[32];
+ float sumf[N_DST] = {0.f};
+ float all_sum;
+
+ global float * y4 = y + ix * QK_K + 32 * it;
+
+ for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
+ for (int i = 0; i < 32; ++i) {
+ yl[i] = y4[i];
+ }
+
+ // only ne01 output rows exist; reading past them can pull in an inf scale
+ for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
+ const ulong b = offset_src0 + ib + row*row_stride;
+ uint a0 = as_uint(vload4(8*b + it, src0_qs));
+ uint a1 = src0_sas[8*b + it];
+
+ float db = (float)src0_d[b] * (0.5f + (float)(a1 >> 28)) * 0.25f;
+
+ float acc = 0.f;
+ for (int l = 0; l < 4; ++l) {
+ uint gi = (a0 >> (8*l)) & 0xFF;
+ uchar sg = iq2_xxs_signs((a1 >> (7*l)) & 127);
+ uint lo = iq2_xxs_grid[2*gi+0];
+ uint hi = iq2_xxs_grid[2*gi+1];
+ for (int j = 0; j < 4; ++j) {
+ acc += yl[8*l+j+0] * (float)((lo >> (8*j)) & 0xFF) * ((sg & (1 << j)) ? -1.f : 1.f);
+ acc += yl[8*l+j+4] * (float)((hi >> (8*j)) & 0xFF) * ((sg & (1 << (j+4))) ? -1.f : 1.f);
+ }
+ }
+
+ sumf[row] += db * acc;
+ }
+
+ y4 += BLOCK_STRIDE * QK_K;
+ }
+
+ global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
+
+ for (int row = 0; row < N_DST; ++row) {
+ all_sum = sub_group_reduce_add(sumf[row]);
+ if (first_row + row < ne01) {
+ if (get_sub_group_local_id() == 0) {
+ dst_f32[first_row + row] = all_sum;
+ }
+ }
+ }
+}