Commit 45451c366 for llama.cpp
commit 45451c366d4256f94fbc8845645e31a6b9155054
Author: Charles Xu <charles.xu@arm.com>
Date: Tue Oct 13 18:37:48 2026 +0200
kleidiai: add SME2 Q4_K and Q6_K matmul support (#29219)
Change-Id: Ib4aa4c9814c010f9cf084783cc80612e364f9a27
Change-Id: I406a6606c3fea51f8cd32331c1ed71466845d363
Change-Id: Ib4aa4c9814c010f9cf084783cc80612e364f9a27
Signed-off-by: Charles Xu <charles.xu@arm.com>
diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt
index 50341fd6a..d347757e1 100644
--- a/ggml/src/ggml-cpu/CMakeLists.txt
+++ b/ggml/src/ggml-cpu/CMakeLists.txt
@@ -650,9 +650,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
# Fetch KleidiAI sources:
include(FetchContent)
- set(KLEIDIAI_COMMIT_TAG "v1.24.0")
+ set(KLEIDIAI_COMMIT_TAG "v1.31.0")
set(KLEIDIAI_DOWNLOAD_URL "https://github.com/ARM-software/kleidiai/releases/download/${KLEIDIAI_COMMIT_TAG}/kleidiai-${KLEIDIAI_COMMIT_TAG}-src.tar.gz")
- set(KLEIDIAI_RELEASE_ARCHIVE_MD5 "2f02ebe29573d45813e671eb304f2a00")
+ set(KLEIDIAI_RELEASE_ARCHIVE_MD5 "930495f3aeb16b25dc18a2bde45924f4")
set(KLEIDIAI_FETCH_ARGS
URL ${KLEIDIAI_DOWNLOAD_URL}
diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.cpp b/ggml/src/ggml-cpu/kleidiai/kernels.cpp
index d4551298f..89e95968c 100644
--- a/ggml/src/ggml-cpu/kleidiai/kernels.cpp
+++ b/ggml/src/ggml-cpu/kleidiai/kernels.cpp
@@ -3,6 +3,8 @@
//
// KleidiAI micro-kernels
+#include <cstddef>
+
#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p_qsi4c32p_interface.h"
#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp_qsi8cxp_interface.h"
#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.h"
@@ -33,6 +35,10 @@
#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32_neon.h"
#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qai8dxp_f32.h"
+#include "kai/ukernels/matmul/kai_matmul.h"
+#include "kai/ukernels/matmul/kai_matmul_pack_lhs.h"
+#include "kai/ukernels/matmul/kai_matmul_pack_rhs.h"
+
#include "kai/ukernels/matmul/pack/kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.h"
#include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme.h"
#include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0.h"
@@ -42,8 +48,6 @@
#include "kai/kai_common.h"
-#include "simd-mappings.h"
-
#define GGML_COMMON_DECL_CPP
#include "ggml-common.h"
@@ -154,16 +158,6 @@ static inline size_t rhs_ps_fn2(size_t n, size_t k, size_t /*nr*/, size_t /*kr*/
return Fn(n, k);
}
-template<size_t(*Fn)(size_t,size_t,size_t,size_t)>
-static inline size_t rhs_stride_fn4(size_t k, size_t nr, size_t kr, size_t bl) {
- return Fn(k, nr, kr, bl);
-}
-
-template<size_t(*Fn)(size_t)>
-static inline size_t rhs_stride_fn1(size_t k, size_t /*nr*/, size_t /*kr*/, size_t /*bl*/) {
- return Fn(k);
-}
-
template<void(*Fn)(size_t,size_t,size_t,size_t,size_t,size_t,size_t,const uint8_t*,const float*,void*,size_t,const struct kai_rhs_pack_qs4cxs1s0_param*)>
static inline void rhs_pack_fn12(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl,
size_t /*rhs_stride*/, const void* rhs, const void* bias, const void* /*scale*/,
@@ -194,137 +188,205 @@ static inline void rhs_pack_fn13(size_t num_groups, size_t n, size_t k, size_t n
Fn(num_groups, n, k, nr, kr, sr, rhs_stride, rhs, bias, scale, rhs_packed, extra_bytes, params);
}
-static const size_t INT4_PER_BYTE = 2;
-static const size_t INT4_BITS = 4;
-static const int Q4_0_ZERO_POINT = 8;
-const size_t INT4_PER_UINT16 = 4;
-
-static void dequantize_row_qsi4c32pscalef16(
- const void *packed_data,
- int32_t row_idx,
- int64_t nc,
- float *out,
- size_t nr_pack,
- size_t packed_row_stride,
- size_t kr,
- size_t bl,
- size_t num_bytes_multiplier
-) {
- size_t group_idx = row_idx / nr_pack;
- size_t row_in_group = row_idx % nr_pack;
- const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride;
- size_t num_blocks = nc / bl;
- const uint8_t *block_ptr = packed_group;
-
- for (size_t b = 0; b < num_blocks; ++b) {
- uint16_t scale_f16 = *((const uint16_t *)(block_ptr + row_in_group * num_bytes_multiplier));
- float scale = GGML_CPU_FP16_TO_FP32(scale_f16);
-
- const uint8_t *segment_ptr = block_ptr + nr_pack * num_bytes_multiplier;
- size_t num_segments = bl / kr;
- size_t num_bytes_per_segment = kr / INT4_PER_BYTE;
-
- for (size_t s = 0; s < num_segments; ++s) {
- const uint8_t *seg_base = segment_ptr + s * nr_pack * num_bytes_per_segment;
- const uint8_t *qbytes = seg_base + row_in_group * num_bytes_per_segment;
- for (size_t k = 0; k < num_bytes_per_segment; ++k) {
- uint8_t byte = qbytes[k] ^ 0x88;
- int x0 = (byte & 0x0F) - Q4_0_ZERO_POINT;
- int x1 = (byte >> INT4_BITS) - Q4_0_ZERO_POINT;
- out[b * bl + s * num_bytes_per_segment + k] = x0 * scale;
- out[b * bl + s * num_bytes_per_segment + k + bl/2] = x1 * scale;
- }
- }
- block_ptr += nr_pack * num_bytes_multiplier + num_segments * nr_pack * num_bytes_per_segment;
- }
+static constexpr size_t KLEIDIAI_Q4_K_BLOCK_LENGTH = QK_K / 8;
+
+static_assert(QK_K == 256, "KleidiAI Q4_K kernels require 256-value super-blocks");
+static_assert(sizeof(block_q4_K) == 144, "Unexpected ggml Q4_K block layout");
+static_assert(offsetof(block_q4_K, scales) == 4, "Unexpected ggml Q4_K metadata offset");
+static_assert(offsetof(block_q4_K, qs) == 16, "Unexpected ggml Q4_K data offset");
+
+static struct kai_matmul_uker_config q4_k_matmul_config() {
+ struct kai_matmul_uker_config config = {};
+ config.format.bl = KLEIDIAI_Q4_K_BLOCK_LENGTH;
+ return config;
}
-static void dequantize_row_qsi4c32ps1s0scalef16(
- const void *packed_data,
- int32_t row_idx,
- int64_t k,
- float *out,
- size_t nr,
- size_t packed_row_stride,
- size_t kr,
- size_t bl,
- size_t num_bytes_multiplier
-) {
- const size_t num_blocks = k / bl;
- const size_t bl4 = bl / INT4_PER_UINT16;
-
- size_t group_idx = row_idx / nr;
- size_t row_in_group = row_idx % nr;
-
- const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride;
- const uint16_t *qdata = (const uint16_t *)packed_group;
- const uint16_t *scales = (const uint16_t *)(packed_group + packed_row_stride - (nr * num_blocks * num_bytes_multiplier));
-
- for (size_t block_idx = 0; block_idx < num_blocks; ++block_idx) {
- uint16_t scale_f16 = scales[row_in_group + block_idx * nr];
- float scale = GGML_CPU_FP16_TO_FP32(scale_f16);
-
- for (size_t bl4_idx = 0; bl4_idx < bl4; ++bl4_idx) {
- uint16_t q = qdata[(block_idx * bl4 + bl4_idx) * nr + row_in_group];
-
- for (size_t qidx = 0; qidx < INT4_PER_UINT16; ++qidx) {
- int v = ((q >> (qidx * 4)) & 0xF) - Q4_0_ZERO_POINT;
- out[block_idx * bl + bl4_idx * INT4_BITS + qidx] = v * scale;
- }
- }
- }
- GGML_UNUSED(kr);
+template<struct kai_matmul_uker_api (*GetApi)(void)>
+static size_t q4_k_get_m_step() {
+ const struct kai_matmul_uker_api api = GetApi();
+ const struct kai_matmul_uker_config config = q4_k_matmul_config();
+ return api.get_step(&config).m;
}
-static void dequantize_row_qsi8cxp(
- const void *packed_data,
- int32_t row_idx,
- int64_t k,
- float *out,
- size_t nr,
- size_t packed_row_stride,
- size_t kr,
- size_t bl,
- size_t num_bytes_multiplier
-) {
- GGML_UNUSED(bl);
- GGML_UNUSED(num_bytes_multiplier);
-
- const size_t k_internal = ((size_t) k + QK8_0 - 1) / QK8_0 * QK8_0;
- const size_t group_idx = row_idx / nr;
- const size_t row_in_group = row_idx % nr;
-
- const uint8_t * group_ptr = static_cast<const uint8_t *>(packed_data) + group_idx * packed_row_stride;
- const int8_t * data_base = reinterpret_cast<const int8_t *>(group_ptr);
-
- const size_t num_blocks = k_internal / kr;
-
- for (size_t block = 0; block < num_blocks; ++block) {
- const int8_t * block_ptr = data_base + (block * nr + row_in_group) * kr;
- for (size_t i = 0; i < kr; ++i) {
- const size_t k_idx = block * kr + i;
- if (k_idx < (size_t) k) {
- out[k_idx] = static_cast<float>(block_ptr[i]);
- }
- }
- }
+template<struct kai_matmul_uker_api (*GetApi)(void)>
+static size_t q4_k_get_n_step() {
+ const struct kai_matmul_uker_api api = GetApi();
+ const struct kai_matmul_uker_config config = q4_k_matmul_config();
+ return api.get_step(&config).n;
+}
- const uint8_t * sums_ptr = group_ptr + nr * k_internal;
- GGML_UNUSED(sums_ptr);
+template<size_t Value>
+static size_t q4_k_get_constant() {
+ return Value;
+}
- const float * scale_ptr = reinterpret_cast<const float *>(sums_ptr + nr * sizeof(int32_t));
- const float scale = scale_ptr[row_in_group];
+template<struct kai_matmul_uker_api (*GetApi)(void)>
+static size_t q4_k_get_dst_offset(size_t m_idx, size_t n_idx, size_t dst_stride) {
+ const struct kai_matmul_uker_api api = GetApi();
+ const struct kai_matmul_uker_config config = q4_k_matmul_config();
+ const struct kai_matmul_uker_dst_dim_args index = { m_idx, n_idx };
+ const struct kai_matmul_uker_dst_stride_args stride = { dst_stride };
+ return api.get_dst_offset(&config, &index, &stride);
+}
- if (scale == 0.0f) {
- for (size_t i = 0; i < (size_t) k; ++i) {
- out[i] = 0.0f;
- }
- return;
- }
+template<struct kai_matmul_uker_api (*GetApi)(void)>
+static size_t q4_k_get_lhs_packed_offset(size_t m_idx, size_t k, size_t bl) {
+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
+
+ const struct kai_matmul_uker_api api = GetApi();
+ const struct kai_matmul_uker_config config = q4_k_matmul_config();
+ const struct kai_matmul_uker_lhs_dim_args shape = { m_idx, k };
+ const struct kai_matmul_uker_lhs_dim_args index = { m_idx, 0 };
+ const struct kai_matmul_uker_lhs_stride_args stride = api.get_lhs_stride(&config, &shape);
+ return api.get_lhs_offset(&config, &index, &stride);
+}
+
+template<struct kai_matmul_uker_api (*GetApi)(void)>
+static size_t q4_k_get_rhs_packed_offset(size_t n_idx, size_t k, size_t bl) {
+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
+
+ const struct kai_matmul_uker_api api = GetApi();
+ const struct kai_matmul_uker_config config = q4_k_matmul_config();
+ const struct kai_matmul_uker_rhs_dim_args shape = { n_idx, k };
+ const struct kai_matmul_uker_rhs_dim_args index = { n_idx, 0 };
+ const struct kai_matmul_uker_rhs_stride_args stride = api.get_rhs_stride(&config, &shape);
+ return api.get_rhs_offset(&config, &index, &stride);
+}
+
+template<struct kai_matmul_uker_api (*GetApi)(void)>
+static void q4_k_run_matmul(
+ size_t m, size_t n, size_t k, size_t bl,
+ const void * lhs_packed, const void * rhs_packed,
+ void * dst, size_t dst_stride_row, size_t dst_stride_col,
+ float clamp_min, float clamp_max) {
+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
+ GGML_ASSERT(dst_stride_col == sizeof(float));
+
+ const struct kai_matmul_uker_api api = GetApi();
+ const struct kai_matmul_uker_config config = q4_k_matmul_config();
+ const struct kai_matmul_uker_lhs_dim_args lhs_shape = { m, k };
+ const struct kai_matmul_uker_rhs_dim_args rhs_shape = { n, k };
+
+ struct kai_matmul_uker_args args = {};
+ args.flags = KAI_MATMUL_UKER_FLAGS_ARGS_CLAMP;
+ args.shape = { m, n, k };
+ args.operand.lhs.ptr = lhs_packed;
+ args.operand.lhs.stride = api.get_lhs_stride(&config, &lhs_shape);
+ args.operand.rhs.ptr = rhs_packed;
+ args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape);
+ args.operand.dst.ptr = dst;
+ args.operand.dst.stride.m = dst_stride_row;
+ args.activation.clamp.min_ptr = &clamp_min;
+ args.activation.clamp.max_ptr = &clamp_max;
+ api.run(&config, &args);
+}
+
+static struct kai_matmul_pack_lhs_uker_config q4_k_lhs_pack_config() {
+ struct kai_matmul_pack_lhs_uker_config config = {};
+ return config;
+}
+
+static size_t q4_k_lhs_get_offset(size_t m_idx, size_t lhs_stride) {
+ const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon();
+ const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config();
+ const struct kai_matmul_pack_lhs_uker_lhs_dim_args index = { m_idx, 0 };
+ const struct kai_matmul_pack_lhs_uker_lhs_stride_args stride = { lhs_stride };
+ return api.get_lhs_offset(&config, &index, &stride);
+}
+
+static size_t q4_k_lhs_get_packed_offset(
+ size_t m_idx, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) {
+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
+ GGML_ASSERT(mr == 1 && kr == 4 && sr == 2);
+
+ const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon();
+ const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config();
+ const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args shape = { m_idx, k };
+ const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args index = { m_idx, 0 };
+ const struct kai_matmul_pack_lhs_uker_lhs_packed_stride_args stride = api.get_lhs_packed_stride(&config, &shape);
+ return api.get_lhs_packed_offset(&config, &index, &stride);
+}
+
+static size_t q4_k_lhs_get_packed_size(
+ size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) {
+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
+ GGML_ASSERT(mr == 1 && kr == 4 && sr == 2);
+
+ const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon();
+ const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config();
+ const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args shape = { m, k };
+ const struct kai_matmul_pack_lhs_uker_lhs_packed_stride_args stride = api.get_lhs_packed_stride(&config, &shape);
+ return api.get_lhs_packed_size(&config, &shape, &stride);
+}
+
+static void q4_k_lhs_pack(
+ size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr,
+ size_t m_idx_start, const void * lhs, size_t lhs_stride, void * lhs_packed) {
+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
+ GGML_ASSERT(mr == 1 && kr == 4 && sr == 2);
+ GGML_ASSERT(m_idx_start == 0);
+
+ const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon();
+ const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config();
+ const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args packed_shape = { m, k };
+
+ struct kai_matmul_pack_lhs_uker_args args = {};
+ args.flags = 0;
+ args.shape = { m, k };
+ args.operand.lhs.ptr = lhs;
+ args.operand.lhs.stride.m = lhs_stride;
+ args.operand.lhs_packed.ptr = lhs_packed;
+ args.operand.lhs_packed.stride = api.get_lhs_packed_stride(&config, &packed_shape);
+ api.run(&config, &args);
+}
- for (size_t i = 0; i < (size_t) k; ++i) {
- out[i] *= scale;
+static struct kai_matmul_pack_rhs_uker_config q4_k_rhs_pack_config() {
+ struct kai_matmul_pack_rhs_uker_config config = {};
+ return config;
+}
+
+static size_t q4_k_rhs_get_packed_size(size_t n, size_t k, size_t nr, size_t /*kr*/, size_t bl) {
+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
+
+ const struct kai_matmul_pack_rhs_uker_api api =
+ kai_matmul_pack_rhs_nxk_qai4c32p16vsx4s1s0sf16_qai4c32k256sf16s32s0_sme();
+ const struct kai_matmul_pack_rhs_uker_config config = q4_k_rhs_pack_config();
+ GGML_ASSERT(api.get_step(&config).n == nr);
+ const struct kai_matmul_pack_rhs_uker_rhs_packed_dim_args shape = { n, k };
+ const struct kai_matmul_pack_rhs_uker_rhs_packed_stride_args stride = api.get_rhs_packed_stride(&config, &shape);
+ return api.get_rhs_packed_size(&config, &shape, &stride);
+}
+
+static void q4_k_rhs_pack(
+ size_t num_groups, size_t n, size_t k, size_t nr, size_t /*kr*/, size_t /*sr*/, size_t bl,
+ size_t rhs_stride, const void * rhs, const void * bias, const void * scale,
+ void * rhs_packed, size_t extra_bytes, const void * params) {
+ GGML_ASSERT(num_groups == 1);
+ GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
+ GGML_ASSERT(bias == nullptr);
+ GGML_ASSERT(scale == nullptr);
+ GGML_ASSERT(extra_bytes == 0);
+ GGML_ASSERT(params == nullptr);
+
+ const struct kai_matmul_pack_rhs_uker_api api =
+ kai_matmul_pack_rhs_nxk_qai4c32p16vsx4s1s0sf16_qai4c32k256sf16s32s0_sme();
+ const struct kai_matmul_pack_rhs_uker_config config = q4_k_rhs_pack_config();
+ GGML_ASSERT(api.get_step(&config).n == nr);
+ const struct kai_matmul_pack_rhs_uker_rhs_dim_args rhs_shape = { n, k };
+ const struct kai_matmul_pack_rhs_uker_rhs_packed_dim_args packed_shape = { n, k };
+
+ struct kai_matmul_pack_rhs_uker_args args = {};
+ args.flags = 0;
+ args.shape = { n, k };
+ args.operand.rhs.ptr = rhs;
+ args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape);
+ if (rhs_stride != 0) {
+ GGML_ASSERT(rhs_stride >= args.operand.rhs.stride.n);
+ args.operand.rhs.stride.n = rhs_stride;
}
+ args.operand.rhs_packed.ptr = rhs_packed;
+ args.operand.rhs_packed.stride = api.get_rhs_packed_stride(&config, &packed_shape);
+ api.run(&config, &args);
}
static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
@@ -338,7 +400,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa>,
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa>,
@@ -359,7 +420,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot>,
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot>,
@@ -371,10 +431,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32_neon>,
},
/* .rhs_info = */ {
- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon,
- /* .to_float = */ dequantize_row_qsi4c32ps1s0scalef16,
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,
- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,
/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,
},
/* .required_cpu = */ CPU_FEATURE_SME2 | CPU_FEATURE_FP16,
@@ -392,7 +449,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa>,
/* .run_kernel_ex = */ &kernel_run_fn10<kai_run_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa>,
@@ -412,7 +468,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
/* .get_lhs_offset_ex = */ nullptr,
/* .get_rhs_packed_offset_ex = */ nullptr,
/* .run_kernel_ex = */ nullptr,
@@ -424,10 +479,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .pack_func_ex = */ &lhs_pack_void_fn9<kai_run_lhs_pack_bf16p2vlx2_f32_sme>,
},
/* .rhs_info = */ {
- /* .packed_stride = */ nullptr,
- /* .to_float = */ nullptr,
/* .packed_size_ex = */ &rhs_ps_fn2<kai_get_rhs_packed_size_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>,
- /* .packed_stride_ex = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>,
/* .pack_func_ex = */ &rhs_pack_fn13<kai_run_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>,
},
/* .required_cpu = */ CPU_FEATURE_SME2,
@@ -446,7 +498,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,
@@ -466,7 +517,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,
@@ -478,10 +528,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,
},
/* .rhs_info = */ {
- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,
- /* .to_float = */ dequantize_row_qsi4c32pscalef16,
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
},
/* .required_cpu = */ CPU_FEATURE_DOTPROD,
@@ -499,7 +546,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,
@@ -519,7 +565,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,
@@ -531,10 +576,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,
},
/* .rhs_info = */ {
- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,
- /* .to_float = */ dequantize_row_qsi4c32pscalef16,
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
},
/* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,
@@ -553,7 +595,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm>,
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm>,
@@ -573,7 +614,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod>,
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod>,
@@ -585,10 +625,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,
},
/* .rhs_info = */ {
- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,
- /* .to_float = */ dequantize_row_qsi4c32pscalef16,
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
},
/* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,
@@ -606,7 +643,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,
@@ -626,7 +662,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,
@@ -638,10 +673,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,
},
/* .rhs_info = */ {
- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,
- /* .to_float = */ dequantize_row_qsi4c32pscalef16,
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
},
/* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,
@@ -659,7 +691,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,
@@ -679,7 +710,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,
@@ -691,10 +721,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,
},
/* .rhs_info = */ {
- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,
- /* .to_float = */ dequantize_row_qsi4c32pscalef16,
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
},
/* .required_cpu = */ CPU_FEATURE_DOTPROD,
@@ -706,6 +733,59 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
{ /* Sentinel */ }
};
+static ggml_kleidiai_kernels gemm_gemv_kernels_q4_k[] = {
+ {
+ /* SME2 GEMM */
+ {
+ /* .get_m_step = */ &q4_k_get_m_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
+ /* .get_n_step = */ &q4_k_get_n_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
+ /* .get_mr = */ &q4_k_get_m_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
+ /* .get_nr = */ &q4_k_get_n_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
+ /* .get_kr = */ &q4_k_get_constant<4>,
+ /* .get_sr = */ &q4_k_get_constant<2>,
+ /* .get_dst_offset = */ &q4_k_get_dst_offset<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
+ /* .get_lhs_offset_ex = */ &q4_k_get_lhs_packed_offset<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
+ /* .get_rhs_packed_offset_ex = */ &q4_k_get_rhs_packed_offset<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
+ /* .run_kernel_ex = */ &q4_k_run_matmul<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
+ },
+ {
+ /* .get_offset = */ kai_get_lhs_offset_lhs_pack_f16pmrx2_f32_neon,
+ /* .get_packed_offset_ex = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_pack_f16pmrx2_f32_neon>,
+ /* .packed_size_ex = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_pack_f16pmrx2_f32_neon>,
+ /* .pack_func_ex = */ &lhs_pack_void_fn10<kai_run_lhs_pack_f16pmrx2_f32_neon>,
+ },
+ /* SME2 GEMV */
+ {
+ /* .get_m_step = */ &q4_k_get_m_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
+ /* .get_n_step = */ &q4_k_get_n_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
+ /* .get_mr = */ &q4_k_get_m_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
+ /* .get_nr = */ &q4_k_get_n_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
+ /* .get_kr = */ &q4_k_get_constant<4>,
+ /* .get_sr = */ &q4_k_get_constant<2>,
+ /* .get_dst_offset = */ &q4_k_get_dst_offset<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
+ /* .get_lhs_offset_ex = */ &q4_k_get_lhs_packed_offset<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
+ /* .get_rhs_packed_offset_ex = */ &q4_k_get_rhs_packed_offset<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
+ /* .run_kernel_ex = */ &q4_k_run_matmul<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
+ },
+ {
+ /* .get_offset = */ q4_k_lhs_get_offset,
+ /* .get_packed_offset_ex = */ q4_k_lhs_get_packed_offset,
+ /* .packed_size_ex = */ q4_k_lhs_get_packed_size,
+ /* .pack_func_ex = */ q4_k_lhs_pack,
+ },
+ {
+ /* .packed_size_ex = */ q4_k_rhs_get_packed_size,
+ /* .pack_func_ex = */ q4_k_rhs_pack,
+ /* .repack_mode = */ RHS_REPACK_SINGLE_ONLY,
+ },
+ /* .required_cpu = */ CPU_FEATURE_SME2 | CPU_FEATURE_FP16,
+ /* .lhs_type = */ GGML_TYPE_F32,
+ /* .rhs_type = */ GGML_TYPE_Q4_K,
+ /* .op_type = */ GGML_TYPE_F32,
+ },
+ { /* Sentinel */ }
+};
+
static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
{
/* SME2 GEMM */
@@ -717,7 +797,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa>,
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa>,
@@ -737,7 +816,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot>,
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot>,
@@ -749,10 +827,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
/* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,
},
/* .rhs_info = */ {
- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,
- /* .to_float = */ dequantize_row_qsi8cxp,
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
/* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
},
/* .required_cpu = */ CPU_FEATURE_SME2,
@@ -770,7 +845,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa>,
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa>,
@@ -790,7 +864,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot>,
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot>,
@@ -802,10 +875,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
/* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,
},
/* .rhs_info = */ {
- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,
- /* .to_float = */ dequantize_row_qsi8cxp,
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
/* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
},
/* .required_cpu = */ CPU_FEATURE_SME,
@@ -823,7 +893,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm>,
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm>,
@@ -843,7 +912,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod>,
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod>,
@@ -855,10 +923,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
/* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,
},
/* .rhs_info = */ {
- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,
- /* .to_float = */ dequantize_row_qsi8cxp,
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
/* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
},
/* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,
@@ -876,7 +941,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod>,
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod>,
@@ -896,7 +960,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod>,
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod>,
@@ -908,10 +971,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
/* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,
},
/* .rhs_info = */ {
- /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,
- /* .to_float = */ dequantize_row_qsi8cxp,
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
- /* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
/* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
},
/* .required_cpu = */ CPU_FEATURE_DOTPROD,
@@ -933,7 +993,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa>,
/* .run_kernel_ex = */ &kernel_run_fn10<kai_run_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa>,
@@ -953,7 +1012,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>,
/* .run_kernel_ex = */ &kernel_run_lhs_stride_fn10<kai_run_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>,
@@ -965,10 +1023,7 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {
/* .pack_func_ex = */ nullptr,
},
/* .rhs_info = */ {
- /* .packed_stride = */ nullptr,
- /* .to_float = */ nullptr,
/* .packed_size_ex = */ &rhs_ps_fn2<kai_get_rhs_packed_size_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,
- /* .packed_stride_ex = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,
/* .pack_func_ex = */ &rhs_pack_fn13<kai_run_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,
},
/* .required_cpu = */ CPU_FEATURE_SME2,
@@ -986,7 +1041,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa>,
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa>,
/* .run_kernel_ex = */ &kernel_run_fn10<kai_run_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa>,
@@ -1006,7 +1060,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
- /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
/* .get_lhs_offset_ex = */ nullptr,
/* .get_rhs_packed_offset_ex = */ nullptr,
/* .run_kernel_ex = */ nullptr,
@@ -1018,10 +1071,7 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {
/* .pack_func_ex = */ &lhs_pack_void_fn9<kai_run_lhs_pack_f32p2vlx1_f32_sme>,
},
/* .rhs_info = */ {
- /* .packed_stride = */ nullptr,
- /* .to_float = */ nullptr,
/* .packed_size_ex = */ &rhs_ps_fn2<kai_get_rhs_packed_size_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,
- /* .packed_stride_ex = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,
/* .pack_func_ex = */ &rhs_pack_fn13<kai_run_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,
},
/* .required_cpu = */ CPU_FEATURE_SME,
@@ -1049,8 +1099,23 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, c
return false;
};
- if (tensor->src[0]->type == GGML_TYPE_Q8_0) {
- try_table(gemm_gemv_kernels_q8);
+ if (tensor->src[0]->type == GGML_TYPE_Q4_K) {
+ try_table(gemm_gemv_kernels_q4_k);
+ } else if (tensor->src[0]->type == GGML_TYPE_Q8_0 || tensor->src[0]->type == GGML_TYPE_Q6_K) {
+ const ggml_type original_type = tensor->src[0]->type;
+ if (original_type == GGML_TYPE_Q6_K) {
+ for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q8) - 1; ++i) {
+ if ((cpu_features & gemm_gemv_kernels_q8[i].required_cpu) == gemm_gemv_kernels_q8[i].required_cpu &&
+ gemm_gemv_kernels_q8[i].lhs_type == tensor->src[1]->type &&
+ gemm_gemv_kernels_q8[i].rhs_type == GGML_TYPE_Q8_0 &&
+ gemm_gemv_kernels_q8[i].op_type == tensor->type) {
+ kernel = &gemm_gemv_kernels_q8[i];
+ break;
+ }
+ }
+ } else {
+ try_table(gemm_gemv_kernels_q8);
+ }
} else if (tensor->src[0]->type == GGML_TYPE_F32) {
try_table(ggml_kleidiai_kernels_f32);
} else {
@@ -1075,6 +1140,19 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features)
return kernels;
}
+ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_k(cpu_feature features) {
+ ggml_kleidiai_kernels * kernels = nullptr;
+
+ for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q4_k) - 1; ++i) {
+ if ((features & gemm_gemv_kernels_q4_k[i].required_cpu) == gemm_gemv_kernels_q4_k[i].required_cpu) {
+ kernels = &gemm_gemv_kernels_q4_k[i];
+ break;
+ }
+ }
+
+ return kernels;
+}
+
ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features) {
ggml_kleidiai_kernels * kernels = nullptr;
diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.h b/ggml/src/ggml-cpu/kleidiai/kernels.h
index 1da8610ea..308ea9451 100644
--- a/ggml/src/ggml-cpu/kleidiai/kernels.h
+++ b/ggml/src/ggml-cpu/kleidiai/kernels.h
@@ -33,7 +33,6 @@ struct kernel_info {
size_t (*get_sr)(void);
size_t (*get_dst_offset)(size_t m_idx, size_t n_idx, size_t stride);
- size_t (*get_dst_size)(size_t m, size_t n);
size_t (*get_lhs_offset_ex)(size_t m_idx, size_t k, size_t bl);
@@ -59,21 +58,12 @@ struct lhs_packing_info {
enum rhs_repack_mode {
RHS_REPACK_PER_KERNEL,
- RHS_REPACK_SHARED,
RHS_REPACK_SINGLE_ONLY,
};
struct rhs_packing_info {
- size_t (*packed_stride)(size_t k, size_t nr, size_t kr, size_t bl);
-
- void (*to_float)(const void *packed_data, int32_t row_idx, int64_t nc, float *out,
- size_t nr_pack, size_t packed_row_stride, size_t kr, size_t bl,
- size_t num_bytes_multiplier);
-
size_t (*packed_size_ex)(size_t n, size_t k, size_t nr, size_t kr, size_t bl);
- size_t (*packed_stride_ex)(size_t k, size_t nr, size_t kr, size_t bl);
-
void (*pack_func_ex)(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl,
size_t rhs_stride, const void * rhs, const void * bias, const void * scale, void * rhs_packed, size_t extra_bytes, const void * params);
@@ -97,5 +87,6 @@ struct ggml_kleidiai_kernels {
ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, const ggml_tensor * tensor);
ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features);
+ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_k(cpu_feature features);
ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features);
ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_f32(cpu_feature features);
diff --git a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
index 1384ca927..dd9bcbf63 100644
--- a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
+++ b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
@@ -43,6 +43,7 @@
#include "ggml-impl.h"
#include "ggml-feats.h"
#include "ggml-backend-impl.h"
+#include "ggml-quants.h"
#include "ggml-threading.h"
#include "traits.h"
@@ -57,16 +58,20 @@ static constexpr int GGML_KLEIDIAI_MAX_KERNEL_SLOTS = 2;
static constexpr uint32_t GGML_KLEIDIAI_PACK_MAGIC = 0x4b4c4149; // "KLAI"
static constexpr uint16_t GGML_KLEIDIAI_PACK_VERSION = 1;
static constexpr size_t GGML_KLEIDIAI_PACK_ALIGN = 64;
+static constexpr size_t GGML_KLEIDIAI_Q4_K_BL = QK_K / 8;
+
+static_assert(QK_K == 256, "KleidiAI Q4_K kernels require 256-value super-blocks");
struct ggml_kleidiai_context {
cpu_feature features;
ggml_kleidiai_kernels * kernels_q4;
+ ggml_kleidiai_kernels * kernels_q4k;
ggml_kleidiai_kernels * kernels_q8;
ggml_kleidiai_kernels * kernels_f32;
int sme_thread_cap; // <= 0 means "SME disabled/unknown"
int thread_hint; // <= 0 means "no hint"
int chunk_multiplier;
-} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, 0, -1, 4 };
+} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, nullptr, 0, -1, 4 };
static inline bool is_sme_family(cpu_feature f) {
return (f & (CPU_FEATURE_SME | CPU_FEATURE_SME2)) != CPU_FEATURE_NONE;
@@ -387,6 +392,7 @@ static void init_kleidiai_context(void) {
// Kernel selection
ctx.kernels_q4 = ggml_kleidiai_select_kernels_q4_0(ctx.features);
+ ctx.kernels_q4k = ggml_kleidiai_select_kernels_q4_k(ctx.features);
ctx.kernels_q8 = ggml_kleidiai_select_kernels_q8_0(ctx.features);
ctx.kernels_f32 = ggml_kleidiai_select_kernels_f32(ctx.features);
@@ -402,6 +408,12 @@ static void init_kleidiai_context(void) {
GGML_LOG_INFO("kleidiai: primary q8 kernel feature %s\n", cpu_feature_to_string(ctx.kernels_q8->required_cpu));
}
+ if (!ctx.kernels_q4k) {
+ GGML_LOG_INFO("kleidiai: no compatible q4_k kernels found for CPU features mask %d\n", (int) ctx.features);
+ } else {
+ GGML_LOG_INFO("kleidiai: primary q4_k kernel feature %s\n", cpu_feature_to_string(ctx.kernels_q4k->required_cpu));
+ }
+
if (!ctx.kernels_f32) {
GGML_LOG_INFO("kleidiai: no compatible f32 kernels found for CPU features mask %d\n", (int)ctx.features);
} else {
@@ -410,6 +422,7 @@ static void init_kleidiai_context(void) {
const bool has_selected_sme_family_kernel =
(ctx.kernels_q4 && is_sme_family(ctx.kernels_q4->required_cpu)) ||
+ (ctx.kernels_q4k && is_sme_family(ctx.kernels_q4k->required_cpu)) ||
(ctx.kernels_q8 && is_sme_family(ctx.kernels_q8->required_cpu)) ||
(ctx.kernels_f32 && is_sme_family(ctx.kernels_f32->required_cpu));
ctx.sme_thread_cap = has_selected_sme_family_kernel ? sme_cores : 0;
@@ -486,6 +499,8 @@ static inline kleidiai_block_args kleidiai_get_block_args(ggml_type rhs_type) {
switch (rhs_type) {
case GGML_TYPE_Q4_0:
return { QK4_0, QK4_0, QK4_0 };
+ case GGML_TYPE_Q4_K:
+ return { GGML_KLEIDIAI_Q4_K_BL, GGML_KLEIDIAI_Q4_K_BL, GGML_KLEIDIAI_Q4_K_BL };
case GGML_TYPE_Q8_0:
return { 0, 0, QK8_0 };
default:
@@ -556,6 +571,10 @@ static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q4() {
return ctx.kernels_q4;
}
+static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q4k() {
+ return ctx.kernels_q4k;
+}
+
static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q8() {
return ctx.kernels_q8;
}
@@ -610,6 +629,12 @@ static int kleidiai_collect_q4_chain(std::array<ggml_kleidiai_kernels *, GGML_KL
[&](cpu_feature mask) { return ggml_kleidiai_select_kernels_q4_0(mask); });
}
+static int kleidiai_collect_q4k_chain(std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> & out) {
+ ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q4k();
+ return kleidiai_collect_kernel_chain_common(primary, ctx.features, out,
+ [&](cpu_feature mask) { return ggml_kleidiai_select_kernels_q4_k(mask); });
+}
+
static int kleidiai_collect_q8_chain(std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> & out) {
ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q8();
return kleidiai_collect_kernel_chain_common(primary, ctx.features, out,
@@ -662,8 +687,10 @@ class tensor_traits : public ggml::cpu::tensor_traits {
const size_t n = op->src[0]->ne[1];
const size_t m = op->src[1]->ne[1];
- if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) {
- const size_t qk = (op->src[0]->type == GGML_TYPE_Q4_0) ? QK4_0 : QK8_0;
+ if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_K ||
+ op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_Q6_K) {
+ const kleidiai_block_args block_args = kleidiai_get_block_args(kernel_chain[0]->rhs_type);
+ const size_t qk = block_args.pack_bl;
size_t cursor = 0;
bool any_slot = false;
@@ -789,17 +816,14 @@ class tensor_traits : public ggml::cpu::tensor_traits {
bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * dst) override {
if (dst->op == GGML_OP_MUL_MAT) {
- if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) {
+ if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q4_K ||
+ dst->src[0]->type == GGML_TYPE_Q8_0 || dst->src[0]->type == GGML_TYPE_Q6_K) {
return compute_forward_qx(params, dst);
} else if (dst->src[0]->type == GGML_TYPE_F32) {
return compute_forward_f32(params, dst);
} else if (dst->src[0]->type == GGML_TYPE_F16) {
return compute_forward_fp16(params, dst);
}
- } else if (dst->op == GGML_OP_GET_ROWS) {
- if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) {
- return compute_forward_get_rows(params, dst);
- }
}
return false;
}
@@ -870,9 +894,12 @@ class tensor_traits : public ggml::cpu::tensor_traits {
const bool disable_chunking = ggml_is_numa();
GGML_ASSERT(n <= (size_t) INT_MAX);
- for (int64_t batch_idx = 0; batch_idx < ne12; ++batch_idx) {
- const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + batch_idx * src1->nb[2];
- uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + batch_idx * dst->nb[2];
+ const int64_t batch_size = ne12 * ne13;
+ for (int64_t batch_idx = 0; batch_idx < batch_size; ++batch_idx) {
+ const int64_t i12 = batch_idx % ne12;
+ const int64_t i13 = batch_idx / ne12;
+ const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + i12 * src1->nb[2] + i13 * src1->nb[3];
+ uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + i12 * dst->nb[2] + i13 * dst->nb[3];
if (!use_gemv) {
const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)mr);
@@ -947,7 +974,7 @@ class tensor_traits : public ggml::cpu::tensor_traits {
current_col = ggml_threadpool_chunk_add(params->threadpool, (int) chunk_cols);
}
- if (batch_idx != ne12 - 1) {
+ if (batch_idx != batch_size - 1) {
ggml_barrier(params->threadpool);
}
}
@@ -1107,7 +1134,8 @@ class tensor_traits : public ggml::cpu::tensor_traits {
}
bool compute_forward_qx(struct ggml_compute_params * params, struct ggml_tensor * dst) {
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0);
+ GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q4_K ||
+ dst->src[0]->type == GGML_TYPE_Q8_0 || dst->src[0]->type == GGML_TYPE_Q6_K);
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
@@ -1265,7 +1293,9 @@ class tensor_traits : public ggml::cpu::tensor_traits {
}
}
- int sme_cap = kleidiai_sme_thread_cap();
+ const bool sme_single_only = sme_slot != -1 &&
+ runtime[sme_slot].kernels->rhs_info.repack_mode == RHS_REPACK_SINGLE_ONLY;
+ int sme_cap = sme_single_only ? nth_total : kleidiai_sme_thread_cap();
if (sme_cap < 0) {
sme_cap = nth_total;
}
@@ -1301,23 +1331,14 @@ class tensor_traits : public ggml::cpu::tensor_traits {
threads_remaining -= share;
}
- if (threads_remaining > 0) {
- const int fallback_slot = (sme_slot != -1) ? sme_slot : 0;
- runtime[fallback_slot].assigned_threads += threads_remaining;
- threads_remaining = 0;
- }
-
int thread_cursor = 0;
for (int i = 0; i < runtime_count; ++i) {
runtime[i].thread_begin = thread_cursor;
thread_cursor += runtime[i].assigned_threads;
runtime[i].thread_end = thread_cursor;
}
-
- if (thread_cursor < nth_total && runtime_count > 0) {
- runtime[runtime_count - 1].assigned_threads += nth_total - thread_cursor;
- runtime[runtime_count - 1].thread_end = nth_total;
- }
+ GGML_ASSERT(thread_cursor > 0);
+ const int active_threads = thread_cursor;
int local_slot = -1;
int local_ith = 0;
@@ -1328,10 +1349,6 @@ class tensor_traits : public ggml::cpu::tensor_traits {
break;
}
}
- if (local_slot == -1) {
- return false;
- }
-
const size_t k = ne00;
const size_t m = ne11;
const size_t n = ne01;
@@ -1363,7 +1380,7 @@ class tensor_traits : public ggml::cpu::tensor_traits {
const bool disable_chunking = ggml_is_numa();
const size_t chunk_multiplier = std::max(1, ctx.chunk_multiplier);
- const size_t chunk_divisor = (nth_total == 1 || disable_chunking) ? (size_t)nth_total : (size_t)nth_total * chunk_multiplier;
+ const size_t chunk_divisor = (active_threads == 1 || disable_chunking) ? (size_t) active_threads : (size_t) active_threads * chunk_multiplier;
size_t chunk_cols = align_up(std::max<size_t>(1, ceil_div_size(n, chunk_divisor)), common_step);
if (chunk_cols == 0) {
chunk_cols = common_step;
@@ -1397,7 +1414,7 @@ class tensor_traits : public ggml::cpu::tensor_traits {
const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + batch_idx * src1->nb[2];
uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + batch_idx * dst->nb[2];
- if (runtime[local_slot].assigned_threads > 0) {
+ if (local_slot != -1) {
runtime_slot & slot = runtime[local_slot];
const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)slot.mr);
int64_t max_threads = slot.mr ? (m_roundup_mr / (int64_t)slot.mr) : slot.assigned_threads;
@@ -1438,28 +1455,30 @@ class tensor_traits : public ggml::cpu::tensor_traits {
}
if (ith_total == 0) {
- ggml_threadpool_chunk_set(params->threadpool, nth_total);
+ ggml_threadpool_chunk_set(params->threadpool, active_threads);
}
// Publishes both LHS packing and the initialized dynamic chunk queue.
ggml_barrier(params->threadpool);
- runtime_slot & slot = runtime[local_slot];
- int current_chunk = ith_total;
- while (current_chunk < nchunk) {
- const size_t global_start = (size_t)current_chunk * chunk_cols;
- if (global_start >= n) {
- break;
- }
+ if (local_slot != -1) {
+ runtime_slot & slot = runtime[local_slot];
+ int current_chunk = ith_total;
+ while (current_chunk < nchunk) {
+ const size_t global_start = (size_t)current_chunk * chunk_cols;
+ if (global_start >= n) {
+ break;
+ }
- const size_t cols = std::min(chunk_cols, n - global_start);
- if (cols > 0) {
- // KleidiAI GEMM/GEMV kernels accept arbitrary final tail widths;
- // only non-tail chunks are guaranteed to be n_step-aligned.
- run_chunk(slot, global_start, cols, dst_batch_base);
- }
+ const size_t cols = std::min(chunk_cols, n - global_start);
+ if (cols > 0) {
+ // KleidiAI GEMM/GEMV kernels accept arbitrary final tail widths;
+ // only non-tail chunks are guaranteed to be n_step-aligned.
+ run_chunk(slot, global_start, cols, dst_batch_base);
+ }
- current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);
+ current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);
+ }
}
if (batch_idx != ne12 - 1) {
@@ -1470,101 +1489,11 @@ class tensor_traits : public ggml::cpu::tensor_traits {
return true;
}
- bool compute_forward_get_rows(struct ggml_compute_params * params, struct ggml_tensor * dst) {
- GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0);
- const ggml_tensor * src0 = dst->src[0];
- const ggml_tensor * src1 = dst->src[1];
-
- GGML_TENSOR_BINARY_OP_LOCALS
-
- const kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(src0->data);
- const bool has_header = kleidiai_is_weight_header_valid(header);
-
- std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;
- const bool want_q8 = src0->type == GGML_TYPE_Q8_0;
- const int chain_count = want_q8 ? kleidiai_collect_q8_chain(kernel_chain)
- : kleidiai_collect_q4_chain(kernel_chain);
-
- ggml_kleidiai_kernels * kernels = nullptr;
- const uint8_t * packed_base = static_cast<const uint8_t *>(src0->data);
-
- if (has_header && chain_count > 0) {
- int select_slot = 0;
- if (select_slot >= header->slot_count) {
- select_slot = header->slot_count - 1;
- }
- if (select_slot >= 0 && select_slot < chain_count) {
- kernels = kernel_chain[select_slot];
- const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, select_slot);
- if (slot_ptr) {
- packed_base = slot_ptr;
- }
- }
- }
-
- if (!kernels && chain_count > 0) {
- kernels = kernel_chain[0];
- if (has_header) {
- const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, 0);
- if (slot_ptr) {
- packed_base = slot_ptr;
- }
- }
- }
-
- if (!kernels) {
- return false;
- }
-
- rhs_packing_info * rhs_info = &kernels->rhs_info;
- kernel_info * kernel = &kernels->gemm;
- if (!rhs_info->to_float || !kernel->get_nr) {
- return false;
- }
-
- const int64_t nc = ne00;
- const int64_t nr = ggml_nelements(src1);
-
- const ggml_type rhs_type = kernels->rhs_type;
- size_t block_len = 0;
- size_t num_bytes_multiplier = 0;
- if (rhs_type == GGML_TYPE_Q4_0) {
- block_len = QK4_0;
- num_bytes_multiplier = sizeof(uint16_t);
- } else if (rhs_type == GGML_TYPE_Q8_0) {
- block_len = QK8_0;
- num_bytes_multiplier = sizeof(float);
- } else {
- return false;
- }
-
- const size_t block_rows = kernel->get_nr();
- const size_t kr = kernel->get_kr();
-
- const size_t packed_stride = rhs_info->packed_stride(nc, block_rows, kr, block_len);
-
- const int ith = params->ith;
- const int nth = params->nth;
-
- const int dr = (nr + nth - 1) / nth;
- const int ir0 = dr * ith;
- const int ir1 = MIN(ir0 + dr, nr);
-
- for (int64_t i = ir0; i < ir1; ++i) {
- GGML_ASSERT(src1->type == GGML_TYPE_I32);
- int64_t row_idx = ((const int32_t *)src1->data)[i];
- GGML_ASSERT(row_idx >= 0 && row_idx < src0->ne[1]);
-
- float *out = (float *)((char *)dst->data + i * nb1);
- rhs_info->to_float(packed_base, row_idx, nc, out, block_rows, packed_stride, kr, block_len, num_bytes_multiplier);
- }
-
- return true;
- }
-
public:
int repack(struct ggml_tensor * tensor, const void * data, size_t data_size) {
- GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_F32);
+ GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q4_K ||
+ tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K ||
+ tensor->type == GGML_TYPE_F32);
const size_t n = tensor->ne[1];
const size_t k = tensor->ne[0];
@@ -1582,16 +1511,19 @@ public:
cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);
std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;
- const bool want_q8 = tensor->type == GGML_TYPE_Q8_0;
+ const bool want_q4k = tensor->type == GGML_TYPE_Q4_K;
+ const bool want_q8 = tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K;
const bool want_f32 = tensor->type == GGML_TYPE_F32;
const int slot_total = want_f32 ? kleidiai_collect_f32_chain(kernel_chain)
: want_q8 ? kleidiai_collect_q8_chain(kernel_chain)
- : kleidiai_collect_q4_chain(kernel_chain);
+ : want_q4k ? kleidiai_collect_q4k_chain(kernel_chain)
+ : kleidiai_collect_q4_chain(kernel_chain);
const bool allow_fallback = kleidiai_pack_fallback_allowed();
std::vector<int8_t> qdata;
std::vector<float> scales;
std::vector<float> bias;
+ std::vector<float> row_f32;
if (want_q8 && slot_total > 0) {
qdata.resize(n * k, 0);
@@ -1600,21 +1532,38 @@ public:
const size_t row_stride = tensor->nb[1];
const size_t k_blocks = (k + QK8_0 - 1) / QK8_0;
+ if (tensor->type == GGML_TYPE_Q6_K) {
+ if (k % QK_K != 0 || k % QK8_0 != 0) {
+ return -1;
+ }
+ row_f32.resize(k);
+ }
+
for (size_t row = 0; row < n; ++row) {
- const auto * row_blocks = reinterpret_cast<const block_q8_0 *>(
- static_cast<const uint8_t *>(data) + row * row_stride);
+ const uint8_t * row_data = static_cast<const uint8_t *>(data) + row * row_stride;
+
+ if (tensor->type == GGML_TYPE_Q6_K) {
+ dequantize_row_q6_K(reinterpret_cast<const block_q6_K *>(row_data), row_f32.data(), k);
+ }
float max_abs = 0.0f;
- for (size_t block = 0; block < k_blocks; ++block) {
- const block_q8_0 & blk = row_blocks[block];
- const float d = GGML_FP16_TO_FP32(blk.d);
- for (size_t l = 0; l < QK8_0; ++l) {
- const size_t linear_idx = block * QK8_0 + l;
- if (linear_idx >= k) {
- break;
+ if (tensor->type == GGML_TYPE_Q6_K) {
+ for (size_t linear_idx = 0; linear_idx < k; ++linear_idx) {
+ max_abs = std::max(max_abs, std::fabs(row_f32[linear_idx]));
+ }
+ } else {
+ const block_q8_0 * row_blocks = reinterpret_cast<const block_q8_0 *>(row_data);
+ for (size_t block = 0; block < k_blocks; ++block) {
+ const block_q8_0 & blk = row_blocks[block];
+ const float d = GGML_FP16_TO_FP32(blk.d);
+ for (size_t l = 0; l < QK8_0; ++l) {
+ const size_t linear_idx = block * QK8_0 + l;
+ if (linear_idx >= k) {
+ break;
+ }
+ const float value = d * static_cast<float>(blk.qs[l]);
+ max_abs = std::max(max_abs, std::fabs(value));
}
- const float value = d * static_cast<float>(blk.qs[l]);
- max_abs = std::max(max_abs, std::fabs(value));
}
}
@@ -1622,19 +1571,28 @@ public:
scales[row] = scale;
const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f;
- for (size_t block = 0; block < k_blocks; ++block) {
- const block_q8_0 & blk = row_blocks[block];
- const float d = GGML_FP16_TO_FP32(blk.d);
- for (size_t l = 0; l < QK8_0; ++l) {
- const size_t linear_idx = block * QK8_0 + l;
- if (linear_idx >= k) {
- break;
- }
- const float value = d * static_cast<float>(blk.qs[l]);
- int32_t q = scale > 0.0f ? static_cast<int32_t>(std::lround(value * inv_scale)) : 0;
+ if (tensor->type == GGML_TYPE_Q6_K) {
+ for (size_t linear_idx = 0; linear_idx < k; ++linear_idx) {
+ int32_t q = scale > 0.0f ? static_cast<int32_t>(std::lround(row_f32[linear_idx] * inv_scale)) : 0;
q = std::clamp(q, -127, 127);
qdata[row * k + linear_idx] = static_cast<int8_t>(q);
}
+ } else {
+ const block_q8_0 * row_blocks = reinterpret_cast<const block_q8_0 *>(row_data);
+ for (size_t block = 0; block < k_blocks; ++block) {
+ const block_q8_0 & blk = row_blocks[block];
+ const float d = GGML_FP16_TO_FP32(blk.d);
+ for (size_t l = 0; l < QK8_0; ++l) {
+ const size_t linear_idx = block * QK8_0 + l;
+ if (linear_idx >= k) {
+ break;
+ }
+ const float value = d * static_cast<float>(blk.qs[l]);
+ int32_t q = scale > 0.0f ? static_cast<int32_t>(std::lround(value * inv_scale)) : 0;
+ q = std::clamp(q, -127, 127);
+ qdata[row * k + linear_idx] = static_cast<int8_t>(q);
+ }
+ }
}
}
}
@@ -1660,6 +1618,7 @@ public:
const ggml_type rhs_type = kernels->rhs_type;
const size_t block_len = rhs_type == GGML_TYPE_Q8_0 ? QK8_0 :
rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :
+ rhs_type == GGML_TYPE_Q4_K ? GGML_KLEIDIAI_Q4_K_BL :
rhs_type == GGML_TYPE_F32 ? 0 : SIZE_MAX;
if (block_len == SIZE_MAX) {
continue;
@@ -1677,6 +1636,9 @@ public:
rhs_info->pack_func_ex(1, n, k, nr, kr, sr, QK4_0, 0,
static_cast<const uint8_t *>(data), nullptr, nullptr,
dst_ptr, 0, ¶ms);
+ } else if (rhs_type == GGML_TYPE_Q4_K) {
+ rhs_info->pack_func_ex(1, n, k, nr, kr, sr, GGML_KLEIDIAI_Q4_K_BL, tensor->nb[1],
+ data, nullptr, nullptr, dst_ptr, 0, nullptr);
} else if (rhs_type == GGML_TYPE_Q8_0) {
struct kai_rhs_pack_qsi8cx_params params;
params.lhs_zero_point = 1;
@@ -1719,7 +1681,6 @@ static enum ggml_status ggml_backend_cpu_kleidiai_buffer_init_tensor(ggml_backen
tensor->extra = (void *) ggml::cpu::kleidiai::get_tensor_traits(buffer, tensor);
return GGML_STATUS_SUCCESS;
- GGML_UNUSED(buffer);
}
static void ggml_backend_cpu_kleidiai_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor,
@@ -1762,7 +1723,9 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alignment(ggml_backend_b
static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor) {
GGML_UNUSED(buft);
- if (tensor->type != GGML_TYPE_Q4_0 && tensor->type != GGML_TYPE_Q8_0 && tensor->type != GGML_TYPE_F32) {
+ if (tensor->type != GGML_TYPE_Q4_0 && tensor->type != GGML_TYPE_Q4_K &&
+ tensor->type != GGML_TYPE_Q8_0 && tensor->type != GGML_TYPE_Q6_K &&
+ tensor->type != GGML_TYPE_F32) {
return ggml_nbytes(tensor);
}
@@ -1773,11 +1736,13 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_
cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);
std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;
- const bool want_q8 = tensor->type == GGML_TYPE_Q8_0;
+ const bool want_q4k = tensor->type == GGML_TYPE_Q4_K;
+ const bool want_q8 = tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K;
const bool want_f32 = tensor->type == GGML_TYPE_F32;
const int slot_total = want_f32 ? kleidiai_collect_f32_chain(kernel_chain)
: want_q8 ? kleidiai_collect_q8_chain(kernel_chain)
- : kleidiai_collect_q4_chain(kernel_chain);
+ : want_q4k ? kleidiai_collect_q4k_chain(kernel_chain)
+ : kleidiai_collect_q4_chain(kernel_chain);
const bool allow_fallback = kleidiai_pack_fallback_allowed();
size_t slot_count = 0;
@@ -1797,6 +1762,7 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_
const ggml_type rhs_type = kernels->rhs_type;
const size_t block_len = rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :
+ rhs_type == GGML_TYPE_Q4_K ? GGML_KLEIDIAI_Q4_K_BL :
rhs_type == GGML_TYPE_Q8_0 ? QK8_0 :
rhs_type == GGML_TYPE_F32 ? 0 : SIZE_MAX;
if (block_len == SIZE_MAX) {
@@ -1818,6 +1784,11 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_
namespace ggml::cpu::kleidiai {
class extra_buffer_type : ggml::cpu::extra_buffer_type {
bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override {
+ // GET_ROWS needs the original weights in a native buffer.
+ if (op->op != GGML_OP_MUL_MAT) {
+ return false;
+ }
+
std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;
const int slot_total = kleidiai_collect_kernel_chain(op, kernel_chain);
const bool src0_is_kleidiai =
@@ -1826,8 +1797,9 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
op->src[0]->buffer->buft->context == this &&
slot_total > 0;
- if ((op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) &&
- (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_F32) &&
+ if ((op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_K ||
+ op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_Q6_K ||
+ op->src[0]->type == GGML_TYPE_F32) &&
src0_is_kleidiai) {
if (op->src[0]->type == GGML_TYPE_Q4_0 && ctx.kernels_q4 == nullptr) {
return false;
@@ -1835,6 +1807,12 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
if (op->src[0]->type == GGML_TYPE_Q8_0 && ctx.kernels_q8 == nullptr) {
return false;
}
+ if (op->src[0]->type == GGML_TYPE_Q6_K && ctx.kernels_q8 == nullptr) {
+ return false;
+ }
+ if (op->src[0]->type == GGML_TYPE_Q4_K && ctx.kernels_q4k == nullptr) {
+ return false;
+ }
if (op->src[0]->type == GGML_TYPE_F32 && ctx.kernels_f32 == nullptr) {
return false;
}
@@ -1842,8 +1820,18 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
return false;
}
- if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) {
- if ((op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_I32) &&
+ if (op->src[0]->type == GGML_TYPE_Q4_K) {
+ if (op->op != GGML_OP_MUL_MAT || op->src[1]->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) {
+ return false;
+ }
+ return ggml_ne(op->src[1], 2) == 1 && ggml_ne(op->src[1], 3) == 1 &&
+ op->src[1]->nb[0] == sizeof(float) &&
+ (op->src[1]->ne[1] == 1 || op->src[1]->nb[1] == op->src[1]->ne[0] * sizeof(float));
+ }
+
+ if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 ||
+ op->src[0]->type == GGML_TYPE_Q6_K) {
+ if (op->src[1]->type == GGML_TYPE_F32 &&
ggml_ne(op->src[1], 3) == 1) {
return true;
}
@@ -1861,21 +1849,22 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
}
ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override {
- if (op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) {
+ if (op->op == GGML_OP_MUL_MAT) {
if (op->src[0]->buffer && op->src[0]->buffer->buft->context == this) {
return (ggml::cpu::tensor_traits *) op->src[0]->extra;
} else {
- // KleidiAI only has kernels for Q4_0 and Q8_0. For a quantized weight of any
+ // KleidiAI only has kernels for Q4_0, Q4_K, Q8_0 and Q6_K. For a quantized weight of any
// other type (K-quants, IQ) it declines the op and returns nullptr below, so
// KleidiAI does not accelerate it. Another CPU backend may still take the op,
// and this can run during graph planning, so the message says what KleidiAI
// did rather than what ends up executing. Warn once per process.
if (ggml_is_quantized(op->src[0]->type) &&
- op->src[0]->type != GGML_TYPE_Q4_0 && op->src[0]->type != GGML_TYPE_Q8_0) {
+ op->src[0]->type != GGML_TYPE_Q4_0 && op->src[0]->type != GGML_TYPE_Q4_K &&
+ op->src[0]->type != GGML_TYPE_Q8_0 && op->src[0]->type != GGML_TYPE_Q6_K) {
static std::atomic<bool> warned(false);
if (!warned.exchange(true)) {
GGML_LOG_WARN("kleidiai: no kernel for tensor type %s, not accelerated by KleidiAI "
- "(kernels available for Q4_0 and Q8_0)\n",
+ "(kernels available for Q4_0, Q4_K, Q8_0 and Q6_K)\n",
ggml_type_name(op->src[0]->type));
}
}