Commit 1ec818809 for llama.cpp
commit 1ec81880944a63bc4aaf1abfe9a6d35c7569a757
Author: Marco Colombo <ing.marco.colombo@gmail.com>
Date: Wed Sep 16 18:00:31 2026 +0200
hexagon: Support for K-Quants Q4_K and Q6_K (#28994)
implement q6k/q4k kernels
Squashed from:
feat: implement q6k kernel
hex-q6k: improve unpack accuracy
hex-q4_k: add support for Q4_K kernels
Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index 36b9f817c..3f1495645 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -21,6 +21,7 @@
#include <queue>
#include <deque>
#include <algorithm>
+#include <cmath>
#ifdef _WIN32
# define WIN32_LEAN_AND_MEAN
@@ -250,7 +251,20 @@ enum ggml_hexagon_tensor_flags {
static inline bool ggml_hexagon_is_repack_type(enum ggml_type type) {
return type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 ||
type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_NL ||
- type == GGML_TYPE_MXFP4;
+ type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q6_K ||
+ type == GGML_TYPE_Q4_K;
+}
+
+// Size of one repacked row in the DSP tiled layout. The Q6_K and Q4_K tiles store uncompressed scales/mins,
+// so they are larger than the ggml blocks. For the other repack types the tile has the same size as the ggml blocks.
+static inline size_t ggml_hexagon_tiled_row_size(enum ggml_type type, int64_t ne0) {
+ if (type == GGML_TYPE_Q6_K) {
+ return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q6_K / 32);
+ }
+ if (type == GGML_TYPE_Q4_K) {
+ return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q4_1 / 32);
+ }
+ return ggml_row_size(type, ne0);
}
static inline bool ggml_hexagon_is_hmx_weight_type(enum ggml_type type) {
@@ -1302,6 +1316,377 @@ static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t offset
}
}
+// unsigned 6-bit value (0..63) of element e of a Q6_K block, same bit layout as dequantize_row_q6_K
+static inline uint8_t q6_K_get_quant(const block_q6_K * b, int e) {
+ const int c = e / 128;
+ const int w = e % 128;
+ const int g = w / 32;
+ const int l = w % 32;
+ const uint8_t * ql = b->ql + c * 64;
+ const uint8_t * qh = b->qh + c * 32;
+ uint8_t lo, hi;
+ switch (g) {
+ case 0: lo = ql[l] & 0xF; hi = (qh[l] >> 0) & 3; break;
+ case 1: lo = ql[l + 32] & 0xF; hi = (qh[l] >> 2) & 3; break;
+ case 2: lo = ql[l] >> 4; hi = (qh[l] >> 4) & 3; break;
+ default: lo = ql[l + 32] >> 4; hi = (qh[l] >> 6) & 3; break;
+ }
+ return (uint8_t) (lo | (hi << 4));
+}
+
+// tile layout: see HTP_MM_WEIGHT_TILE_SIZE_Q6_K in htp/matmul-ops.h
+static void repack_q6_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
+ GGML_ASSERT(offset == 0);
+
+ const block_q6_K * src_matrix = (const block_q6_K *) data;
+ int64_t ne0 = t->ne[0];
+ int64_t ne1 = t->ne[1];
+ int64_t ne2 = t->ne[2];
+ int64_t ne3 = t->ne[3];
+ int64_t ne0_padded = hex_round_up(ne0, 32);
+ int64_t ne1_padded = hex_round_up(ne1, 32);
+
+ GGML_ASSERT(ne0 % QK_K == 0);
+
+ const int n_col_tiles = ne1_padded / 32;
+ const int n_k_tiles = ne0_padded / 32;
+ const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q6_K;
+ const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;
+
+ const int64_t sb_per_row = ne0 / QK_K;
+
+ for (int i3 = 0; i3 < ne3; i3++) {
+ for (int i2 = 0; i2 < ne2; i2++) {
+ const block_q6_K * src_slice = src_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row);
+ uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;
+
+ memset(matrix_dst, 0, matrix_size); // padding rows and the OR-ed nibbles below need zeroed tiles
+
+ for (int64_t r = 0; r < ne1; r++) {
+ const int ct = (int) (r / 32);
+ const int row = (int) (r % 32);
+ const block_q6_K * src_row = src_slice + r * sb_per_row;
+
+ for (int kt = 0; kt < n_k_tiles; kt++) {
+ const int kt_local = kt % 8; // k-tile within the super-block
+ const block_q6_K * b = &src_row[kt / 8];
+ const float d = GGML_FP16_TO_FP32(b->d);
+
+ uint8_t * tile = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size;
+ uint8_t * lo_pl = tile;
+ uint8_t * hi_pl = tile + 512;
+ ggml_half * sc_pl = (ggml_half *) (tile + 768);
+
+ for (int lk = 0; lk < 32; lk++) {
+ const uint8_t q6 = q6_K_get_quant(b, kt_local * 32 + lk);
+ const int g = lk >> 2;
+ const int pos = row * 4 + (lk & 3);
+ lo_pl[(g >> 1) * 128 + pos] |= (uint8_t) ((q6 & 0xF) << ((g & 1) * 4));
+ hi_pl[(g >> 2) * 128 + pos] |= (uint8_t) ((q6 >> 4) << ((g & 3) * 2));
+ }
+ for (int sub = 0; sub < 2; sub++) {
+ sc_pl[sub * 32 + row] = GGML_FP32_TO_FP16(d * (float) b->scales[kt_local * 2 + sub]);
+ }
+ }
+ }
+ }
+ }
+
+ GGML_UNUSED(size);
+}
+
+// Reverse of repack_q6_K_tiled. Unpacks quants losslessly and normalizes sub-block scales. Read-back only.
+static void repack_tiled_q6_K(void * data, const ggml_tensor * t, size_t offset, size_t size) {
+ GGML_ASSERT(offset == 0);
+
+ block_q6_K * dst_matrix = (block_q6_K *) data;
+ int64_t ne0 = t->ne[0];
+ int64_t ne1 = t->ne[1];
+ int64_t ne2 = t->ne[2];
+ int64_t ne3 = t->ne[3];
+ int64_t ne0_padded = hex_round_up(ne0, 32);
+ int64_t ne1_padded = hex_round_up(ne1, 32);
+
+ GGML_ASSERT(ne0 % QK_K == 0);
+
+ const int n_col_tiles = ne1_padded / 32;
+ const int n_k_tiles = ne0_padded / 32;
+ const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q6_K;
+ const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;
+
+ const int64_t sb_per_row = ne0 / QK_K;
+
+ for (int i3 = 0; i3 < ne3; i3++) {
+ for (int i2 = 0; i2 < ne2; i2++) {
+ block_q6_K * dst_slice = dst_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row);
+ const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;
+
+ for (int64_t r = 0; r < ne1; r++) {
+ const int ct = (int) (r / 32);
+ const int row = (int) (r % 32);
+ block_q6_K * dst_row = dst_slice + r * sb_per_row;
+
+ for (int64_t sb = 0; sb < sb_per_row; sb++) {
+ block_q6_K * b = &dst_row[sb];
+ memset(b, 0, sizeof(block_q6_K));
+
+ float sub_scales[16];
+ for (int kt_local = 0; kt_local < 8; kt_local++) {
+ const int kt = sb * 8 + kt_local;
+ const uint8_t * tile = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size;
+ const uint8_t * lo_pl = tile;
+ const uint8_t * hi_pl = tile + 512;
+ const ggml_half * sc_pl = (const ggml_half *) (tile + 768);
+
+ const int c = kt_local / 4;
+ const int g = kt_local % 4;
+ uint8_t * ql = b->ql + c * 64;
+ uint8_t * qh = b->qh + c * 32;
+
+ for (int lk = 0; lk < 32; lk++) {
+ const int g_tile = lk >> 2;
+ const int pos = row * 4 + (lk & 3);
+ const uint8_t lo = (lo_pl[(g_tile >> 1) * 128 + pos] >> ((g_tile & 1) * 4)) & 0xF;
+ const uint8_t hi = (hi_pl[(g_tile >> 2) * 128 + pos] >> ((g_tile & 3) * 2)) & 3;
+
+ switch (g) {
+ case 0:
+ ql[lk] |= lo;
+ qh[lk] |= (hi << 0);
+ break;
+ case 1:
+ ql[lk + 32] |= lo;
+ qh[lk] |= (hi << 2);
+ break;
+ case 2:
+ ql[lk] |= (lo << 4);
+ qh[lk] |= (hi << 4);
+ break;
+ default:
+ ql[lk + 32] |= (lo << 4);
+ qh[lk] |= (hi << 6);
+ break;
+ }
+ }
+
+ for (int sub = 0; sub < 2; sub++) {
+ sub_scales[kt_local * 2 + sub] = GGML_FP16_TO_FP32(sc_pl[sub * 32 + row]);
+ }
+ }
+
+ float max_abs_scale = 0.0f;
+ for (int s = 0; s < 16; s++) {
+ float abs_scale = fabsf(sub_scales[s]);
+ if (abs_scale > max_abs_scale) {
+ max_abs_scale = abs_scale;
+ }
+ }
+
+ if (max_abs_scale == 0.0f) {
+ b->d = GGML_FP32_TO_FP16(0.0f);
+ memset(b->scales, 0, sizeof(b->scales));
+ } else {
+ float d_flt = max_abs_scale / 127.0f;
+ b->d = GGML_FP32_TO_FP16(d_flt);
+ float d_actual = GGML_FP16_TO_FP32(b->d);
+ float inv_d = (d_actual != 0.0f) ? (1.0f / d_actual) : 0.0f;
+ for (int s = 0; s < 16; s++) {
+ int sc = (int) roundf(sub_scales[s] * inv_d);
+ b->scales[s] = (int8_t) (std::max)(-128, (std::min)(127, sc));
+ }
+ }
+ }
+ }
+ }
+ }
+
+ GGML_UNUSED(size);
+}
+
+static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) {
+ if (j < 4) {
+ *d = q[j] & 63;
+ *m = q[j + 4] & 63;
+ } else {
+ *d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);
+ *m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4);
+ }
+}
+
+// tile layout: see HTP_MM_WEIGHT_TILE_SIZE_Q4_1 in htp/matmul-ops.h
+static void repack_q4_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
+ GGML_ASSERT(offset == 0);
+
+ const block_q4_K * src_matrix = (const block_q4_K *) data;
+ int64_t ne0 = t->ne[0];
+ int64_t ne1 = t->ne[1];
+ int64_t ne2 = t->ne[2];
+ int64_t ne3 = t->ne[3];
+ int64_t ne0_padded = hex_round_up(ne0, 32);
+ int64_t ne1_padded = hex_round_up(ne1, 32);
+
+ GGML_ASSERT(ne0 % QK_K == 0);
+
+ const int n_col_tiles = ne1_padded / 32;
+ const int n_k_tiles = ne0_padded / 32;
+ const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1;
+ const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;
+
+ const int64_t sb_per_row = ne0 / QK_K;
+
+ for (int i3 = 0; i3 < ne3; i3++) {
+ for (int i2 = 0; i2 < ne2; i2++) {
+ const block_q4_K * src_slice = src_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row);
+ uint8_t * matrix_dst = (uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;
+
+ memset(matrix_dst, 0, matrix_size);
+
+ for (int64_t r = 0; r < ne1; r++) {
+ const int ct = (int) (r / 32);
+ const int row = (int) (r % 32);
+ const block_q4_K * src_row = src_slice + r * sb_per_row;
+
+ for (int kt = 0; kt < n_k_tiles; kt++) {
+ const int kt_local = kt % 8;
+ const block_q4_K * b = &src_row[kt / 8];
+ const float d = GGML_FP16_TO_FP32(b->d);
+ const float dmin = GGML_FP16_TO_FP32(b->dmin);
+
+ uint8_t * tile_dst = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size;
+
+ uint8_t sc, m;
+ get_scale_min_k4(kt_local, b->scales, &sc, &m);
+
+ const float D = d * (float) sc;
+ const float M = -dmin * (float) m;
+
+ const uint8_t * qs_sub = b->qs + (kt_local / 2) * 32;
+ const int shift = (kt_local & 1) ? 4 : 0;
+
+ for (int cp = 0; cp < 16; cp++) {
+ const uint8_t q0 = (qs_sub[2 * cp + 0] >> shift) & 0x0F;
+ const uint8_t q1 = (qs_sub[2 * cp + 1] >> shift) & 0x0F;
+ tile_dst[cp * 32 + row] = (uint8_t) ((q1 << 4) | q0);
+ }
+
+ ggml_half * scale_dst = (ggml_half *) (tile_dst + 512);
+ scale_dst[2 * row + 0] = GGML_FP32_TO_FP16(D);
+ scale_dst[2 * row + 1] = GGML_FP32_TO_FP16(M);
+ }
+ }
+ }
+ }
+
+ GGML_UNUSED(size);
+}
+
+// Reverse of repack_q4_K_tiled. Unpacks quants and normalizes scales/mins. Read-back only.
+static void repack_tiled_q4_K(void * data, const ggml_tensor * t, size_t offset, size_t size) {
+ GGML_ASSERT(offset == 0);
+
+ block_q4_K * dst_matrix = (block_q4_K *) data;
+ int64_t ne0 = t->ne[0];
+ int64_t ne1 = t->ne[1];
+ int64_t ne2 = t->ne[2];
+ int64_t ne3 = t->ne[3];
+ int64_t ne0_padded = hex_round_up(ne0, 32);
+ int64_t ne1_padded = hex_round_up(ne1, 32);
+
+ GGML_ASSERT(ne0 % QK_K == 0);
+
+ const int n_col_tiles = ne1_padded / 32;
+ const int n_k_tiles = ne0_padded / 32;
+ const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1;
+ const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;
+
+ const int64_t sb_per_row = ne0 / QK_K;
+
+ for (int i3 = 0; i3 < ne3; i3++) {
+ for (int i2 = 0; i2 < ne2; i2++) {
+ block_q4_K * dst_slice = dst_matrix + (i3 * ne2 + i2) * (ne1 * sb_per_row);
+ const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size;
+
+ for (int64_t r = 0; r < ne1; r++) {
+ const int ct = (int) (r / 32);
+ const int row = (int) (r % 32);
+ block_q4_K * dst_row = dst_slice + r * sb_per_row;
+
+ for (int64_t sb = 0; sb < sb_per_row; sb++) {
+ block_q4_K * b = &dst_row[sb];
+ memset(b, 0, sizeof(block_q4_K));
+
+ float sub_scales[8];
+ float sub_mins[8];
+
+ for (int kt_local = 0; kt_local < 8; kt_local++) {
+ const int kt = sb * 8 + kt_local;
+ const uint8_t * tile_src = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size;
+ const ggml_half * scale_src = (const ggml_half *) (tile_src + 512);
+
+ uint8_t * qs_sub = b->qs + (kt_local / 2) * 32;
+ const int shift = (kt_local & 1) ? 4 : 0;
+
+ for (int cp = 0; cp < 16; cp++) {
+ const uint8_t val = tile_src[cp * 32 + row];
+ const uint8_t q0 = val & 0x0F;
+ const uint8_t q1 = val >> 4;
+ qs_sub[2 * cp + 0] |= (uint8_t) (q0 << shift);
+ qs_sub[2 * cp + 1] |= (uint8_t) (q1 << shift);
+ }
+
+ const float D = GGML_FP16_TO_FP32(scale_src[2 * row + 0]);
+ const float M = GGML_FP16_TO_FP32(scale_src[2 * row + 1]);
+ sub_scales[kt_local] = (D > 0.0f) ? D : 0.0f;
+ sub_mins[kt_local] = (-M > 0.0f) ? -M : 0.0f;
+ }
+
+ float max_scale = 0.0f;
+ float max_min = 0.0f;
+ for (int j = 0; j < 8; j++) {
+ if (sub_scales[j] > max_scale) max_scale = sub_scales[j];
+ if (sub_mins[j] > max_min) max_min = sub_mins[j];
+ }
+
+ float inv_scale = 0.0f;
+ if (max_scale > 0.0f) {
+ b->d = GGML_FP32_TO_FP16(max_scale / 63.0f);
+ const float d_actual = GGML_FP16_TO_FP32(b->d);
+ inv_scale = (d_actual > 0.0f) ? (1.0f / d_actual) : 0.0f;
+ } else {
+ b->d = GGML_FP32_TO_FP16(0.0f);
+ }
+
+ float inv_min = 0.0f;
+ if (max_min > 0.0f) {
+ b->dmin = GGML_FP32_TO_FP16(max_min / 63.0f);
+ const float dmin_actual = GGML_FP16_TO_FP32(b->dmin);
+ inv_min = (dmin_actual > 0.0f) ? (1.0f / dmin_actual) : 0.0f;
+ } else {
+ b->dmin = GGML_FP32_TO_FP16(0.0f);
+ }
+
+ for (int j = 0; j < 8; j++) {
+ uint8_t ls = (uint8_t) roundf(inv_scale * sub_scales[j]);
+ uint8_t lm = (uint8_t) roundf(inv_min * sub_mins[j]);
+ ls = (std::min)((uint8_t) 63, ls);
+ lm = (std::min)((uint8_t) 63, lm);
+ if (j < 4) {
+ b->scales[j] = ls;
+ b->scales[j + 4] = lm;
+ } else {
+ b->scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4);
+ b->scales[j - 4] |= ((ls >> 4) << 6);
+ b->scales[j - 0] |= ((lm >> 4) << 6);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ GGML_UNUSED(size);
+}
+
static void repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t size) {
switch (tensor->type) {
case GGML_TYPE_Q4_0:
@@ -1312,6 +1697,10 @@ static void repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t
repack_q4_1_tiled(tensor, data, 0, size);
break;
+ case GGML_TYPE_Q4_K:
+ repack_q4_K_tiled(tensor, data, 0, size);
+ break;
+
case GGML_TYPE_Q8_0:
repack_q8_0_tiled(tensor, data, 0, size);
break;
@@ -1324,6 +1713,10 @@ static void repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t
repack_mxfp4_tiled(tensor, data, 0, size);
break;
+ case GGML_TYPE_Q6_K:
+ repack_q6_K_tiled(tensor, data, 0, size);
+ break;
+
default:
break;
}
@@ -1402,6 +1795,12 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,
repack_tiled_q4_1(data, tensor, offset, size);
break;
+ case GGML_TYPE_Q4_K:
+ GGML_ASSERT(offset == 0);
+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
+ repack_tiled_q4_K(data, tensor, offset, size);
+ break;
+
case GGML_TYPE_Q8_0:
GGML_ASSERT(offset == 0);
GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
@@ -1420,6 +1819,12 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,
repack_tiled_mxfp4(data, tensor, offset, size);
break;
+ case GGML_TYPE_Q6_K:
+ GGML_ASSERT(offset == 0);
+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
+ repack_tiled_q6_K(data, tensor, offset, size);
+ break;
+
default:
memcpy(data, (const char *) tensor->data + offset, size);
break;
@@ -1525,6 +1930,10 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff
repack_tiled_q4_1(temp_buf.data(), tensor, offset, temp_size);
break;
+ case GGML_TYPE_Q4_K:
+ repack_tiled_q4_K(temp_buf.data(), tensor, offset, temp_size);
+ break;
+
case GGML_TYPE_Q8_0:
repack_tiled_q8_0(temp_buf.data(), tensor, offset, temp_size);
break;
@@ -1537,6 +1946,10 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff
repack_tiled_mxfp4(temp_buf.data(), tensor, offset, temp_size);
break;
+ case GGML_TYPE_Q6_K:
+ repack_tiled_q6_K(temp_buf.data(), tensor, offset, temp_size);
+ break;
+
default:
memcpy(temp_buf.data(), (const uint8_t *) tensor->data + offset, temp_size);
break;
@@ -1645,7 +2058,7 @@ static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffe
int64_t ne1 = hex_round_up(t->ne[1], 32);
int64_t ne2 = t->ne[2];
int64_t ne3 = t->ne[3];
- return ggml_row_size(t->type, ne0) * ne1 * ne2 * ne3;
+ return ggml_hexagon_tiled_row_size(t->type, ne0) * ne1 * ne2 * ne3;
}
return ggml_nbytes(t);
@@ -1806,7 +2219,7 @@ struct ggml_hexagon_opbatch {
ne0 = hex_round_up(ne0, 32);
ne1 = hex_round_up(ne1, 32);
}
- int64_t nb1 = is_repack ? ggml_row_size(t->type, ne0) : t->nb[1];
+ int64_t nb1 = is_repack ? (int64_t) ggml_hexagon_tiled_row_size(t->type, ne0) : t->nb[1];
int64_t nb2 = is_repack ? nb1 * ne1 : t->nb[2];
int64_t nb3 = is_repack ? nb2 * t->ne[2] : t->nb[3];
@@ -1855,7 +2268,7 @@ struct ggml_hexagon_opbatch {
h.ne[3] = t->ne[3];
h.nb[0] = t->nb[0];
- h.nb[1] = ggml_row_size(t->type, h.ne[0]);
+ h.nb[1] = ggml_hexagon_tiled_row_size(t->type, h.ne[0]);
h.nb[2] = h.nb[1] * h.ne[1];
h.nb[3] = h.nb[2] * h.ne[2];
h.size = h.nb[3] * h.ne[3];
@@ -3932,7 +4345,7 @@ static bool ggml_hexagon_precompute_hmx_mm_params(
kparams->n_act_threads = act_threads_selected;
kparams->tile_size = htp_mm_get_weight_tile_size(wtype);
kparams->aligned_tile_size = aligned_tile_size;
- kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
+ kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
kparams->vtcm_size = vtcm_size;
kparams->vtcm_src0_size = 0;
kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected);
@@ -3982,7 +4395,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(
if (is_matmul_id) {
kparams->kernel_type = (src1_nrows < (int) sess->n_threads) ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW;
- kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
+ kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
struct htp_mm_hvx_vtcm_layout L;
uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16;
@@ -4011,7 +4424,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(
} else {
bool try_tiled = (k_align && opt_mm_select >= 2);
if (try_tiled) {
- kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
+ kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
if (src1_nrows < (int)sess->n_threads) {
kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_BLOCK;
} else {
@@ -4052,7 +4465,7 @@ static void ggml_hexagon_precompute_hvx_mm_params(
// Flat HVX fallback
{
- kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
+ kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT;
struct htp_mm_hvx_vtcm_layout L;
@@ -4444,7 +4857,7 @@ static void ggml_hexagon_precompute_fused_mmnx_params(
{
const int src1_nrows = ne11 * ne12 * ne13;
- const size_t src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
+ const size_t src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
const size_t src0_row_size = src0->nb[1];
uint32_t best_n_prefetch = 16;
@@ -4484,7 +4897,7 @@ static void ggml_hexagon_precompute_fused_mmnx_params(
kparams->n_weights = n_weights;
} else {
kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT;
- size_t flat_src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
+ size_t flat_src1_row_size = (wtype == GGML_TYPE_Q4_1 || wtype == GGML_TYPE_Q4_K) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
htp_mm_hvx_vtcm_layout_build(
&L, HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT, wtype, ne10, src1_nrows, sess->n_threads,
@@ -4547,7 +4960,9 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s
case GGML_TYPE_Q8_0:
case GGML_TYPE_IQ4_NL:
case GGML_TYPE_MXFP4:
- if (src0->ne[0] % 32) {
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q6_K:
+ if (src0->ne[0] % ((src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q4_K) ? QK_K : 32)) {
return false;
}
@@ -4611,7 +5026,9 @@ static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session
case GGML_TYPE_Q8_0:
case GGML_TYPE_IQ4_NL:
case GGML_TYPE_MXFP4:
- if ((src0->ne[0] % 32)) {
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q6_K:
+ if (src0->ne[0] % ((src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q4_K) ? QK_K : 32)) {
return false;
}
@@ -5347,8 +5764,8 @@ static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struc
return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;
}
- if (!ggml_hexagon_is_repack_type(src0->type)) {
- return false;
+ if (!ggml_hexagon_is_repack_type(src0->type) || src0->type == GGML_TYPE_Q6_K) {
+ return false; // Q6_K has no fused HVX kernel
}
return kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT;
@@ -5378,7 +5795,7 @@ static bool is_mergeable_mul_mat(const ggml_tensor * t) {
return ggml_hexagon_is_hmx_weight_type(src0->type);
}
- return ggml_hexagon_is_repack_type(src0->type);
+ return ggml_hexagon_is_repack_type(src0->type) && src0->type != GGML_TYPE_Q6_K;
}
static bool is_mergeable_mul_mat_pair(const ggml_tensor * n1, const ggml_tensor * n2) {
@@ -6767,6 +7184,10 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
"please update hexagon_type to match ggml_type");
static_assert((unsigned int) HTP_TYPE_IQ4_NL == (unsigned int) GGML_TYPE_IQ4_NL,
"please update hexagon_type to match ggml_type");
+ static_assert((unsigned int) HTP_TYPE_Q4_K == (unsigned int) GGML_TYPE_Q4_K,
+ "please update hexagon_type to match ggml_type");
+ static_assert((unsigned int) HTP_TYPE_Q6_K == (unsigned int) GGML_TYPE_Q6_K,
+ "please update hexagon_type to match ggml_type");
const char * str_verbose = getenv("GGML_HEXAGON_VERBOSE");
const char * str_opbatch = getenv("GGML_HEXAGON_OPBATCH");
diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h
index 0011abba5..d6d40586c 100644
--- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h
+++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h
@@ -506,6 +506,41 @@ static void dequantize_tiled_weight_to_fp16_task_q8_0(
}
}
+// Q6_K stores 6-bit weights and one fp16 scale per 16 k, see HTP_MM_WEIGHT_TILE_SIZE_Q6_K.
+// A k-group holds 4 k per row, the HMX tile holds 2, so each group is dealt into two tiles.
+static void dequantize_tiled_weight_to_fp16_task_q6_k(
+ const tiled_dequantize_state_t *state,
+ uint32_t start_tile, uint32_t end_tile) {
+
+ const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F);
+ const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
+ const HVX_Vector i32 = Q6_Vb_vsplat_R(32);
+
+ for (uint32_t t = start_tile; t < end_tile; t++) {
+ const HVX_Vector * vptr = (const HVX_Vector *) (state->src + t * state->aligned_tile_size);
+ __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS;
+
+ HVX_Vector v_sc = vptr[6];
+ HVX_Vector v_sc_k16 = Q6_V_vror_VR(v_sc, 64);
+ HVX_Vector v_scale_k0 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2));
+ HVX_Vector v_scale_k16 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc_k16, v_sc_k16, -2));
+
+ #pragma unroll
+ for (int g = 0; g < 8; g++) {
+ const HVX_Vector v_scale = (g < 4) ? v_scale_k0 : v_scale_k16;
+
+ HVX_Vector v_q = unpack_q6_k_group(vptr, g, mask_0f, mask_03, i32);
+ HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q);
+ HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4);
+
+ hvx_vmem(dst_ptr + (2 * g + 0) * 64) =
+ Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale));
+ hvx_vmem(dst_ptr + (2 * g + 1) * 64) =
+ Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale));
+ }
+ }
+}
+
static __attribute__((noinline))
void convert_f16_weight_to_fp16_tiles_task(
const tiled_dequantize_state_t *state,
@@ -803,15 +838,12 @@ static void transfer_output_chunk_fp16_to_fp32_col_chunk(
HVX_Vector v = ((const HVX_Vector *) tile)[r1];
HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one);
- HVX_Vector *pv_out0 = (HVX_Vector *) (output_row_base + c + 0);
- HVX_Vector *pv_out1 = (HVX_Vector *) (output_row_base + c + dst_stride);
-
HVX_Vector v_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp));
if (src2_row_base) {
HVX_Vector v_src2_0 = hvx_vmemu(src2_row_base + c + 0);
v_out0 = hvx_vec_add_f32_f32(v_out0, v_src2_0);
}
- *pv_out0 = v_out0;
+ hvx_vmemu(output_row_base + c + 0) = v_out0;
if (r + 1 < n_rows) {
HVX_Vector v_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp));
@@ -819,7 +851,7 @@ static void transfer_output_chunk_fp16_to_fp32_col_chunk(
HVX_Vector v_src2_1 = hvx_vmemu(src2_row_base + c + src2_stride);
v_out1 = hvx_vec_add_f32_f32(v_out1, v_src2_1);
}
- *pv_out1 = v_out1;
+ hvx_vmemu(output_row_base + c + dst_stride) = v_out1;
}
}
@@ -1366,12 +1398,9 @@ static void transfer_output_chunk_fp16_to_fp32_scattered(
HVX_Vector v = ((const HVX_Vector *) tile)[r1];
HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one);
- HVX_Vector *pv_out0 = (HVX_Vector *) (output_row0 + c);
- HVX_Vector *pv_out1 = output_row1 ? (HVX_Vector *) (output_row1 + c) : NULL;
-
- *pv_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp));
- if (pv_out1) {
- *pv_out1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp));
+ hvx_vmemu(output_row0 + c) = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp));
+ if (output_row1) {
+ hvx_vmemu(output_row1 + c) = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vp));
}
}
}
diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h
index 869b19b8c..98a5f6d5c 100644
--- a/ggml/src/ggml-hexagon/htp/htp-ops.h
+++ b/ggml/src/ggml-hexagon/htp/htp-ops.h
@@ -22,6 +22,8 @@ enum htp_data_type {
HTP_TYPE_Q4_0 = 2,
HTP_TYPE_Q4_1 = 3,
HTP_TYPE_Q8_0 = 8,
+ HTP_TYPE_Q4_K = 12,
+ HTP_TYPE_Q6_K = 14,
HTP_TYPE_IQ4_NL = 20,
HTP_TYPE_I32 = 26,
HTP_TYPE_I64 = 27,
diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h
index 328a83118..5c1372cf1 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h
@@ -744,6 +744,143 @@ static void flat_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float
}
}
+static void flat_vec_dot_q6_k_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
+ const uint8_t * restrict tile_ptr = vx;
+ const uint8_t * restrict y_q = vy;
+
+ HVX_Vector v_sum_float = Q6_V_vzero();
+ HVX_Vector i32 = Q6_Vb_vsplat_R(32);
+
+ static const uint8_t __attribute__((aligned(128))) repl[128] = {
+ 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ };
+ HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl;
+
+ const uint32_t quants_size = hex_round_up(n, 128);
+ const __fp16 * restrict y_scales = (const __fp16 *) (y_q + quants_size);
+
+ uint32_t n_k_tiles = n / 32;
+ for (uint32_t kt = 0; kt < n_k_tiles; kt++) {
+ const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 896);
+
+ uint32_t block_idx = kt / 4;
+ uint32_t sub_idx = kt % 4;
+
+ HVX_Vector vx_i8 = * (const HVX_Vector *) (y_q + block_idx * 128);
+ HVX_Vector v_act_raw = Q6_V_vror_VR(vx_i8, sub_idx * 32);
+
+ HVX_Vector v_act_rep[8];
+ v_act_rep[0] = Q6_V_vdelta_VV(v_act_raw, v_repl_ctrl);
+ v_act_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 4), v_repl_ctrl);
+ v_act_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 8), v_repl_ctrl);
+ v_act_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 12), v_repl_ctrl);
+ v_act_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 16), v_repl_ctrl);
+ v_act_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 20), v_repl_ctrl);
+ v_act_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 24), v_repl_ctrl);
+ v_act_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act_raw, 28), v_repl_ctrl);
+
+ HVX_VectorPair v_sums = accum_q6_k_32x1(vptr, v_act_rep, i32);
+
+ __fp16 scale_a_val = y_scales[kt];
+ HVX_Vector v_scale_a = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a_val));
+
+ v_sum_float = hvx_vec_add_f32_f32(v_sum_float, scale_q6_k_32x1(v_sums, vptr[6], v_scale_a));
+ }
+
+ if (sz) {
+ hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz)));
+ } else {
+ hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float);
+ }
+}
+
+static void flat_vec_dot_q6_k_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) {
+ const uint8_t * restrict tile_ptr = vx;
+ const uint8_t * restrict y0_q = vy0;
+ const uint8_t * restrict y1_q = vy1;
+
+ HVX_Vector v_sum_float_c0 = Q6_V_vzero();
+ HVX_Vector v_sum_float_c1 = Q6_V_vzero();
+ HVX_Vector i32 = Q6_Vb_vsplat_R(32);
+
+ static const uint8_t __attribute__((aligned(128))) repl[128] = {
+ 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ 0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ 0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ 0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
+ };
+ HVX_Vector v_repl_ctrl = * (const HVX_Vector *) repl;
+
+ const uint32_t quants_size = hex_round_up(n, 128);
+ const __fp16 * restrict y0_scales = (const __fp16 *) (y0_q + quants_size);
+ const __fp16 * restrict y1_scales = (const __fp16 *) (y1_q + quants_size);
+
+ uint32_t n_k_tiles = n / 32;
+ for (uint32_t kt = 0; kt < n_k_tiles; kt++) {
+ const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 896);
+
+ uint32_t block_idx = kt / 4;
+ uint32_t sub_idx = kt % 4;
+
+ HVX_Vector vx0_i8 = * (const HVX_Vector *) (y0_q + block_idx * 128);
+ HVX_Vector vx1_i8 = * (const HVX_Vector *) (y1_q + block_idx * 128);
+ HVX_Vector v_act0_raw = Q6_V_vror_VR(vx0_i8, sub_idx * 32);
+ HVX_Vector v_act1_raw = Q6_V_vror_VR(vx1_i8, sub_idx * 32);
+
+ HVX_Vector v_act0_rep[8];
+ HVX_Vector v_act1_rep[8];
+ v_act0_rep[0] = Q6_V_vdelta_VV(v_act0_raw, v_repl_ctrl);
+ v_act0_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 4), v_repl_ctrl);
+ v_act0_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 8), v_repl_ctrl);
+ v_act0_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 12), v_repl_ctrl);
+ v_act0_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 16), v_repl_ctrl);
+ v_act0_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 20), v_repl_ctrl);
+ v_act0_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 24), v_repl_ctrl);
+ v_act0_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act0_raw, 28), v_repl_ctrl);
+ v_act1_rep[0] = Q6_V_vdelta_VV(v_act1_raw, v_repl_ctrl);
+ v_act1_rep[1] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 4), v_repl_ctrl);
+ v_act1_rep[2] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 8), v_repl_ctrl);
+ v_act1_rep[3] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 12), v_repl_ctrl);
+ v_act1_rep[4] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 16), v_repl_ctrl);
+ v_act1_rep[5] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 20), v_repl_ctrl);
+ v_act1_rep[6] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 24), v_repl_ctrl);
+ v_act1_rep[7] = Q6_V_vdelta_VV(Q6_V_vror_VR(v_act1_raw, 28), v_repl_ctrl);
+
+ HVX_VectorPair v_sums0, v_sums1;
+ accum_q6_k_32x2(vptr, v_act0_rep, v_act1_rep, i32, &v_sums0, &v_sums1);
+
+ __fp16 scale_a0_val = y0_scales[kt];
+ __fp16 scale_a1_val = y1_scales[kt];
+ HVX_Vector v_scale_a0 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a0_val));
+ HVX_Vector v_scale_a1 = hvx_vec_repl_f16(Q6_Vh_vsplat_R(*(const int16_t *)&scale_a1_val));
+
+ v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, scale_q6_k_32x1(v_sums0, vptr[6], v_scale_a0));
+ v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, scale_q6_k_32x1(v_sums1, vptr[6], v_scale_a1));
+ }
+
+ if (sz0) {
+ hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0)));
+ } else {
+ hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0);
+ }
+ if (sz1) {
+ hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1)));
+ } else {
+ hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1);
+ }
+}
+
static void flat_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
const uint8_t * restrict tile_ptr = vx;
const uint8_t * restrict y_q = vy;
diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h
index 40b65aa3b..c889538ac 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h
@@ -378,6 +378,74 @@ static inline HVX_VectorPair accum_q8_0_32x2(
return Q6_W_vcombine_VV(v_sum1, v_sum0);
}
+// Q6_K weights are stored unsigned (0..63), see HTP_MM_WEIGHT_TILE_SIZE_Q6_K. Unpack k-group g of a tile to signed bytes (q - 32)
+static inline HVX_Vector unpack_q6_k_group(const HVX_Vector * restrict vptr, int g, HVX_Vector mask_0f, HVX_Vector mask_03, HVX_Vector i32) {
+ HVX_Vector v_lo = (g & 1) ? Q6_Vub_vlsr_VubR(vptr[g >> 1], 4) : Q6_V_vand_VV(vptr[g >> 1], mask_0f);
+ HVX_Vector v_hi = (g & 3) ? Q6_Vub_vlsr_VubR(vptr[4 + (g >> 2)], 2 * (g & 3)) : vptr[4 + (g >> 2)];
+ HVX_Vector v_q = Q6_V_vor_VV(v_lo, Q6_Vw_vasl_VwR(Q6_V_vand_VV(v_hi, mask_03), 4));
+ return Q6_Vb_vsub_VbVb(v_q, i32);
+}
+
+// k 0..15 and k 16..31 of a Q6_K tile have different scales: lo half of the pair sums k 0..15, hi half sums k 16..31
+static inline HVX_VectorPair accum_q6_k_32x1(
+ const HVX_Vector * restrict vptr,
+ const HVX_Vector * restrict v_act,
+ HVX_Vector i32
+) {
+ HVX_Vector v_sum_lo = Q6_V_vzero();
+ HVX_Vector v_sum_hi = Q6_V_vzero();
+ HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F);
+ HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
+
+ #pragma unroll
+ for (int g = 0; g < 4; g++) {
+ HVX_Vector v_W_lo = unpack_q6_k_group(vptr, g, mask_0f, mask_03, i32);
+ HVX_Vector v_W_hi = unpack_q6_k_group(vptr, g + 4, mask_0f, mask_03, i32);
+ v_sum_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum_lo, v_W_lo, v_act[g]);
+ v_sum_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum_hi, v_W_hi, v_act[g + 4]);
+ }
+
+ return Q6_W_vcombine_VV(v_sum_hi, v_sum_lo);
+}
+
+static inline void accum_q6_k_32x2(
+ const HVX_Vector * restrict vptr,
+ const HVX_Vector * restrict v_act0,
+ const HVX_Vector * restrict v_act1,
+ HVX_Vector i32,
+ HVX_VectorPair * v_sums0,
+ HVX_VectorPair * v_sums1
+) {
+ HVX_Vector v_sum0_lo = Q6_V_vzero();
+ HVX_Vector v_sum0_hi = Q6_V_vzero();
+ HVX_Vector v_sum1_lo = Q6_V_vzero();
+ HVX_Vector v_sum1_hi = Q6_V_vzero();
+ HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F);
+ HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
+
+ #pragma unroll
+ for (int g = 0; g < 4; g++) {
+ HVX_Vector v_W_lo = unpack_q6_k_group(vptr, g, mask_0f, mask_03, i32);
+ HVX_Vector v_W_hi = unpack_q6_k_group(vptr, g + 4, mask_0f, mask_03, i32);
+ v_sum0_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_lo, v_W_lo, v_act0[g]);
+ v_sum0_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_hi, v_W_hi, v_act0[g + 4]);
+ v_sum1_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_lo, v_W_lo, v_act1[g]);
+ v_sum1_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_hi, v_W_hi, v_act1[g + 4]);
+ }
+
+ *v_sums0 = Q6_W_vcombine_VV(v_sum0_hi, v_sum0_lo);
+ *v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo);
+}
+
+// scale the two half sums with the per-row tile scales (v_scale_w = vptr[6]) and the activation scale
+static inline HVX_Vector scale_q6_k_32x1(HVX_VectorPair v_sums, HVX_Vector v_scale_w, HVX_Vector v_scale_a) {
+ HVX_Vector v_scale_lo = hvx_vec_mul_f16_f16_to_f32_lower32(v_scale_w, v_scale_a);
+ HVX_Vector v_scale_hi = hvx_vec_mul_f16_f16_to_f32_lower32(Q6_V_vror_VR(v_scale_w, 64), v_scale_a);
+ HVX_Vector v_lo = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(v_sums)), v_scale_lo);
+ HVX_Vector v_hi = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(v_sums)), v_scale_hi);
+ return hvx_vec_add_f32_f32(v_lo, v_hi);
+}
+
static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
const uint8_t * restrict tile_ptr = vx;
const uint8_t * restrict y_q = vy;
@@ -785,6 +853,63 @@ static void tiled_vec_dot_q8_0_32x2(const uint32_t n, float * restrict s0, float
}
}
+static void tiled_vec_dot_q6_k_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
+ const uint8_t * restrict tile_ptr = vx;
+ const uint8_t * restrict y_q = vy;
+
+ HVX_Vector v_sum_float = Q6_V_vzero();
+ HVX_Vector i32 = Q6_Vb_vsplat_R(32);
+
+ uint32_t n_k_tiles = n / 32;
+ for (uint32_t kt = 0; kt < n_k_tiles; kt++) {
+ const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 896);
+ const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1152);
+
+ HVX_VectorPair v_sums = accum_q6_k_32x1(vptr, v_act, i32);
+ v_sum_float = hvx_vec_add_f32_f32(v_sum_float, scale_q6_k_32x1(v_sums, vptr[6], v_act[8]));
+ }
+
+ if (sz) {
+ hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz)));
+ } else {
+ hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float);
+ }
+}
+
+static void tiled_vec_dot_q6_k_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) {
+ const uint8_t * restrict tile_ptr = vx;
+ const uint8_t * restrict y0_q = vy0;
+ const uint8_t * restrict y1_q = vy1;
+
+ HVX_Vector v_sum_float_c0 = Q6_V_vzero();
+ HVX_Vector v_sum_float_c1 = Q6_V_vzero();
+ HVX_Vector i32 = Q6_Vb_vsplat_R(32);
+
+ uint32_t n_k_tiles = n / 32;
+ for (uint32_t kt = 0; kt < n_k_tiles; kt++) {
+ const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 896);
+ const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1152);
+ const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1152);
+
+ HVX_VectorPair v_sums0, v_sums1;
+ accum_q6_k_32x2(vptr, v_act0, v_act1, i32, &v_sums0, &v_sums1);
+
+ v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, scale_q6_k_32x1(v_sums0, vptr[6], v_act0[8]));
+ v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, scale_q6_k_32x1(v_sums1, vptr[6], v_act1[8]));
+ }
+
+ if (sz0) {
+ hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0)));
+ } else {
+ hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0);
+ }
+ if (sz1) {
+ hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1)));
+ } else {
+ hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1);
+ }
+}
+
static void tiled_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
const uint8_t * restrict tile_ptr = vx;
const uint8_t * restrict y_q = vy;
diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c
index 1b597dcd9..e16cfdcbe 100644
--- a/ggml/src/ggml-hexagon/htp/matmul-ops.c
+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c
@@ -325,8 +325,9 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) {
}
}
-#include "hmx-mm-kernels-tiled.h"
+// hvx kernels first: the HMX Q6_K dequantizer reuses unpack_q6_k_group from there
#include "hvx-mm-kernels-tiled.h"
+#include "hmx-mm-kernels-tiled.h"
#include "hvx-mm-kernels-flat.h"
// Specialized repacked matmul macros
@@ -637,12 +638,14 @@ static void hvx_mm_nx_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, v
MATMUL_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x2, tiled_vec_dot_q4_0_32x1)
MATMUL_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x2, tiled_vec_dot_q4_1_32x1)
MATMUL_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_dot_q8_0_32x1)
+MATMUL_2D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x2, tiled_vec_dot_q6_k_32x1)
MATMUL_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1)
MATMUL_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1)
MATMUL_2D_REPACKED_IMPL(q4_0_flat, 576, flat_vec_dot_q4_0_32x2, flat_vec_dot_q4_0_32x1)
MATMUL_2D_REPACKED_IMPL(q4_1_flat, 640, flat_vec_dot_q4_1_32x2, flat_vec_dot_q4_1_32x1)
MATMUL_2D_REPACKED_IMPL(q8_0_flat, 1088, flat_vec_dot_q8_0_32x2, flat_vec_dot_q8_0_32x1)
+MATMUL_2D_REPACKED_IMPL(q6_k_flat, 896, flat_vec_dot_q6_k_32x2, flat_vec_dot_q6_k_32x1)
MATMUL_2D_REPACKED_IMPL(iq4nl_flat, 576, flat_vec_dot_iq4nl_32x2, flat_vec_dot_iq4nl_32x1)
MATMUL_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x2, flat_vec_dot_mxfp4_32x1)
@@ -737,12 +740,14 @@ static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, vo
MATVEC_2D_REPACKED_IMPL(q4_0, 576, tiled_vec_dot_q4_0_32x1)
MATVEC_2D_REPACKED_IMPL(q4_1, 640, tiled_vec_dot_q4_1_32x1)
MATVEC_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x1)
+MATVEC_2D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x1)
MATVEC_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x1)
MATVEC_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x1)
MATVEC_2D_REPACKED_IMPL(q4_0_flat, 576, flat_vec_dot_q4_0_32x1)
MATVEC_2D_REPACKED_IMPL(q4_1_flat, 640, flat_vec_dot_q4_1_32x1)
MATVEC_2D_REPACKED_IMPL(q8_0_flat, 1088, flat_vec_dot_q8_0_32x1)
+MATVEC_2D_REPACKED_IMPL(q6_k_flat, 896, flat_vec_dot_q6_k_32x1)
MATVEC_2D_REPACKED_IMPL(iq4nl_flat, 576, flat_vec_dot_iq4nl_32x1)
MATVEC_2D_REPACKED_IMPL(mxfp4_flat, 544, flat_vec_dot_mxfp4_32x1)
@@ -1339,6 +1344,7 @@ static int hvx_mm_init_vec_dot(struct htp_mm_context * mmctx, enum htp_data_type
mmctx->vec_dot_32x1 = tiled_vec_dot_q4_0_32x1;
return 0;
case HTP_TYPE_Q4_1:
+ case HTP_TYPE_Q4_K:
mmctx->type = "q4_1_tiled-f32";
mmctx->vec_dot_32x1 = tiled_vec_dot_q4_1_32x1;
return 0;
@@ -1346,6 +1352,10 @@ static int hvx_mm_init_vec_dot(struct htp_mm_context * mmctx, enum htp_data_type
mmctx->type = "q8_0_tiled-f32";
mmctx->vec_dot_32x1 = tiled_vec_dot_q8_0_32x1;
return 0;
+ case HTP_TYPE_Q6_K:
+ mmctx->type = "q6_k_tiled-f32";
+ mmctx->vec_dot_32x1 = tiled_vec_dot_q6_k_32x1;
+ return 0;
case HTP_TYPE_IQ4_NL:
mmctx->type = "iq4nl_tiled-f32";
mmctx->vec_dot_32x1 = tiled_vec_dot_iq4nl_32x1;
@@ -1395,7 +1405,8 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
bool is_repacked = (src0->type == HTP_TYPE_Q4_0 || src0->type == HTP_TYPE_Q4_1 ||
src0->type == HTP_TYPE_Q8_0 || src0->type == HTP_TYPE_IQ4_NL ||
- src0->type == HTP_TYPE_MXFP4);
+ src0->type == HTP_TYPE_MXFP4 || src0->type == HTP_TYPE_Q6_K ||
+ src0->type == HTP_TYPE_Q4_K);
// Compute src0_nrows_per_thread
mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div);
@@ -1419,8 +1430,10 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
if (is_repacked) {
switch (src0->type) {
case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_2d_repacked_q4_0; break;
- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_2d_repacked_q4_1; break;
+ case HTP_TYPE_Q4_1:
+ case HTP_TYPE_Q4_K: matmul_job_func = hvx_mm_2d_repacked_q4_1; break;
case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_2d_repacked_q8_0; break;
+ case HTP_TYPE_Q6_K: matmul_job_func = hvx_mm_2d_repacked_q6_k; break;
case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_2d_repacked_iq4nl; break;
case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_2d_repacked_mxfp4; break;
default: return HTP_STATUS_NO_SUPPORT;
@@ -1432,8 +1445,10 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
if (is_repacked) {
switch (src0->type) {
case HTP_TYPE_Q4_0: matmul_job_func = hvx_mv_2d_repacked_q4_0; break;
- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mv_2d_repacked_q4_1; break;
+ case HTP_TYPE_Q4_1:
+ case HTP_TYPE_Q4_K: matmul_job_func = hvx_mv_2d_repacked_q4_1; break;
case HTP_TYPE_Q8_0: matmul_job_func = hvx_mv_2d_repacked_q8_0; break;
+ case HTP_TYPE_Q6_K: matmul_job_func = hvx_mv_2d_repacked_q6_k; break;
case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mv_2d_repacked_iq4nl; break;
case HTP_TYPE_MXFP4: matmul_job_func = hvx_mv_2d_repacked_mxfp4; break;
default: return HTP_STATUS_NO_SUPPORT;
@@ -1505,14 +1520,16 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
case HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT: {
n_quant_tasks = MIN(src1_nrows, octx->n_threads);
- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat;
- src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat;
+ src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
if (src1_nrows > 1) {
switch (src0->type) {
case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_2d_repacked_q4_0_flat; break;
- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_2d_repacked_q4_1_flat; break;
+ case HTP_TYPE_Q4_1:
+ case HTP_TYPE_Q4_K: matmul_job_func = hvx_mm_2d_repacked_q4_1_flat; break;
case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_2d_repacked_q8_0_flat; break;
+ case HTP_TYPE_Q6_K: matmul_job_func = hvx_mm_2d_repacked_q6_k_flat; break;
case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_2d_repacked_iq4nl_flat; break;
case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_2d_repacked_mxfp4_flat; break;
default: return HTP_STATUS_NO_SUPPORT;
@@ -1520,8 +1537,10 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
} else {
switch (src0->type) {
case HTP_TYPE_Q4_0: matmul_job_func = hvx_mv_2d_repacked_q4_0_flat; break;
- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mv_2d_repacked_q4_1_flat; break;
+ case HTP_TYPE_Q4_1:
+ case HTP_TYPE_Q4_K: matmul_job_func = hvx_mv_2d_repacked_q4_1_flat; break;
case HTP_TYPE_Q8_0: matmul_job_func = hvx_mv_2d_repacked_q8_0_flat; break;
+ case HTP_TYPE_Q6_K: matmul_job_func = hvx_mv_2d_repacked_q6_k_flat; break;
case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mv_2d_repacked_iq4nl_flat; break;
case HTP_TYPE_MXFP4: matmul_job_func = hvx_mv_2d_repacked_mxfp4_flat; break;
default: return HTP_STATUS_NO_SUPPORT;
@@ -1543,7 +1562,7 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
if (src1_nrows < octx->n_threads) {
n_quant_tasks = MIN(total_nb, octx->n_threads);
- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;
+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;
for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) {
uint32_t ib_first = (total_nb * ith) / n_quant_tasks;
uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks;
@@ -1554,9 +1573,9 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
}
} else {
n_quant_tasks = MIN(src1_nrows, octx->n_threads);
- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;
+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;
}
- src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
+ src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
break;
}
@@ -1742,6 +1761,7 @@ DEQUANTIZE_WORKER_LOOP_IMPL(q4_1)
DEQUANTIZE_WORKER_LOOP_IMPL(iq4_nl)
DEQUANTIZE_WORKER_LOOP_IMPL(mxfp4)
DEQUANTIZE_WORKER_LOOP_IMPL(q8_0)
+DEQUANTIZE_WORKER_LOOP_IMPL(q6_k)
static void convert_f16_worker_loop(unsigned int n, unsigned int i, void *data) {
tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data;
@@ -2476,9 +2496,11 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
switch (weight_type) {
case HTP_TYPE_Q4_0: dequant_worker_fn = dequantize_tiled_worker_loop_q4_0; break;
case HTP_TYPE_IQ4_NL: dequant_worker_fn = dequantize_tiled_worker_loop_iq4_nl; break;
- case HTP_TYPE_Q4_1: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break;
+ case HTP_TYPE_Q4_1:
+ case HTP_TYPE_Q4_K: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break;
case HTP_TYPE_MXFP4: dequant_worker_fn = dequantize_tiled_worker_loop_mxfp4; break;
case HTP_TYPE_Q8_0: dequant_worker_fn = dequantize_tiled_worker_loop_q8_0; break;
+ case HTP_TYPE_Q6_K: dequant_worker_fn = dequantize_tiled_worker_loop_q6_k; break;
case HTP_TYPE_F16: dequant_worker_fn = convert_f16_worker_loop; break;
case HTP_TYPE_F32: dequant_worker_fn = quantize_f32_worker_loop; break;
default:
@@ -2732,9 +2754,11 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
switch (weight_type) {
case HTP_TYPE_Q4_0: dequant_worker_fn = dequantize_tiled_worker_loop_q4_0; break;
case HTP_TYPE_IQ4_NL: dequant_worker_fn = dequantize_tiled_worker_loop_iq4_nl; break;
- case HTP_TYPE_Q4_1: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break;
+ case HTP_TYPE_Q4_1:
+ case HTP_TYPE_Q4_K: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break;
case HTP_TYPE_MXFP4: dequant_worker_fn = dequantize_tiled_worker_loop_mxfp4; break;
case HTP_TYPE_Q8_0: dequant_worker_fn = dequantize_tiled_worker_loop_q8_0; break;
+ case HTP_TYPE_Q6_K: dequant_worker_fn = dequantize_tiled_worker_loop_q6_k; break;
case HTP_TYPE_F16: dequant_worker_fn = convert_f16_worker_loop; break;
case HTP_TYPE_F32: dequant_worker_fn = quantize_f32_worker_loop; break;
default:
@@ -3324,9 +3348,11 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
switch (weight_type) {
case HTP_TYPE_Q4_0: dequant_worker_fn = dequantize_tiled_worker_loop_q4_0; break;
case HTP_TYPE_IQ4_NL: dequant_worker_fn = dequantize_tiled_worker_loop_iq4_nl; break;
- case HTP_TYPE_Q4_1: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break;
+ case HTP_TYPE_Q4_1:
+ case HTP_TYPE_Q4_K: dequant_worker_fn = dequantize_tiled_worker_loop_q4_1; break;
case HTP_TYPE_MXFP4: dequant_worker_fn = dequantize_tiled_worker_loop_mxfp4; break;
case HTP_TYPE_Q8_0: dequant_worker_fn = dequantize_tiled_worker_loop_q8_0; break;
+ case HTP_TYPE_Q6_K: dequant_worker_fn = dequantize_tiled_worker_loop_q6_k; break;
case HTP_TYPE_F16: dequant_worker_fn = convert_f16_worker_loop; break;
case HTP_TYPE_F32: dequant_worker_fn = quantize_f32_worker_loop; break;
default:
@@ -3626,7 +3652,7 @@ static int hvx_mm_matmul_id(
uint32_t n_quant_tasks = 1;
if (src1_nrows < octx->n_threads) {
n_quant_tasks = MIN(total_nb, octx->n_threads);
- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;
+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;
for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) {
uint32_t ib_first = (total_nb * ith) / n_quant_tasks;
uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks;
@@ -3637,9 +3663,9 @@ static int hvx_mm_matmul_id(
}
} else {
n_quant_tasks = MIN(src1_nrows, octx->n_threads);
- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;
+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;
}
- size_t src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
+ size_t src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
struct htp_mm_hvx_vtcm_layout L;
htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, src1_nrows, octx->n_threads,
@@ -3773,7 +3799,7 @@ static int hvx_mm_matmul_id_nx(
uint32_t n_quant_tasks = 1;
if (src1_nrows < octx->n_threads) {
n_quant_tasks = MIN(total_nb, octx->n_threads);
- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;
+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;
for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) {
uint32_t ib_first = (total_nb * ith) / n_quant_tasks;
uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks;
@@ -3784,9 +3810,9 @@ static int hvx_mm_matmul_id_nx(
}
} else {
n_quant_tasks = MIN(src1_nrows, octx->n_threads);
- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;
+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;
}
- size_t src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(act->ne[0]) : htp_mm_q8_0_tiled_row_size(act->ne[0]);
+ size_t src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(act->ne[0]) : htp_mm_q8_0_tiled_row_size(act->ne[0]);
struct htp_mm_hvx_vtcm_layout L;
htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], src1_nrows, octx->n_threads,
@@ -4129,7 +4155,7 @@ int op_matmul_nx(struct htp_ops_context * octx) {
bool is_repacked = (src0->type == HTP_TYPE_Q4_0 || src0->type == HTP_TYPE_Q4_1 ||
src0->type == HTP_TYPE_Q8_0 || src0->type == HTP_TYPE_IQ4_NL ||
- src0->type == HTP_TYPE_MXFP4);
+ src0->type == HTP_TYPE_MXFP4 || src0->type == HTP_TYPE_Q4_K);
struct htp_mm_context mmctx_struct = {0};
struct htp_mm_context * mmctx = &mmctx_struct;
@@ -4153,10 +4179,10 @@ int op_matmul_nx(struct htp_ops_context * octx) {
uint32_t n_quant_tasks = 1;
if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) {
n_quant_tasks = MIN(src1_nrows, octx->n_threads);
- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat;
+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_flat : quantize_f32_q8_0_flat;
} else if (src1_nrows < octx->n_threads) {
n_quant_tasks = MIN(total_nb, octx->n_threads);
- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;
+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled_block : quantize_f32_q8_0_tiled_block;
for (uint32_t ith = 0; ith < n_quant_tasks; ++ith) {
uint32_t ib_first = (total_nb * ith) / n_quant_tasks;
uint32_t ib_last = (total_nb * (ith + 1)) / n_quant_tasks;
@@ -4167,14 +4193,14 @@ int op_matmul_nx(struct htp_ops_context * octx) {
}
} else {
n_quant_tasks = MIN(src1_nrows, octx->n_threads);
- quant_task_func = (src0->type == HTP_TYPE_Q4_1) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;
+ quant_task_func = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? quantize_f32_q8_1_tiled : quantize_f32_q8_0_tiled;
}
size_t src1_row_size;
if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) {
- src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(act->ne[0]) : htp_mm_q8_0_flat_row_size(act->ne[0]);
+ src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? htp_mm_q8_1_flat_row_size(act->ne[0]) : htp_mm_q8_0_flat_row_size(act->ne[0]);
} else {
- src1_row_size = (src0->type == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(act->ne[0]) : htp_mm_q8_0_tiled_row_size(act->ne[0]);
+ src1_row_size = (src0->type == HTP_TYPE_Q4_1 || src0->type == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(act->ne[0]) : htp_mm_q8_0_tiled_row_size(act->ne[0]);
}
struct htp_mm_hvx_vtcm_layout L;
@@ -4219,7 +4245,8 @@ int op_matmul_nx(struct htp_ops_context * octx) {
if (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT) {
switch (src0->type) {
case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_nx_2d_repacked_q4_0_flat; break;
- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1_flat; break;
+ case HTP_TYPE_Q4_1:
+ case HTP_TYPE_Q4_K: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1_flat; break;
case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_nx_2d_repacked_q8_0_flat; break;
case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_nx_2d_repacked_iq4nl_flat; break;
case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_nx_2d_repacked_mxfp4_flat; break;
@@ -4228,7 +4255,8 @@ int op_matmul_nx(struct htp_ops_context * octx) {
} else {
switch (src0->type) {
case HTP_TYPE_Q4_0: matmul_job_func = hvx_mm_nx_2d_repacked_q4_0; break;
- case HTP_TYPE_Q4_1: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1; break;
+ case HTP_TYPE_Q4_1:
+ case HTP_TYPE_Q4_K: matmul_job_func = hvx_mm_nx_2d_repacked_q4_1; break;
case HTP_TYPE_Q8_0: matmul_job_func = hvx_mm_nx_2d_repacked_q8_0; break;
case HTP_TYPE_IQ4_NL: matmul_job_func = hvx_mm_nx_2d_repacked_iq4nl; break;
case HTP_TYPE_MXFP4: matmul_job_func = hvx_mm_nx_2d_repacked_mxfp4; break;
diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h
index 2dbcb0c2e..1df8c2933 100644
--- a/ggml/src/ggml-hexagon/htp/matmul-ops.h
+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h
@@ -25,6 +25,11 @@ extern "C" {
#define HTP_MM_WEIGHT_TILE_SIZE_Q8_0 1088
#define HTP_MM_WEIGHT_TILE_SIZE_IQ4_NL 576
#define HTP_MM_WEIGHT_TILE_SIZE_MXFP4 544
+// Q6_K native 6-bit tile (32 rows x 32 k), vrmpy-ready: byte 4*row+b of a vector holds k = 4*group+b
+// vectors 0..3: low nibbles, vector i holds group 2i (low nibble) and group 2i+1 (high nibble)
+// vectors 4..5: high 2 bits, vector m holds groups 4m..4m+3 at bit offsets 0,2,4,6
+// vector 6: fp16 scales per row, d * scales[]: k 0..15 in lanes 0..31, k 16..31 in lanes 32..63
+#define HTP_MM_WEIGHT_TILE_SIZE_Q6_K 896
// --- Weight Repacked Aligned Tile Sizes ---
#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_0 640
@@ -32,6 +37,7 @@ extern "C" {
#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q8_0 1152
#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_IQ4_NL 640
#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_MXFP4 640
+#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q6_K 896
// --- Activation Tiled Block Sizes (including padding) ---
#define HTP_MM_ACT_TILE_SIZE_Q8_0 1152
@@ -195,9 +201,12 @@ static inline uint32_t htp_mm_get_weight_tile_size(int weight_type) {
case HTP_TYPE_IQ4_NL:
return HTP_MM_WEIGHT_TILE_SIZE_Q4_0;
case HTP_TYPE_Q4_1:
+ case HTP_TYPE_Q4_K:
return HTP_MM_WEIGHT_TILE_SIZE_Q4_1;
case HTP_TYPE_Q8_0:
return HTP_MM_WEIGHT_TILE_SIZE_Q8_0;
+ case HTP_TYPE_Q6_K:
+ return HTP_MM_WEIGHT_TILE_SIZE_Q6_K;
case HTP_TYPE_MXFP4:
return HTP_MM_WEIGHT_TILE_SIZE_MXFP4;
default:
@@ -211,9 +220,12 @@ static inline uint32_t htp_mm_get_weight_aligned_tile_size(int weight_type) {
case HTP_TYPE_IQ4_NL:
return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_0;
case HTP_TYPE_Q4_1:
+ case HTP_TYPE_Q4_K:
return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_1;
case HTP_TYPE_Q8_0:
return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q8_0;
+ case HTP_TYPE_Q6_K:
+ return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q6_K;
case HTP_TYPE_MXFP4:
return HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_MXFP4;
default:
@@ -254,7 +266,9 @@ static inline size_t htp_mm_get_tiled_row_stride(int weight_type, uint32_t k) {
case HTP_TYPE_Q4_0:
case HTP_TYPE_IQ4_NL:
case HTP_TYPE_Q4_1:
+ case HTP_TYPE_Q4_K:
case HTP_TYPE_Q8_0:
+ case HTP_TYPE_Q6_K:
case HTP_TYPE_MXFP4:
return (size_t) nb * htp_mm_get_weight_tile_size(weight_type);
case HTP_TYPE_F16:
@@ -484,7 +498,8 @@ static inline void htp_mm_hvx_vtcm_layout_build(
const bool is_repack = (wtype == HTP_TYPE_Q4_0 || wtype == HTP_TYPE_Q4_1 ||
wtype == HTP_TYPE_Q8_0 || wtype == HTP_TYPE_IQ4_NL ||
- wtype == HTP_TYPE_MXFP4);
+ wtype == HTP_TYPE_MXFP4 || wtype == HTP_TYPE_Q6_K ||
+ wtype == HTP_TYPE_Q4_K);
if (is_fused_nx) {
const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128);
@@ -502,8 +517,8 @@ static inline void htp_mm_hvx_vtcm_layout_build(
weight_sz_per_thread = hex_round_up(n_prefetch * src0_row_size_padded, 128);
}
- size_t flat_act_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
- size_t tiled_act_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
+ size_t flat_act_row_size = (wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q4_K) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
+ size_t tiled_act_row_size = (wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
size_t act_sz = (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT)
? hex_round_up(flat_act_row_size * src1_nrows, 128)
@@ -516,8 +531,8 @@ static inline void htp_mm_hvx_vtcm_layout_build(
dst_sz = quant_scratch_size;
} else if (is_matmul_id) {
const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128);
- const size_t src1_row_size_tiled = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10)
- : htp_mm_q8_0_tiled_row_size(ne10);
+ const size_t src1_row_size_tiled = (wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10)
+ : htp_mm_q8_0_tiled_row_size(ne10);
size_t src0_sz_per_thread = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256);
src1_sz = htp_mm_round_up(src1_row_size_tiled * src1_nrows, 256);
@@ -562,7 +577,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(
}
case HTP_MM_KERNEL_HVX_QUANT_BLOCK:
case HTP_MM_KERNEL_HVX_QUANT_ROW: {
- size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
+ size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q4_K) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256);
src1_sz = htp_mm_round_up(q_src1_row_size * src1_nrows, 256);
@@ -584,7 +599,7 @@ static inline void htp_mm_hvx_vtcm_layout_build(
break;
}
case HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT: {
- size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
+ size_t q_src1_row_size = (wtype == HTP_TYPE_Q4_1 || wtype == HTP_TYPE_Q4_K) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10);
src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256);
src1_sz = htp_mm_round_up(q_src1_row_size * src1_nrows, 256);