Commit aa5e0092f for llama.cpp
commit aa5e0092fdb8980b9e89498dfb73049b4fe204c2
Author: kurquhar <kurquhar@qti.qualcomm.com>
Date: Wed Oct 7 17:04:45 2026 -0700
hexagon: support tiled Q4_K and Q6_K GET_ROWS (#30115)
* hexagon: support tiled Q4_K GET_ROWS
Assisted-by: OpenCode
* properly reject Q4_K views
Assisted-by: OpenCode
* hexagon: support tiled Q6_K GET_ROWS
Assisted-by: OpenCode
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index f0d768388..26b79582e 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -7232,7 +7232,7 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *
const struct ggml_tensor * src1 = op->src[1]; // indices
const struct ggml_tensor * dst = op;
- if (src0->type == GGML_TYPE_Q4_0 && src0->view_src) {
+ if ((src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_K || src0->type == GGML_TYPE_Q6_K) && src0->view_src) {
return false;
}
@@ -7241,7 +7241,7 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *
if (src0_base->buffer && ggml_backend_buffer_is_hexagon(src0_base->buffer) && src0_base->extra) {
const auto * extra = (const ggml_hexagon_tensor_extra *) src0_base->extra;
is_repacked = (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0;
- if (is_repacked && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0) {
+ if (is_repacked && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_Q8_0) {
return false;
}
}
@@ -7252,7 +7252,7 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *
return false;
}
- if (src0->type == GGML_TYPE_Q4_0 && src0->buffer && !is_repacked) {
+ if ((src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_K || src0->type == GGML_TYPE_Q6_K) && src0->buffer && ggml_backend_buffer_get_size(src0->buffer) != 0 && !is_repacked) {
return false;
}
@@ -7261,7 +7261,11 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *
}
if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 &&
- src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) {
+ src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) {
+ return false;
+ }
+
+ if ((src0->type == GGML_TYPE_Q4_K || src0->type == GGML_TYPE_Q6_K) && (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || src0->ne[0] % QK_K)) {
return false;
}
@@ -7290,8 +7294,8 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *
return false;
}
- // Q4_0 has no raw fallback. Mark only accepted tensors for repacking.
- if (src0->type == GGML_TYPE_Q4_0 && !src0->buffer) {
+ // Tiled quantized weights have no raw fallback. Mark only accepted tensors for repacking.
+ if ((src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_K || src0->type == GGML_TYPE_Q6_K) && !src0->buffer) {
sess->needs_repack.insert(src0);
}
diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c
index f354b813a..90f38fa56 100644
--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c
+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c
@@ -217,7 +217,7 @@ GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { compute_get_rows_q8_0((float
GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { compute_get_rows_q8_0((float *)dst_spad, src_spad, cur_elems); })
-static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const uint8_t * tile, uint32_t row, bool q4) {
+static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const uint8_t * tile, uint32_t row, bool q4, bool q4_k) {
const HVX_VectorPred first2 = Q6_Q_vsetq_R(2);
const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
HVX_Vector vq = Q6_V_vzero();
@@ -235,7 +235,9 @@ static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const
const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F));
const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4);
vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1));
- vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8));
+ if (!q4_k) {
+ vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8));
+ }
} else {
for (int group = 7; group >= 0; --group) {
const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), 2 * row);
@@ -245,14 +247,46 @@ static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const
}
}
const HVX_Vector scales = hvx_vmem(tile + (q4 ? 512 : 1024));
- const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
+ const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, (q4_k ? 4 : 2) * row));
const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);
const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
- const HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
+ HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
+ if (q4_k) {
+ const HVX_Vector offset_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row + 2));
+ const HVX_Vector offset = Q6_V_lo_W(hvx_vec_f16_to_f32(offset_hf));
+ values = hvx_vec_add_f32_f32(values, offset);
+ }
*(HVX_Vector *) dst = values;
}
+static __attribute__((noinline)) void compute_get_rows_q6_k(float * dst, const uint8_t * tile, uint32_t row) {
+ const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
+ const HVX_VectorPred first16 = Q6_Q_vsetq_R(16 * sizeof(float));
+ const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F);
+ const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
+ HVX_Vector vq = Q6_V_vzero();
+
+ for (int group = 7; group >= 0; --group) {
+ const HVX_Vector lo_plane = Q6_V_vror_VR(hvx_vmem(tile + (group >> 1) * VLEN), 4 * row);
+ const HVX_Vector hi_plane = Q6_V_vror_VR(hvx_vmem(tile + 512 + (group >> 2) * VLEN), 4 * row);
+ const HVX_Vector lo = (group & 1) ? Q6_Vub_vlsr_VubR(lo_plane, 4) : Q6_V_vand_VV(lo_plane, mask_0f);
+ const HVX_Vector hi = Q6_Vub_vlsr_VubR(hi_plane, 2 * (group & 3));
+ const HVX_Vector packed = Q6_V_vor_VV(lo, Q6_Vw_vasl_VwR(Q6_V_vand_VV(hi, mask_03), 4));
+ vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4));
+ }
+
+ const HVX_Vector scales = hvx_vmem(tile + 768);
+ const HVX_Vector scale_lo_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
+ const HVX_Vector scale_hi_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 64 + 2 * row));
+ const HVX_Vector scale_lo = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_lo_hf));
+ const HVX_Vector scale_hi = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hi_hf));
+ const HVX_Vector scale = Q6_V_vmux_QVV(first16, scale_lo, scale_hi);
+ const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(32)));
+ const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
+ *(HVX_Vector *) dst = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
+}
+
struct get_rows_tiled_task {
dma_addr_t tile_src_base;
dma_addr_t dst_data;
@@ -315,7 +349,9 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat
const uint32_t tile_size = grctx->tile_size;
const uint32_t tile_stride = grctx->tile_stride;
const uint32_t dst_bytes = ne00 * sizeof(float);
- const bool is_q4 = (octx->src[0]->type == HTP_TYPE_Q4_0);
+ const bool is_q4 = octx->src[0]->type == HTP_TYPE_Q4_0 || octx->src[0]->type == HTP_TYPE_Q4_K;
+ const bool is_q4_k = octx->src[0]->type == HTP_TYPE_Q4_K;
+ const bool is_q6_k = octx->src[0]->type == HTP_TYPE_Q6_K;
for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, ++spad_idx) {
const uint32_t i = ir0 + step;
@@ -343,7 +379,11 @@ static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * dat
for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
const uint8_t * tile = src_spad + k_tile * tile_stride;
float * dst_block = dst_spad + k_tile * HTP_MM_HMX_TILE_N_COLS;
- compute_get_rows_tiled(dst_block, tile, task.row, is_q4);
+ if (is_q6_k) {
+ compute_get_rows_q6_k(dst_block, tile, task.row);
+ } else {
+ compute_get_rows_tiled(dst_block, tile, task.row, is_q4, is_q4_k);
+ }
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
@@ -369,10 +409,12 @@ int op_get_rows(struct htp_ops_context * octx) {
const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params *) octx->kernel_params;
if (octx->src[0]->type != HTP_TYPE_F32 &&
- octx->src[0]->type != HTP_TYPE_F16 &&
- octx->src[0]->type != HTP_TYPE_Q4_0 &&
- octx->src[0]->type != HTP_TYPE_Q8_0 &&
- octx->src[0]->type != HTP_TYPE_I32) {
+ octx->src[0]->type != HTP_TYPE_F16 &&
+ octx->src[0]->type != HTP_TYPE_Q4_0 &&
+ octx->src[0]->type != HTP_TYPE_Q4_K &&
+ octx->src[0]->type != HTP_TYPE_Q6_K &&
+ octx->src[0]->type != HTP_TYPE_Q8_0 &&
+ octx->src[0]->type != HTP_TYPE_I32) {
return HTP_STATUS_NO_SUPPORT;
}
@@ -426,7 +468,7 @@ int op_get_rows(struct htp_ops_context * octx) {
grctx.task_start = task_start;
grctx.tasks = tasks;
grctx.tasks_per_thread = octx->ctx->mdev.count == 1 ? kparams->tasks_per_thread : fastdiv(tasks + n_threads - 1, &octx->n_threads_div);
- grctx.tile_size = octx->src[0]->type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0;
+ grctx.tile_size = htp_mm_get_weight_tile_size(octx->src[0]->type);
grctx.tile_stride = (grctx.tile_size + 127) & ~127;
grctx.index_i32 = octx->src[1]->type == HTP_TYPE_I32;
diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.h
index 06ca1ea74..69fa54464 100644
--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.h
+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h
@@ -55,7 +55,7 @@ static inline void htp_get_rows_vtcm_layout_build(
}
if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) {
- const size_t tile_size = type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0;
+ const size_t tile_size = htp_mm_get_weight_tile_size(type);
const size_t tile_stride = (tile_size + 127) & ~127;
const uint32_t n_k_tiles = ne00 / HTP_MM_HMX_TILE_N_COLS;
const size_t row_tiles_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride;