Commit 7ac59a6e3 for llama.cpp
commit 7ac59a6e3ad851cd41af00f678effab0598ba9a8
Author: kurquhar <kurquhar@qti.qualcomm.com>
Date: Sat Sep 26 23:02:37 2026 -0700
hexagon: support tiled Q4_0 and Q8_0 GET_ROWS (#29511)
* hexagon: support tiled Q4_0 and Q8_0 GET_ROWS
* hex-get-rows: fix macros
* hex-get-rows: use tiled HVX dequantization
Assisted-by: OpenCode
* hex-get-rows: fix register spills and clean up checks for unsupported ops
* hex-get-rows: improve dma pipeline
* hex-get-rows: improve/simplify kernel selection logic
* hex-build: reenable vectorizer, didnt notice the regression earlier in the sampler update
---------
Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index d95687e34..521a97a73 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -5208,49 +5208,51 @@ static void ggml_hexagon_precompute_get_rows_params(
const uint32_t ne12 = src1->ne[2];
const uint32_t nr = ne10 * ne11 * ne12;
- const size_t nb01 = src0->nb[1];
- const size_t nb1 = dst->nb[1];
-
- const bool can_use_dma = (src0->type == dst->type) && (nb01 == nb1);
- const bool use_dma = can_use_dma && (ne00 >= 2048);
+ const ggml_tensor * src0_base = src0->view_src ? src0->view_src : src0;
+ const auto * extra = src0_base->buffer && ggml_backend_buffer_is_hexagon(src0_base->buffer) ?
+ (const ggml_hexagon_tensor_extra *) src0_base->extra : nullptr;
+ const bool tiled = src0->type == GGML_TYPE_Q4_0 || (extra && (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0) ||
+ sess->needs_repack.count(src0_base) || sess->needs_repack.count(src0);
+
+ if (src0->type == dst->type) {
+ kparams->kernel_type = HTP_GET_ROWS_KERNEL_SAMETYPE;
+ } else if (tiled) {
+ kparams->kernel_type = HTP_GET_ROWS_KERNEL_TILED;
+ } else {
+ kparams->kernel_type = HTP_GET_ROWS_KERNEL_FLAT;
+ }
- kparams->use_dma = use_dma ? 1 : 0;
+ const uint32_t chunks_per_row = 1;
+ const uint32_t chunk_size = ne00;
+ const uint32_t total_tasks = nr;
- uint32_t chunks_per_row = 1;
- uint32_t chunk_size = ne00;
- uint32_t total_tasks = nr;
+ kparams->n_threads = (std::min)((uint32_t)sess->n_threads, total_tasks);
- if (use_dma) {
- kparams->n_threads = (std::min)((uint32_t)sess->n_threads, nr);
- kparams->tasks_per_thread = (nr + kparams->n_threads - 1) / kparams->n_threads;
- } else {
- if (src0->type == GGML_TYPE_F32 && nr < sess->n_threads) {
- const uint32_t min_chunk_size = 1024;
- uint32_t max_chunks = ne00 / min_chunk_size;
- if (max_chunks == 0) {
- max_chunks = 1;
- }
- chunks_per_row = (std::min)((sess->n_threads + nr - 1) / nr, max_chunks);
- chunk_size = (ne00 + chunks_per_row - 1) / chunks_per_row;
- total_tasks = nr * chunks_per_row;
+ struct htp_get_rows_vtcm_layout vtcm_layout = {};
+ while (kparams->n_threads > 0) {
+ htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, src0->type, ne00, kparams->n_threads);
+ if (vtcm_layout.total_bytes <= sess->vtcm_size) {
+ break;
}
- kparams->n_threads = (std::min)(total_tasks, (uint32_t)sess->n_threads);
- kparams->tasks_per_thread = (total_tasks + kparams->n_threads - 1) / kparams->n_threads;
+ --kparams->n_threads;
+ }
+
+ if (kparams->n_threads == 0 && total_tasks > 0) {
+ htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, src0->type, ne00, 1);
}
+ kparams->vtcm_size = (total_tasks == 0) ? 0 : vtcm_layout.total_bytes;
+ kparams->tasks_per_thread = kparams->n_threads > 0 ? (total_tasks + kparams->n_threads - 1) / kparams->n_threads : 0;
+
kparams->chunks_per_row = chunks_per_row;
kparams->chunk_size = chunk_size;
kparams->total_tasks = total_tasks;
- kparams->div_ne10 = init_fastdiv_values(ne10);
- kparams->div_ne10_ne11 = init_fastdiv_values(ne10 * ne11);
- kparams->div_chunks_per_row = init_fastdiv_values(chunks_per_row);
- kparams->div_ne02 = init_fastdiv_values(ne02);
- kparams->div_ne03 = init_fastdiv_values(ne03);
-
- struct htp_get_rows_vtcm_layout vtcm_layout;
- htp_get_rows_vtcm_layout_build(&vtcm_layout, src0->type, ne00, kparams->n_threads);
- kparams->vtcm_size = vtcm_layout.total_bytes;
+ kparams->div_ne10 = ne10 > 0 ? init_fastdiv_values(ne10) : fastdiv_values{0, 0};
+ kparams->div_ne10_ne11 = (ne10 * ne11) > 0 ? init_fastdiv_values(ne10 * ne11) : fastdiv_values{0, 0};
+ kparams->div_chunks_per_row = chunks_per_row > 0 ? init_fastdiv_values(chunks_per_row) : fastdiv_values{0, 0};
+ kparams->div_ne02 = ne02 > 0 ? init_fastdiv_values(ne02) : fastdiv_values{0, 0};
+ kparams->div_ne03 = ne03 > 0 ? init_fastdiv_values(ne03) : fastdiv_values{0, 0};
}
static void ggml_hexagon_precompute_set_rows_params(
@@ -6176,19 +6178,36 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *
const struct ggml_tensor * src1 = op->src[1]; // indices
const struct ggml_tensor * dst = op;
- if (src0->extra) {
- const auto * extra = (const ggml_hexagon_tensor_extra *) src0->extra;
- if (extra->flags & GGML_HEXAGON_TENSOR_REPACK) {
+ if (src0->type == GGML_TYPE_Q4_0 && src0->view_src) {
+ return false;
+ }
+
+ const ggml_tensor * src0_base = src0->view_src ? src0->view_src : src0;
+ bool is_repacked = false;
+ if (src0_base->buffer && ggml_backend_buffer_is_hexagon(src0_base->buffer) && src0_base->extra) {
+ const auto * extra = (const ggml_hexagon_tensor_extra *) src0_base->extra;
+ is_repacked = (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0;
+ if (is_repacked && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0) {
return false;
}
}
+ is_repacked = is_repacked || sess->needs_repack.count(src0_base) || sess->needs_repack.count(src0);
- if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->ne[0] < 32) {
+ // View offsets use the raw quantized layout and cannot address a tiled allocation.
+ if (src0->view_src && is_repacked) {
+ return false;
+ }
+
+ if (src0->type == GGML_TYPE_Q4_0 && src0->buffer && !is_repacked) {
+ return false;
+ }
+
+ if (src0->type != dst->type && src0->ne[0] < 32) {
return false;
}
if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 &&
- src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) {
+ src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) {
return false;
}
@@ -6196,15 +6215,32 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *
return false;
}
- if (src0->type == GGML_TYPE_I32) {
- if (dst->type != GGML_TYPE_I32) {
+ if (src0->type == dst->type) {
+ if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->type != GGML_TYPE_F16) {
return false;
}
+ } else if (src0->type == GGML_TYPE_I32) {
+ return false;
+ } else if (dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+
+ // Empty recurrent-state gathers are skipped at execution; do not split the graph for them.
+ if (ggml_is_empty(op)) {
+ return true;
}
- else if (dst->type != GGML_TYPE_F32) {
+
+ struct htp_get_rows_kernel_params kparams;
+ ggml_hexagon_precompute_get_rows_params(sess, src0, src1, dst, &kparams);
+ if (kparams.n_threads == 0 || (size_t) kparams.vtcm_size > sess->vtcm_size) {
return false;
}
+ // Q4_0 has no raw fallback. Mark only accepted tensors for repacking.
+ if (src0->type == GGML_TYPE_Q4_0 && !src0->buffer) {
+ sess->needs_repack.insert(src0);
+ }
+
return true;
GGML_UNUSED(sess);
diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.h
index e3677b267..8ff38545c 100644
--- a/ggml/src/ggml-hexagon/htp-opnode.h
+++ b/ggml/src/ggml-hexagon/htp-opnode.h
@@ -20,6 +20,7 @@
#include "htp/gated-delta-net-ops.h"
#include "htp/softmax-ops.h"
#include "htp/argsort-ops.h"
+#include "htp/get-rows-ops.h"
struct htp_opnode {
ggml_tensor * node { nullptr };
@@ -360,9 +361,7 @@ struct htp_opformat {
} else if (node.opcode == HTP_OP_GATED_DELTA_NET) {
const auto * kparams = (const struct htp_gdn_kernel_params *) node.kernel_params;
const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent";
- snprintf(str, max_size, "%s-%s vtcm %u",
- path,
- kparams->kda ? "kda" : "scalar",
+ snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar",
(unsigned int) (kparams->vtcm_size ? kparams->vtcm_size : kparams->vtcm_per_thread * kparams->n_threads));
} else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID ||
node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {
@@ -374,9 +373,19 @@ struct htp_opformat {
node.opcode == HTP_OP_TOP_K ? "top_k" : "argsort",
(int) kparams->n_threads, (int) kparams->n_chunks,
(int) kparams->chunk_elems, (int) kparams->vtcm_size);
+ } else if (node.opcode == HTP_OP_GET_ROWS) {
+ const auto * kparams = (const struct htp_get_rows_kernel_params *) node.kernel_params;
+ const char * ktype_str = "unknown";
+ switch (kparams->kernel_type) {
+ case HTP_GET_ROWS_KERNEL_SAMETYPE: ktype_str = "sametype"; break;
+ case HTP_GET_ROWS_KERNEL_TILED: ktype_str = "tiled"; break;
+ case HTP_GET_ROWS_KERNEL_FLAT: ktype_str = "flat"; break;
+ }
+ snprintf(str, max_size, "%s%s vtcm %u", ktype_str, kparams->n_threads > 1 ? "-multi" : "", (unsigned int) kparams->vtcm_size);
} else {
snprintf(str, max_size, "----");
}
+
}
void format(const htp_opnode & node) {
diff --git a/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake b/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake
index 10840185f..3eff2a398 100644
--- a/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake
+++ b/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake
@@ -136,7 +136,7 @@ set(CMAKE_SHARED_LIBRARY_SONAME_C_FLAG "-Wl,-soname,")
set(CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG "-Wl,-soname,")
# Compiler Options
-set(COMMON_FLAGS "${ARCH_FLAGS} -fno-vectorize -fno-slp-vectorize -flto -Wall -Werror -fno-zero-initialized-in-bss -G0 -fdata-sections -fpic ${XQF_ARGS}")
+set(COMMON_FLAGS "${ARCH_FLAGS} -fvectorize -flto -Wall -Werror -fno-zero-initialized-in-bss -G0 -fdata-sections -fpic ${XQF_ARGS}")
set(CMAKE_CXX_FLAGS_DEBUG "${COMMON_FLAGS} -O0 -D_DEBUG -g")
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${COMMON_FLAGS} -O2 -g")
diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c
index f51e00c15..f354b813a 100644
--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c
+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c
@@ -17,6 +17,7 @@
#include "htp-tensor.h"
#include "hvx-utils.h"
#include "hvx-quant.h"
+#include "matmul-ops.h"
#include "get-rows-ops.h"
#include "work-queue.h"
@@ -28,6 +29,9 @@ struct get_rows_context {
uint32_t task_start;
uint32_t tasks;
uint32_t tasks_per_thread;
+ uint32_t tile_size;
+ uint32_t tile_stride;
+ bool index_i32;
};
#define get_rows_preamble \
@@ -195,32 +199,191 @@ static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned
dma_queue_flush(dma_q); \
}
-#define F32_BYTES(n) ((n) * sizeof(float))
#define F16_BYTES(n) ((n) * sizeof(__fp16))
#define Q8_0_BYTES(n) (((n) / 32) * sizeof(block_q8_0))
-GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int32_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); })
-GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int64_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); })
+static __attribute__((noinline)) void compute_get_rows_f16(float * dst_spad, const void * src_spad, uint32_t cur_elems) {
+ hvx_dequantize_row_f16_f32(dst_spad, src_spad, cur_elems);
+}
+
+static __attribute__((noinline)) void compute_get_rows_q8_0(float * dst_spad, const void * src_spad, uint32_t cur_elems) {
+ hvx_dequantize_row_q8_0_f32(dst_spad, src_spad, cur_elems);
+}
+
+GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int32_t, { compute_get_rows_f16((float *)dst_spad, src_spad, cur_elems); })
+GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int64_t, { compute_get_rows_f16((float *)dst_spad, src_spad, cur_elems); })
+
+GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { compute_get_rows_q8_0((float *)dst_spad, src_spad, cur_elems); })
+GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { compute_get_rows_q8_0((float *)dst_spad, src_spad, cur_elems); })
+
+
+static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const uint8_t * tile, uint32_t row, bool q4) {
+ const HVX_VectorPred first2 = Q6_Q_vsetq_R(2);
+ const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
+ HVX_Vector vq = Q6_V_vzero();
+ if (q4) {
+ const HVX_VectorPred first1 = Q6_Q_vsetq_R(1);
+ const HVX_VectorPred first3 = Q6_Q_vsetq_R(3);
+ for (int group = 3; group >= 0; --group) {
+ const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), row);
+ // Four planes contribute bytes at 0, 32, 64 and 96 after rotation.
+ HVX_Vector packed = Q6_V_vmux_QVV(first1, v, Q6_V_vror_VR(v, 31));
+ packed = Q6_V_vmux_QVV(first2, packed, Q6_V_vror_VR(v, 62));
+ packed = Q6_V_vmux_QVV(first3, packed, Q6_V_vror_VR(v, 93));
+ vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4));
+ }
+ const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F));
+ const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4);
+ vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1));
+ vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8));
+ } else {
+ for (int group = 7; group >= 0; --group) {
+ const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), 2 * row);
+ // Two planes contribute halfwords at 0 and 64 after rotation.
+ const HVX_Vector packed = Q6_V_vmux_QVV(first2, v, Q6_V_vror_VR(v, 62));
+ vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4));
+ }
+ }
+ const HVX_Vector scales = hvx_vmem(tile + (q4 ? 512 : 1024));
+ const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
+ const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
+ const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);
+ const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
+ const HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
+ *(HVX_Vector *) dst = values;
+}
+
+struct get_rows_tiled_task {
+ dma_addr_t tile_src_base;
+ dma_addr_t dst_data;
+ uint32_t row;
+};
+
+static inline struct get_rows_tiled_task get_rows_tiled_calc_task(
+ const struct htp_ops_context * octx,
+ const struct get_rows_context * grctx,
+ uint32_t i,
+ uint32_t n_k_tiles,
+ uint32_t tile_size
+) {
+ const struct htp_get_rows_kernel_params * kparams = grctx->kparams;
+ get_rows_preamble;
+
+ const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11);
+ const uint32_t rem = i - i12 * ne11 * ne10;
+ const uint32_t i11 = fastdiv(rem, &kparams->div_ne10);
+ const uint32_t i10 = rem - i11 * ne10;
+ const dma_addr_t src1_data = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12;
+ const uint32_t i01 = grctx->index_i32 ? *(const int32_t *)(uintptr_t) src1_data : (uint32_t) *(const int64_t *)(uintptr_t) src1_data;
+ assert(i01 < ne01);
+
+ const uint32_t q02 = fastdiv(i11, &kparams->div_ne02);
+ const uint32_t i02 = i11 - q02 * ne02;
+ const uint32_t q03 = fastdiv(i12, &kparams->div_ne03);
+ const uint32_t i03 = i12 - q03 * ne03;
+ const uint32_t column_tile = i01 / HTP_MM_HMX_TILE_N_ROWS;
+ const uint32_t row = i01 % HTP_MM_HMX_TILE_N_ROWS;
+ const dma_addr_t matrix = octx->src[0]->data + i02*nb02 + i03*nb03;
+
+ struct get_rows_tiled_task task;
+ task.tile_src_base = matrix + (column_tile * n_k_tiles) * tile_size;
+ task.dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3;
+ task.row = row;
+ return task;
+}
+
+static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * data) {
+ struct get_rows_context * grctx = (struct get_rows_context *) data;
+ struct htp_ops_context * octx = grctx->octx;
+ const struct htp_get_rows_kernel_params * kparams = grctx->kparams;
+ get_rows_preamble;
+
+ const uint32_t dr = grctx->tasks_per_thread;
+ const uint32_t ir0 = grctx->task_start + dr * ith;
+ if (ir0 >= grctx->task_start + grctx->tasks) {
+ return;
+ }
+
+ const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);
+ const uint32_t n_k_tiles = ne00 / HTP_MM_HMX_TILE_N_COLS;
+ const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout;
+ uint8_t * src_spad_base = grctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread;
+ uint8_t * dst_spad_base = grctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread;
+ dma_queue * dma_q = octx->ctx->dma[ith];
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];
-GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int32_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); })
-GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int64_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); })
+ const uint32_t tile_size = grctx->tile_size;
+ const uint32_t tile_stride = grctx->tile_stride;
+ const uint32_t dst_bytes = ne00 * sizeof(float);
+ const bool is_q4 = (octx->src[0]->type == HTP_TYPE_Q4_0);
-GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); })
-GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); })
+ for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, ++spad_idx) {
+ const uint32_t i = ir0 + step;
+ struct get_rows_tiled_task task = get_rows_tiled_calc_task(octx, grctx, i, n_k_tiles, tile_size);
+
+ // Dummy writeback to prime the queue with dst descriptor
+ dma_queue_push(dma_q,
+ dma_make_data(task.dst_data, dst_spad_base + spad_idx * vtcm_layout->dst_spad_half_size),
+ dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 0);
+
+ // Prefetch row tiles
+ dma_queue_push(dma_q,
+ dma_make_data(src_spad_base + spad_idx * vtcm_layout->src0_spad_half_size, task.tile_src_base),
+ tile_stride, tile_size, tile_size, n_k_tiles);
+ }
+
+ for (uint32_t step = 0; step < ir1 - ir0; ++step) {
+ const uint32_t i = ir0 + step;
+ float * dst_spad = (float *) dma_queue_pop(dma_q).src;
+ uint8_t * src_spad = (uint8_t *) dma_queue_pop(dma_q).dst;
+
+ struct get_rows_tiled_task task = get_rows_tiled_calc_task(octx, grctx, i, n_k_tiles, tile_size);
+
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
+ for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
+ const uint8_t * tile = src_spad + k_tile * tile_stride;
+ float * dst_block = dst_spad + k_tile * HTP_MM_HMX_TILE_N_COLS;
+ compute_get_rows_tiled(dst_block, tile, task.row, is_q4);
+ }
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
+
+ // Real writeback of dst_spad
+ dma_queue_push(dma_q,
+ dma_make_data(task.dst_data, dst_spad),
+ dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 1);
+
+ const uint32_t next_step = step + 2;
+ if (next_step < ir1 - ir0) {
+ const uint32_t ni = ir0 + next_step;
+ struct get_rows_tiled_task next_task = get_rows_tiled_calc_task(octx, grctx, ni, n_k_tiles, tile_size);
+ dma_queue_push(dma_q,
+ dma_make_data(src_spad, next_task.tile_src_base),
+ tile_stride, tile_size, tile_size, n_k_tiles);
+ }
+ }
+
+ dma_queue_flush(dma_q);
+}
int op_get_rows(struct htp_ops_context * octx) {
const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params *) octx->kernel_params;
if (octx->src[0]->type != HTP_TYPE_F32 &&
- octx->src[0]->type != HTP_TYPE_F16 &&
- octx->src[0]->type != HTP_TYPE_Q8_0 &&
- octx->src[0]->type != HTP_TYPE_I32) {
+ octx->src[0]->type != HTP_TYPE_F16 &&
+ octx->src[0]->type != HTP_TYPE_Q4_0 &&
+ octx->src[0]->type != HTP_TYPE_Q8_0 &&
+ octx->src[0]->type != HTP_TYPE_I32) {
return HTP_STATUS_NO_SUPPORT;
}
- if ((octx->src[0]->type == HTP_TYPE_I32 && octx->dst->type != HTP_TYPE_I32) ||
- (octx->src[0]->type != HTP_TYPE_I32 && octx->dst->type != HTP_TYPE_F32)) {
- return HTP_STATUS_NO_SUPPORT;
+ if (kparams->kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) {
+ if (octx->src[0]->type != octx->dst->type) {
+ return HTP_STATUS_NO_SUPPORT;
+ }
+ } else {
+ if (octx->dst->type != HTP_TYPE_F32) {
+ return HTP_STATUS_NO_SUPPORT;
+ }
}
if (octx->src[1]->type != HTP_TYPE_I32 && octx->src[1]->type != HTP_TYPE_I64) {
@@ -262,33 +425,48 @@ int op_get_rows(struct htp_ops_context * octx) {
grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base;
grctx.task_start = task_start;
grctx.tasks = tasks;
- grctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div);
+ grctx.tasks_per_thread = octx->ctx->mdev.count == 1 ? kparams->tasks_per_thread : fastdiv(tasks + n_threads - 1, &octx->n_threads_div);
+ grctx.tile_size = octx->src[0]->type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0;
+ grctx.tile_stride = (grctx.tile_size + 127) & ~127;
+ grctx.index_i32 = octx->src[1]->type == HTP_TYPE_I32;
const uint32_t ne00 = octx->src[0]->ne[0];
- htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, n_threads);
+ htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, n_threads);
+
+ if (grctx.vtcm_layout.total_bytes > octx->ctx->vtcm_size) {
+ FARF(ERROR, "get-rows: VTCM reservation %zu is too small, needed %zu\n",
+ octx->ctx->vtcm_size, grctx.vtcm_layout.total_bytes);
+ return HTP_STATUS_INVAL_PARAMS;
+ }
const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
work_queue_func_t q_func = NULL;
- if (kparams->use_dma) {
- q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t);
- } else {
- switch (octx->src[0]->type) {
- case HTP_TYPE_F32: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f32_int32_t : get_rows_thread_f32_int64_t); break;
- case HTP_TYPE_F16: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f16_int32_t : get_rows_thread_f16_int64_t); break;
- case HTP_TYPE_Q8_0: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_q8_0_int32_t : get_rows_thread_q8_0_int64_t); break;
- case HTP_TYPE_I32: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t); break;
- default: return HTP_STATUS_NO_SUPPORT;
- }
+ switch (kparams->kernel_type) {
+ case HTP_GET_ROWS_KERNEL_SAMETYPE:
+ q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t);
+ break;
+ case HTP_GET_ROWS_KERNEL_TILED:
+ q_func = get_rows_thread_tiled;
+ break;
+ case HTP_GET_ROWS_KERNEL_FLAT:
+ switch (octx->src[0]->type) {
+ case HTP_TYPE_F16: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f16_int32_t : get_rows_thread_f16_int64_t); break;
+ case HTP_TYPE_Q8_0: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_q8_0_int32_t : get_rows_thread_q8_0_int64_t); break;
+ default: return HTP_STATUS_NO_SUPPORT;
+ }
+ break;
+ default:
+ return HTP_STATUS_NO_SUPPORT;
}
- FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu use-dma %d n-threads %d\n",
+ FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu kernel-type %d n-threads %d\n",
octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3],
octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3],
octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3],
grctx.vtcm_layout.src0_bytes_per_thread * n_threads,
grctx.vtcm_layout.dst_bytes_per_thread * n_threads,
- kparams->use_dma, n_threads);
+ kparams->kernel_type, n_threads);
work_queue_run(octx->ctx->work_queue, q_func, &grctx, n_threads);
return HTP_STATUS_OK;
diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.h
index 0e7c2ca8c..06ca1ea74 100644
--- a/ggml/src/ggml-hexagon/htp/get-rows-ops.h
+++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h
@@ -1,11 +1,21 @@
#ifndef HTP_GET_ROWS_OPS_H
#define HTP_GET_ROWS_OPS_H
+#include <stdbool.h>
+#include <string.h>
+
#include "hex-fastdiv.h"
+#include "matmul-ops.h"
+
+enum htp_get_rows_kernel_type {
+ HTP_GET_ROWS_KERNEL_SAMETYPE = 0,
+ HTP_GET_ROWS_KERNEL_TILED,
+ HTP_GET_ROWS_KERNEL_FLAT,
+};
struct htp_get_rows_kernel_params {
int32_t n_threads;
- int32_t use_dma;
+ int32_t kernel_type;
int32_t chunks_per_row;
int32_t chunk_size;
int32_t total_tasks;
@@ -34,19 +44,37 @@ struct htp_get_rows_vtcm_layout {
static inline void htp_get_rows_vtcm_layout_build(
struct htp_get_rows_vtcm_layout * vtcm_layout,
+ int kernel_type,
int type,
uint32_t ne00,
uint32_t n_threads) {
+ if (kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) {
+ memset(vtcm_layout, 0, sizeof(*vtcm_layout));
+ return;
+ }
+
+ if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) {
+ const size_t tile_size = type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0;
+ const size_t tile_stride = (tile_size + 127) & ~127;
+ const uint32_t n_k_tiles = ne00 / HTP_MM_HMX_TILE_N_COLS;
+ const size_t row_tiles_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride;
+ vtcm_layout->src0_spad_half_size = (row_tiles_size + 255) & ~255;
+ vtcm_layout->dst_spad_half_size = (ne00 * sizeof(float) + 255) & ~255;
+ vtcm_layout->src0_bytes_per_thread = 2 * vtcm_layout->src0_spad_half_size;
+ vtcm_layout->dst_bytes_per_thread = 2 * vtcm_layout->dst_spad_half_size;
+ vtcm_layout->off_src0 = 0;
+ vtcm_layout->off_dst = vtcm_layout->src0_bytes_per_thread * n_threads;
+ vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads;
+ return;
+ }
+
uint32_t src0_row_size = 0;
switch (type) {
- case 0: // HTP_TYPE_F32
- src0_row_size = ne00 * 4;
- break;
- case 1: // HTP_TYPE_F16
+ case HTP_TYPE_F16:
src0_row_size = ne00 * 2;
break;
- case 8: // HTP_TYPE_Q8_0
+ case HTP_TYPE_Q8_0:
src0_row_size = (ne00 / 32) * 34;
break;
default: