Commit dcd387a41 for llama.cpp
commit dcd387a412ca54e172a8d60eb71ef6753850c8ca
Author: Yiwei Shao <44545837+njsyw1997@users.noreply.github.com>
Date: Thu Oct 1 08:38:37 2026 -0700
hexagon: shared strided DMA copy for CPY and CONCAT, any-dim CONCAT via DMA (#29685)
* hexagon: shared strided DMA copy for CPY and CONCAT, any-dim CONCAT via DMA
* hex-cpy: various fixes on top of the concat optimizations
Removed CONCAT_DMA_MIN_ROW logic, it was broken with 64-bit DMA.
While it's kinda silly to use DMA for tiny stuff if that tensor gets mapped to an extended buffer the only way to read it is DMA.
Added missing dma_queue_flush() calls.
Added additional guards for conditions we don't support.
---------
Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c
index 1bb9ce5e3..4dc146393 100644
--- a/ggml/src/ggml-hexagon/htp/concat-ops.c
+++ b/ggml/src/ggml-hexagon/htp/concat-ops.c
@@ -1,15 +1,17 @@
+#include "dma-queue.h"
#include "hex-common.h"
+#include "hex-cpy-dma.h"
+#include "hex-fastdiv.h"
#include "hex-profile.h"
+#include "hexagon_protos.h"
+#include "hexagon_types.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
-#include "hexagon_types.h"
-#include "hexagon_protos.h"
-#include "hvx_hexagon_protos.h"
-#include "dma-queue.h"
#include "htp-vtcm.h"
#include "hvx-utils.h"
-#include "hex-fastdiv.h"
+#include "hvx_hexagon_protos.h"
+
#include <string.h>
struct htp_concat_context {
@@ -285,73 +287,63 @@ static void concat_generic(unsigned int nth, unsigned int ith, void * data) {
}
}
-static bool concat_dim1_contiguous_dma(struct htp_ops_context * octx, int dim, uint32_t type_size) {
+static bool concat_dma(struct htp_ops_context * octx, int dim, uint32_t type_size) {
+ if (dim < 0 || dim >= HTP_OP_MAX_DIMS) {
+ return false;
+ }
+
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
- if (dim != 1 || octx->ctx->mdev.count > 1 ||
+ // Not partitioned across devices: the row/element-split paths handle that.
+ if (octx->ctx->mdev.count > 1 ||
(dst->type != HTP_TYPE_F32 && dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_I32) ||
- src0->type != dst->type || src1->type != dst->type ||
- src0->ne[0] != dst->ne[0] || src1->ne[0] != dst->ne[0] ||
- src0->ne[2] != dst->ne[2] || src1->ne[2] != dst->ne[2] ||
- src0->ne[3] != dst->ne[3] || src1->ne[3] != dst->ne[3] ||
- dst->ne[1] != src0->ne[1] + src1->ne[1] ||
- !htp_tensor_is_contiguous(src0, type_size) ||
- !htp_tensor_is_contiguous(src1, type_size) ||
- !htp_tensor_is_contiguous(dst, type_size)) {
+ src0->type != dst->type || src1->type != dst->type || src0->nb[0] != type_size || src1->nb[0] != type_size ||
+ dst->nb[0] != type_size || (size_t) dst->ne[0] * type_size > DMA_MAX_SIZE_24B ||
+ dst->nb[1] > DMA_MAX_STRIDE_24B || src0->nb[1] > DMA_MAX_STRIDE_24B || src1->nb[1] > DMA_MAX_STRIDE_24B) {
return false;
}
- const uint32_t src0_row_size = src0->ne[0] * type_size;
- const uint32_t src1_row_size = src1->ne[0] * type_size;
+ for (int d = 0; d < HTP_OP_MAX_DIMS; d++) {
+ const uint32_t ne_d = (d == dim) ? src0->ne[d] + src1->ne[d] : src0->ne[d];
+ if (dst->ne[d] != ne_d || (d != dim && src1->ne[d] != dst->ne[d])) {
+ return false;
+ }
+ }
- // v75+ dma_queue_push() writes a 2D descriptor directly and does not split overflow.
-#if __HVX_ARCH__ >= 75
- if (src0_row_size > 0xffffffu || src1_row_size > 0xffffffu ||
- src0->nb[1] > 0xffffffu || src1->nb[1] > 0xffffffu || dst->nb[1] > 0xffffffu ||
- src0->ne[1] > UINT16_MAX || src1->ne[1] > UINT16_MAX) {
- return false;
+ // The two views of dst, shaped like the sources.
+ struct htp_tensor view0 = *dst;
+ struct htp_tensor view1 = *dst;
+ for (int d = 0; d < HTP_OP_MAX_DIMS; d++) {
+ view0.ne[d] = src0->ne[d];
+ view1.ne[d] = src1->ne[d];
}
-#endif
+ view1.data += (uint64_t) src0->ne[dim] * dst->nb[dim];
dma_queue * q = octx->ctx->dma[0];
- for (uint32_t i3 = 0; i3 < dst->ne[3]; ++i3) {
- for (uint32_t i2 = 0; i2 < dst->ne[2]; ++i2) {
- dma_addr_t dst_addr = dst->data + i3 * dst->nb[3] + i2 * dst->nb[2];
- dma_addr_t src0_addr = src0->data + i3 * src0->nb[3] + i2 * src0->nb[2];
- dma_addr_t src1_addr = src1->data + i3 * src1->nb[3] + i2 * src1->nb[2];
-
- if (!dma_queue_push(q, dma_make_data(dst_addr, src0_addr), dst->nb[1], src0->nb[1], src0_row_size, src0->ne[1])) {
- dma_queue_flush(q);
- dma_queue_push(q, dma_make_data(dst_addr, src0_addr), dst->nb[1], src0->nb[1], src0_row_size, src0->ne[1]);
- }
-
- dst_addr += src0->ne[1] * dst->nb[1];
- if (!dma_queue_push(q, dma_make_data(dst_addr, src1_addr), dst->nb[1], src1->nb[1], src1_row_size, src1->ne[1])) {
- dma_queue_flush(q);
- dma_queue_push(q, dma_make_data(dst_addr, src1_addr), dst->nb[1], src1->nb[1], src1_row_size, src1->ne[1]);
- }
- }
- }
-
+ cpy_dma_sametype_sameshape(q, &view0, src0, type_size);
+ cpy_dma_sametype_sameshape(q, &view1, src1, type_size);
dma_queue_flush(q);
return true;
}
int op_concat(struct htp_ops_context * octx) {
+ int dim = octx->op_params[0];
+ if (dim < 0 || dim >= HTP_OP_MAX_DIMS) {
+ return HTP_STATUS_NO_SUPPORT;
+ }
+
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
- int dim = octx->op_params[0];
-
const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;
bool is_src1_transposed = (src1->nb[0] > src1->nb[1]);
bool is_src0_transposed = (src0->nb[0] > src0->nb[1]);
- if (concat_dim1_contiguous_dma(octx, dim, type_size)) {
+ if (concat_dma(octx, dim, type_size)) {
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c
index aef3c0453..df38d3eed 100644
--- a/ggml/src/ggml-hexagon/htp/cpy-ops.c
+++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c
@@ -11,12 +11,12 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
+#include "hex-cpy-dma.h"
#include "htp-ctx.h"
+#include "htp-fence.h"
#include "htp-ops.h"
-#include "htp-ops.h"
-#include "hvx-utils.h"
#include "htp-tensor.h"
-#include "htp-fence.h"
+#include "hvx-utils.h"
struct htp_copy_context {
struct htp_ops_context * octx;
@@ -49,30 +49,6 @@ struct htp_copy_context {
struct fastdiv_values div_ne02_ne01_ne00;
};
-static inline void cpy_dma_sametype_reshape_contig(
- dma_queue * dma_q,
- dma_addr_t dst,
- dma_addr_t src0,
- uint32_t total_bytes
-) {
- if (total_bytes == 0) {
- return;
- }
-
- const uint32_t max_chunk = DMA_SAFE_CHUNK_SIZE;
- while (total_bytes > 0) {
- const uint32_t chunk = MIN(total_bytes, max_chunk);
- if (!dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/ 1)) {
- dma_queue_flush(dma_q);
- dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/ 1);
- }
- dst += chunk;
- src0 += chunk;
- total_bytes -= chunk;
- }
- dma_queue_flush(dma_q);
-}
-
#define cpy_preamble \
const struct htp_tensor *src0 = octx->src[0]; \
const struct htp_tensor *dst = octx->dst; \
@@ -112,6 +88,7 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
dma_addr_t dst_addr = dst->data + ir0 * ne00 * ELEM_SIZE; \
dma_addr_t src0_addr = src0->data + ir0 * ne00 * ELEM_SIZE; \
cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (ir1 - ir0) * ne00 * ELEM_SIZE); \
+ dma_queue_flush(dma_q); \
return; \
} \
const uint32_t ne02_ne01 = ne02 * ne01; \
@@ -157,6 +134,7 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void
dma_addr_t dst_addr = dst->data + th_start * ELEM_SIZE; \
dma_addr_t src0_addr = src0->data + th_start * ELEM_SIZE; \
cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (th_end - th_start) * ELEM_SIZE); \
+ dma_queue_flush(dma_q); \
return; \
} \
\
@@ -381,67 +359,6 @@ static void cpy_thread_f32_i32_sameshape(unsigned int nth, unsigned int ith, voi
}
}
-static inline void cpy_dma_push_2d_chunked(
- dma_queue * dma_q,
- dma_addr_t dst,
- dma_addr_t src,
- size_t dst_stride,
- size_t src_stride,
- size_t row_size,
- uint32_t nrows
-) {
- while (nrows > 0) {
- const uint32_t cur_rows = MIN(nrows, DMA_MAX_NROWS);
- if (!dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows)) {
- dma_queue_flush(dma_q);
- dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows);
- }
- dst += cur_rows * dst_stride;
- src += cur_rows * src_stride;
- nrows -= cur_rows;
- }
-}
-
-static inline void cpy_dma_sametype_sameshape(
- struct htp_ops_context * octx,
- const struct htp_tensor * dst,
- const struct htp_tensor * src0,
- uint32_t elem_size,
- uint32_t ne00, uint32_t ne01, uint32_t ne02, uint32_t ne03,
- uint32_t nb01, uint32_t nb02, uint32_t nb03,
- uint32_t nb1, uint32_t nb2, uint32_t nb3
-) {
- const bool contiguous = htp_tensor_is_contiguous(src0, elem_size) && htp_tensor_is_contiguous(dst, elem_size);
-
- dma_queue * dma_q = octx->ctx->dma[0];
-
- if (contiguous) {
- cpy_dma_sametype_reshape_contig(dma_q, dst->data, src0->data, ne00 * elem_size * ne01 * ne02 * ne03);
- return;
- }
-
- const bool contiguous_outer =
- (ne02 == 1 || (nb02 == ne01 * nb01 && nb2 == ne01 * nb1)) &&
- (ne03 == 1 || (nb03 == ne02 * nb02 && nb3 == ne02 * nb2));
-
- if (contiguous_outer) {
- uint32_t total_rows = ne01 * ne02 * ne03;
- cpy_dma_push_2d_chunked(dma_q, dst->data, src0->data, nb1, nb01, ne00 * elem_size, total_rows);
- dma_queue_flush(dma_q);
- return;
- }
-
- for (uint32_t i03 = 0; i03 < ne03; i03++) {
- for (uint32_t i02 = 0; i02 < ne02; i02++) {
- dma_addr_t dst_data = dst->data + i02 * nb2 + i03 * nb3;
- dma_addr_t src0_data = src0->data + i02 * nb02 + i03 * nb03;
- cpy_dma_push_2d_chunked(dma_q, dst_data, src0_data, nb1, nb01, ne00 * elem_size, ne01);
- }
- }
-
- dma_queue_flush(dma_q);
-}
-
static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {
cpy_preamble;
*use_dma = false;
@@ -551,7 +468,8 @@ static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {
if (sametype && (octx->ctx->mdev.count <= 1 || htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {
if (octx->ctx->mdev.idx == 0) {
*use_dma = true;
- cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3);
+ cpy_dma_sametype_sameshape(octx->ctx->dma[0], dst, src0, ct.src0_type_size);
+ dma_queue_flush(octx->ctx->dma[0]);
}
} else {
work_queue_func_t copy_fun = NULL;
@@ -582,6 +500,7 @@ static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {
if (octx->ctx->mdev.count <= 1 && dst_is_contiguous && src_is_contiguous) {
*use_dma = true;
cpy_dma_sametype_reshape_contig(octx->ctx->dma[0], dst->data, src0->data, total_elems * ct.dst_type_size);
+ dma_queue_flush(octx->ctx->dma[0]);
return HTP_STATUS_OK;
}
diff --git a/ggml/src/ggml-hexagon/htp/hex-cpy-dma.h b/ggml/src/ggml-hexagon/htp/hex-cpy-dma.h
new file mode 100644
index 000000000..c87f495e2
--- /dev/null
+++ b/ggml/src/ggml-hexagon/htp/hex-cpy-dma.h
@@ -0,0 +1,126 @@
+#ifndef HEX_CPY_DMA_H
+#define HEX_CPY_DMA_H
+
+// DDR<->DDR DMA copies of same-type, same-shape tensors with arbitrary strides.
+// Used by CPY for the copy itself and by CONCAT, which is two such copies into
+// two views of its destination. Every helper only pushes descriptors; the
+// caller flushes the queue when it needs the data.
+
+#include "dma-queue.h"
+#include "hex-common.h"
+#include "htp-tensor.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+// Contiguous byte run, as 1d transfers of at most DMA_SAFE_CHUNK_SIZE each.
+static inline void cpy_dma_sametype_reshape_contig(dma_queue * dma_q,
+ dma_addr_t dst,
+ dma_addr_t src0,
+ uint32_t total_bytes) {
+ if (total_bytes == 0) {
+ return;
+ }
+
+ const uint32_t max_chunk = DMA_SAFE_CHUNK_SIZE;
+ while (total_bytes > 0) {
+ const uint32_t chunk = MIN(total_bytes, max_chunk);
+ if (!dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/1)) {
+ dma_queue_flush(dma_q);
+ dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/1);
+ }
+ dst += chunk;
+ src0 += chunk;
+ total_bytes -= chunk;
+ }
+}
+
+// One 2d transfer, split at the 16-bit nrows field.
+static inline void cpy_dma_push_2d_chunked(dma_queue * dma_q,
+ dma_addr_t dst,
+ dma_addr_t src,
+ size_t dst_stride,
+ size_t src_stride,
+ size_t row_size,
+ uint32_t nrows) {
+ if (row_size == 0 || nrows == 0) {
+ return;
+ }
+
+ while (nrows > 0) {
+ const uint32_t cur_rows = MIN(nrows, DMA_MAX_NROWS);
+ if (!dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows)) {
+ dma_queue_flush(dma_q);
+ dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows);
+ }
+ dst += cur_rows * dst_stride;
+ src += cur_rows * src_stride;
+ nrows -= cur_rows;
+ }
+}
+
+// Copy src0 into dst: same type, same ne[], any nb[] above dim 0, dim 0 dense on
+// both sides (nb[0] == elem_size).
+static inline void cpy_dma_sametype_sameshape(dma_queue * dma_q,
+ const struct htp_tensor * dst,
+ const struct htp_tensor * src0,
+ uint32_t elem_size) {
+ const uint32_t ne00 = src0->ne[0];
+ const uint32_t ne01 = src0->ne[1];
+ const uint32_t ne02 = src0->ne[2];
+ const uint32_t ne03 = src0->ne[3];
+
+ if (ne00 == 0 || ne01 == 0 || ne02 == 0 || ne03 == 0) {
+ return;
+ }
+
+ const uint32_t nb01 = src0->nb[1];
+ const uint32_t nb02 = src0->nb[2];
+ const uint32_t nb03 = src0->nb[3];
+
+ const uint32_t nb1 = dst->nb[1];
+ const uint32_t nb2 = dst->nb[2];
+ const uint32_t nb3 = dst->nb[3];
+
+ const bool contiguous = htp_tensor_is_contiguous(src0, elem_size) && htp_tensor_is_contiguous(dst, elem_size);
+
+ if (contiguous) {
+ cpy_dma_sametype_reshape_contig(dma_q, dst->data, src0->data, ne00 * elem_size * ne01 * ne02 * ne03);
+ return;
+ }
+
+ // The single-descriptor path flattens (i01,i02,i03) into one row index, so every
+ // row must sit at a constant stride: nb01 on the source, nb1 on the destination.
+ // Walk the outer dims and require each to continue that progression. A dim of
+ // extent 1 spans no rows, so it is skipped -- but its own stride must NOT then be
+ // used to justify the next dim's stride, which is what comparing nb03 against
+ // ne02*nb02 did: ggml leaves the stride of an extent-1 dim meaningless, so a view
+ // could pass the check while its rows were nowhere near that stride.
+ uint32_t exp_src = ne01 * nb01;
+ uint32_t exp_dst = ne01 * nb1;
+ bool contiguous_outer = true;
+ if (ne02 != 1) {
+ contiguous_outer = contiguous_outer && (nb02 == exp_src) && (nb2 == exp_dst);
+ }
+ exp_src *= ne02;
+ exp_dst *= ne02;
+ if (ne03 != 1) {
+ contiguous_outer = contiguous_outer && (nb03 == exp_src) && (nb3 == exp_dst);
+ }
+
+ if (contiguous_outer) {
+ uint32_t total_rows = ne01 * ne02 * ne03;
+ cpy_dma_push_2d_chunked(dma_q, dst->data, src0->data, nb1, nb01, ne00 * elem_size, total_rows);
+ return;
+ }
+
+ for (uint32_t i03 = 0; i03 < ne03; i03++) {
+ for (uint32_t i02 = 0; i02 < ne02; i02++) {
+ dma_addr_t dst_data = dst->data + i02 * nb2 + i03 * nb3;
+ dma_addr_t src0_data = src0->data + i02 * nb02 + i03 * nb03;
+ cpy_dma_push_2d_chunked(dma_q, dst_data, src0_data, nb1, nb01, ne00 * elem_size, ne01);
+ }
+ }
+}
+
+#endif /* HEX_CPY_DMA_H */