Commit 7fee17846 for llama.cpp
commit 7fee1784646b6196bdff9c144326217720099a3c
Author: Trivikram Reddy <127072883+trivikram-reddy1@users.noreply.github.com>
Date: Tue Sep 29 14:52:10 2026 -0500
hexagon: optimize concat op (#29673)
* hex-concat: reduce pkts in gather/transpose hot loop
gather directly into dst buffer, use special instruction for gather sync
* hex-concat: use fastdiv
replace calls to sw divide with fastpath
* hex-concat: optimize DMA-HVX pipeline and add transpose helpers
diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c
index 382a94660..1bb9ce5e3 100644
--- a/ggml/src/ggml-hexagon/htp/concat-ops.c
+++ b/ggml/src/ggml-hexagon/htp/concat-ops.c
@@ -26,6 +26,12 @@ struct htp_concat_context {
struct fastdiv_values div_ne2;
};
+static inline dma_addr_t concat_plane_addr(const struct htp_tensor * t, uint32_t p, const struct fastdiv_values * div_ne2, uint32_t ne2) {
+ const uint32_t i3 = fastdiv(p, div_ne2);
+ const uint32_t i2 = p - i3 * ne2;
+ return t->data + i2 * t->nb[2] + i3 * t->nb[3];
+}
+
static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * data) {
struct htp_concat_context * cctx = (struct htp_concat_context *) data;
struct htp_ops_context * octx = cctx->octx;
@@ -50,58 +56,74 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void *
const uint32_t block_i = 32;
const uint32_t spad1_stride = block_i * sizeof(float);
- int32_t offsets[32] __attribute__((aligned(128)));
- for(int k=0; k<32; k++) {
- offsets[k] = k * spad1_stride;
- }
- HVX_Vector vv = *(HVX_Vector*)offsets;
+ const HVX_Vector offsets = hvx_vec_gather_offsets_w(spad1_stride);
const uint32_t src1_ne0_padded = hex_round_up(src1_ne0, 32);
- const uint32_t spad0_row_bytes = hex_round_up((src0_ne0 + src1_ne0_padded) * sizeof(float), VLEN);
- uint32_t mu = src1_ne0_padded * spad1_stride;
+ const uint32_t src0_row_bytes = src0_ne0 * sizeof(float);
+ const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN);
+ const uint32_t src0_pre = src0_row_padded - src0_row_bytes;
+ const uint32_t spad0_row_bytes = src0_row_padded + src1_ne0_padded * sizeof(float);
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
- for (uint32_t p = 0; p < cctx->nplanes; p++) {
- const uint32_t i3 = p / dst->ne[2];
- const uint32_t i2 = p - i3 * dst->ne[2];
- const dma_addr_t src0_plane = src0->data + i2 * src0->nb[2] + i3 * src0->nb[3];
- const dma_addr_t src1_plane = src1->data + i2 * src1->nb[2] + i3 * src1->nb[3];
- const dma_addr_t dst_plane = dst->data + i2 * dst->nb[2] + i3 * dst->nb[3];
-
- for (uint32_t i = start_i; i < end_i; i += block_i) {
- uint32_t current_block_i = (end_i - i < block_i) ? (end_i - i) : block_i;
-
- uint32_t src1_width_bytes = current_block_i * sizeof(float);
- const dma_addr_t src1_addr = src1_plane + i * src1->nb[1];
- dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], src1_width_bytes, src1_ne0);
-
- uint32_t src0_row_bytes = src0_ne0 * sizeof(float);
- const dma_addr_t src0_addr = src0_plane + i * src0->nb[1];
- dma_queue_push(dma_q, dma_make_data(spad0_base, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, current_block_i);
-
- dma_queue_pop(dma_q); // src1
-
- HVX_Vector * vtcm_tmp = (HVX_Vector *)(spad1_base + src1_ne0_padded * spad1_stride);
-
- htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
- for (uint32_t j = 0; j < src1_ne0_padded; j += 32) {
- #pragma unroll(4)
- for (uint32_t ii = 0; ii < current_block_i; ii++) {
- size_t rt = (size_t)(spad1_base + j * spad1_stride + ii * sizeof(float));
- Q6_vgather_ARMVw(&vtcm_tmp[ii], rt, mu, vv);
- uint8_t * dst_ptr = spad0_base + ii * spad0_row_bytes + (src0_ne0 + j) * sizeof(float);
- hvx_vmemu(dst_ptr) = vtcm_tmp[ii];
- }
- }
- htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
+ const struct fastdiv_values * div_ne2 = &cctx->div_ne2;
+ const uint32_t ne2 = dst->ne[2];
+
+ uint32_t p = 0;
+ uint32_t i = start_i;
+
+ const dma_addr_t src1_addr = concat_plane_addr(src1, p, div_ne2, ne2) + i * src1->nb[1];
+ dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(end_i - i, block_i) * sizeof(float), src1_ne0);
+
+ const dma_addr_t src0_addr = concat_plane_addr(src0, p, div_ne2, ne2) + i * src0->nb[1];
+ dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, MIN(end_i - i, block_i));
+
+ dma_queue_pop(dma_q); // src1
- dma_queue_pop(dma_q); // src0
+ while (p < cctx->nplanes) {
+ const uint32_t current_block_i = MIN(end_i - i, block_i);
- const dma_addr_t dst_addr = dst_plane + i * dst->nb[1];
- dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(float), current_block_i);
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
+ for (uint32_t j = 0; j < src1_ne0; j += 32) {
+ const uint8_t * src_ptr = spad1_base + j * spad1_stride;
+ uint8_t * dst_ptr = spad0_base + src0_row_padded + j * sizeof(float);
+ hvx_transpose_32x32_w_gather(dst_ptr, spad0_row_bytes, src_ptr, spad1_stride, offsets, current_block_i, MIN(src1_ne0 - j, 32));
+ }
+ hvx_gather_sync(spad0_base + src0_row_padded);
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
+
+ uint32_t np = p;
+ uint32_t ni = i + block_i;
+ if (ni >= end_i) {
+ ni = start_i;
+ np++;
+ }
+ const bool has_next = np < cctx->nplanes;
+ const uint32_t next_block_i = MIN(end_i - ni, block_i);
+
+ // spad1 is free after the gather sync, prefetch next src1 ahead of the dst write
+ if (has_next) {
+ const dma_addr_t nsrc1_addr = concat_plane_addr(src1, np, div_ne2, ne2) + ni * src1->nb[1];
+ dma_queue_push(dma_q, dma_make_data(spad1_base, nsrc1_addr), spad1_stride, src1->nb[0], next_block_i * sizeof(float), src1_ne0);
+ }
+
+ dma_queue_pop(dma_q); // src0
+
+ const dma_addr_t dst_addr = concat_plane_addr(dst, p, div_ne2, ne2) + i * dst->nb[1];
+ dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base + src0_pre), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(float), current_block_i);
+
+ if (has_next) {
+ dma_queue_pop(dma_q); // next src1
+ }
+ dma_queue_pop(dma_q); // dst
- dma_queue_pop(dma_q);
+ // spad0 is free after the dst write
+ if (has_next) {
+ const dma_addr_t nsrc0_addr = concat_plane_addr(src0, np, div_ne2, ne2) + ni * src0->nb[1];
+ dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, nsrc0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, next_block_i);
}
+
+ p = np;
+ i = ni;
}
}
@@ -129,58 +151,74 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void *
const uint32_t block_i = 64;
const uint32_t spad1_stride = block_i * sizeof(__fp16);
- int16_t offsets[64] __attribute__((aligned(128)));
- for(int k=0; k<64; k++) {
- offsets[k] = k * spad1_stride;
- }
- HVX_Vector vv = *(HVX_Vector*)offsets;
+ const HVX_Vector offsets = hvx_vec_gather_offsets_h(spad1_stride);
const uint32_t src1_ne0_padded = hex_round_up(src1_ne0, 64);
- const uint32_t spad0_row_bytes = hex_round_up((src0_ne0 + src1_ne0_padded) * sizeof(__fp16), VLEN);
- uint32_t mu = src1_ne0_padded * spad1_stride;
+ const uint32_t src0_row_bytes = src0_ne0 * sizeof(__fp16);
+ const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN);
+ const uint32_t src0_pre = src0_row_padded - src0_row_bytes;
+ const uint32_t spad0_row_bytes = src0_row_padded + src1_ne0_padded * sizeof(__fp16);
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
- for (uint32_t p = 0; p < cctx->nplanes; p++) {
- const uint32_t i3 = p / dst->ne[2];
- const uint32_t i2 = p - i3 * dst->ne[2];
- const dma_addr_t src0_plane = src0->data + i2 * src0->nb[2] + i3 * src0->nb[3];
- const dma_addr_t src1_plane = src1->data + i2 * src1->nb[2] + i3 * src1->nb[3];
- const dma_addr_t dst_plane = dst->data + i2 * dst->nb[2] + i3 * dst->nb[3];
-
- for (uint32_t i = start_i; i < end_i; i += block_i) {
- uint32_t current_block_i = (end_i - i < block_i) ? (end_i - i) : block_i;
-
- uint32_t src1_width_bytes = current_block_i * sizeof(__fp16);
- const dma_addr_t src1_addr = src1_plane + i * src1->nb[1];
- dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], src1_width_bytes, src1_ne0);
-
- uint32_t src0_row_bytes = src0_ne0 * sizeof(__fp16);
- const dma_addr_t src0_addr = src0_plane + i * src0->nb[1];
- dma_queue_push(dma_q, dma_make_data(spad0_base, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, current_block_i);
-
- dma_queue_pop(dma_q); // src1
-
- HVX_Vector * vtcm_tmp = (HVX_Vector *)(spad1_base + src1_ne0_padded * spad1_stride);
-
- htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
- for (uint32_t j = 0; j < src1_ne0_padded; j += 64) {
- #pragma unroll(4)
- for (uint32_t ii = 0; ii < current_block_i; ii++) {
- size_t rt = (size_t)(spad1_base + j * spad1_stride + ii * sizeof(__fp16));
- Q6_vgather_ARMVh(&vtcm_tmp[ii], rt, mu, vv);
- uint8_t * dst_ptr = spad0_base + ii * spad0_row_bytes + (src0_ne0 + j) * sizeof(__fp16);
- hvx_vmemu(dst_ptr) = vtcm_tmp[ii];
- }
- }
- htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
+ const struct fastdiv_values * div_ne2 = &cctx->div_ne2;
+ const uint32_t ne2 = dst->ne[2];
+
+ uint32_t p = 0;
+ uint32_t i = start_i;
+
+ const dma_addr_t src1_addr = concat_plane_addr(src1, p, div_ne2, ne2) + i * src1->nb[1];
+ dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(end_i - i, block_i) * sizeof(__fp16), src1_ne0);
+
+ const dma_addr_t src0_addr = concat_plane_addr(src0, p, div_ne2, ne2) + i * src0->nb[1];
+ dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, MIN(end_i - i, block_i));
+
+ dma_queue_pop(dma_q); // src1
+
+ while (p < cctx->nplanes) {
+ const uint32_t current_block_i = MIN(end_i - i, block_i);
- dma_queue_pop(dma_q); // src0
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
+ for (uint32_t j = 0; j < src1_ne0; j += 64) {
+ const uint8_t * src_ptr = spad1_base + j * spad1_stride;
+ uint8_t * dst_ptr = spad0_base + src0_row_padded + j * sizeof(__fp16);
+ hvx_transpose_64x64_h_gather(dst_ptr, spad0_row_bytes, src_ptr, spad1_stride, offsets, current_block_i, MIN(src1_ne0 - j, 64));
+ }
+ hvx_gather_sync(spad0_base + src0_row_padded);
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
+
+ uint32_t np = p;
+ uint32_t ni = i + block_i;
+ if (ni >= end_i) {
+ ni = start_i;
+ np++;
+ }
+ const bool has_next = np < cctx->nplanes;
+ const uint32_t next_block_i = MIN(end_i - ni, block_i);
+
+ // spad1 is free after the gather sync, prefetch next src1 ahead of the dst write
+ if (has_next) {
+ const dma_addr_t nsrc1_addr = concat_plane_addr(src1, np, div_ne2, ne2) + ni * src1->nb[1];
+ dma_queue_push(dma_q, dma_make_data(spad1_base, nsrc1_addr), spad1_stride, src1->nb[0], next_block_i * sizeof(__fp16), src1_ne0);
+ }
- const dma_addr_t dst_addr = dst_plane + i * dst->nb[1];
- dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(__fp16), current_block_i);
+ dma_queue_pop(dma_q); // src0
- dma_queue_pop(dma_q);
+ const dma_addr_t dst_addr = concat_plane_addr(dst, p, div_ne2, ne2) + i * dst->nb[1];
+ dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base + src0_pre), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(__fp16), current_block_i);
+
+ if (has_next) {
+ dma_queue_pop(dma_q); // next src1
+ }
+ dma_queue_pop(dma_q); // dst
+
+ // spad0 is free after the dst write
+ if (has_next) {
+ const dma_addr_t nsrc0_addr = concat_plane_addr(src0, np, div_ne2, ne2) + ni * src0->nb[1];
+ dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, nsrc0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, next_block_i);
}
+
+ p = np;
+ i = ni;
}
}
@@ -200,7 +238,7 @@ static void concat_generic(unsigned int nth, unsigned int ith, void * data) {
// Per-device element range aligned to prevent false sharing
const uint32_t elem_start = cctx->elem_start;
const uint32_t nelems = cctx->nelems;
- const uint32_t chunk_size = (nelems + nth - 1) / nth;
+ const uint32_t chunk_size = fastdiv(nelems + nth - 1, &octx->n_threads_div);
const uint32_t start_idx = MIN(elem_start + ith * chunk_size, elem_start + nelems);
const uint32_t end_idx = MIN(start_idx + chunk_size, elem_start + nelems);
@@ -358,10 +396,11 @@ int op_concat(struct htp_ops_context * octx) {
uint32_t spad1_stride = block_i * type_size;
uint32_t src1_ne0_padded = hex_round_up(src1->ne[0], block_i);
- uint32_t spad0_row_bytes = hex_round_up((src0->ne[0] + src1_ne0_padded) * type_size, VLEN);
+ // src0 row is right-aligned to VLEN so the gathered src1 part starts aligned
+ uint32_t spad0_row_bytes = hex_round_up(src0->ne[0] * type_size, VLEN) + src1_ne0_padded * type_size;
octx->src0_spad.size_per_thread = block_i * spad0_row_bytes;
- octx->src1_spad.size_per_thread = src1_ne0_padded * spad1_stride + block_i * VLEN;
+ octx->src1_spad.size_per_thread = src1_ne0_padded * spad1_stride;
octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread;
octx->src1_spad.size = n_threads * octx->src1_spad.size_per_thread;
diff --git a/ggml/src/ggml-hexagon/htp/hvx-base.h b/ggml/src/ggml-hexagon/htp/hvx-base.h
index 0d0ed3190..20e0d6e66 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-base.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-base.h
@@ -303,4 +303,137 @@ static inline HVX_Vector hvx_vec_load_act_tile(const uint8_t * y_q, uint32_t kt,
}
}
+// Full tile, in place. tmp: 32 vectors scratch.
+// Use when result must stay in m. Slowest (spills).
+static inline void hvx_transpose_32x32_w_shuff_inplace(HVX_Vector * restrict m, HVX_Vector * restrict tmp) {
+ for (int i = 0; i < 16; ++i) {
+ HVX_VectorPair p = Q6_W_vshuff_VVR(m[2*i + 1], m[2*i], -4);
+ tmp[2*i + 0] = Q6_V_lo_W(p);
+ tmp[2*i + 1] = Q6_V_hi_W(p);
+ }
+
+ for (int b = 0; b < 32; b += 4) {
+ HVX_VectorPair p0 = Q6_W_vshuff_VVR(tmp[b + 2], tmp[b + 0], -8);
+ HVX_VectorPair p1 = Q6_W_vshuff_VVR(tmp[b + 3], tmp[b + 1], -8);
+ m[b + 0] = Q6_V_lo_W(p0); m[b + 1] = Q6_V_hi_W(p0);
+ m[b + 2] = Q6_V_lo_W(p1); m[b + 3] = Q6_V_hi_W(p1);
+ }
+
+ for (int b = 0; b < 32; b += 8) {
+ for (int i = 0; i < 4; ++i) {
+ HVX_VectorPair p = Q6_W_vshuff_VVR(m[b + i + 4], m[b + i], -16);
+ tmp[b + 2*i + 0] = Q6_V_lo_W(p);
+ tmp[b + 2*i + 1] = Q6_V_hi_W(p);
+ }
+ }
+
+ for (int b = 0; b < 32; b += 16) {
+ for (int i = 0; i < 8; ++i) {
+ HVX_VectorPair p = Q6_W_vshuff_VVR(tmp[b + i + 8], tmp[b + i], -32);
+ m[b + 2*i + 0] = Q6_V_lo_W(p);
+ m[b + 2*i + 1] = Q6_V_hi_W(p);
+ }
+ }
+
+ for (int i = 0; i < 16; ++i) {
+ HVX_VectorPair p = Q6_W_vshuff_VVR(m[i + 16], m[i], -64);
+ tmp[2 * i + 0] = Q6_V_lo_W(p);
+ tmp[2 * i + 1] = Q6_V_hi_W(p);
+ }
+
+ for (int i = 0; i < 32; ++i) {
+ m[i] = tmp[i];
+ }
+}
+
+// Rows [0, nrows) of the transpose of src[0, ncols) to dst. src is clobbered, must not alias dst.
+// Use when not in VTCM or for full tiles.
+static inline void hvx_transpose_32x32_w_shuff(uint8_t * restrict dst, uint32_t dst_stride, HVX_Vector * restrict src,
+ uint32_t nrows, uint32_t ncols) {
+ for (uint32_t g = 0; g < ncols; g += 8) {
+ HVX_Vector * s = src + g;
+
+ HVX_VectorPair a0 = Q6_W_vshuff_VVR(s[1], s[0], -4);
+ HVX_VectorPair a1 = Q6_W_vshuff_VVR(s[3], s[2], -4);
+ HVX_VectorPair a2 = Q6_W_vshuff_VVR(s[5], s[4], -4);
+ HVX_VectorPair a3 = Q6_W_vshuff_VVR(s[7], s[6], -4);
+
+ HVX_VectorPair b0 = Q6_W_vshuff_VVR(Q6_V_lo_W(a1), Q6_V_lo_W(a0), -8);
+ HVX_VectorPair b1 = Q6_W_vshuff_VVR(Q6_V_hi_W(a1), Q6_V_hi_W(a0), -8);
+ HVX_VectorPair b2 = Q6_W_vshuff_VVR(Q6_V_lo_W(a3), Q6_V_lo_W(a2), -8);
+ HVX_VectorPair b3 = Q6_W_vshuff_VVR(Q6_V_hi_W(a3), Q6_V_hi_W(a2), -8);
+
+ HVX_VectorPair c0 = Q6_W_vshuff_VVR(Q6_V_lo_W(b2), Q6_V_lo_W(b0), -16);
+ HVX_VectorPair c1 = Q6_W_vshuff_VVR(Q6_V_hi_W(b2), Q6_V_hi_W(b0), -16);
+ HVX_VectorPair c2 = Q6_W_vshuff_VVR(Q6_V_lo_W(b3), Q6_V_lo_W(b1), -16);
+ HVX_VectorPair c3 = Q6_W_vshuff_VVR(Q6_V_hi_W(b3), Q6_V_hi_W(b1), -16);
+
+ s[0] = Q6_V_lo_W(c0); s[1] = Q6_V_hi_W(c0);
+ s[2] = Q6_V_lo_W(c1); s[3] = Q6_V_hi_W(c1);
+ s[4] = Q6_V_lo_W(c2); s[5] = Q6_V_hi_W(c2);
+ s[6] = Q6_V_lo_W(c3); s[7] = Q6_V_hi_W(c3);
+ }
+
+ for (uint32_t k = 0; 4 * k < nrows; k++) {
+ HVX_VectorPair d0 = Q6_W_vshuff_VVR(src[k + 8], src[k], -32);
+ HVX_VectorPair d1 = Q6_W_vshuff_VVR(src[k + 24], src[k + 16], -32);
+
+ HVX_VectorPair e0 = Q6_W_vshuff_VVR(Q6_V_lo_W(d1), Q6_V_lo_W(d0), -64);
+ HVX_VectorPair e1 = Q6_W_vshuff_VVR(Q6_V_hi_W(d1), Q6_V_hi_W(d0), -64);
+
+ uint8_t * d = dst + 4 * k * dst_stride;
+ *(HVX_Vector *) (d + 0 * dst_stride) = Q6_V_lo_W(e0);
+ *(HVX_Vector *) (d + 1 * dst_stride) = Q6_V_hi_W(e0);
+ *(HVX_Vector *) (d + 2 * dst_stride) = Q6_V_lo_W(e1);
+ *(HVX_Vector *) (d + 3 * dst_stride) = Q6_V_hi_W(e1);
+ }
+}
+
+// Specialized store that will follow outstanding scatters or gathers to make sure they complete
+static inline void hvx_gather_sync(void * vtcm_addr) {
+ asm volatile("vmem(%0+#0):scatter_release\n"
+ "v0 = vmem(%0+#0)\n" :: "r"(vtcm_addr) : "v0", "memory");
+}
+
+// Lane k = k * stride. Offsets for the _gather transposes.
+static inline HVX_Vector hvx_vec_gather_offsets_w(uint32_t stride) {
+ int32_t offsets[32] __attribute__((aligned(128)));
+ for (int k = 0; k < 32; k++) {
+ offsets[k] = k * stride;
+ }
+ return *(HVX_Vector *) offsets;
+}
+
+static inline HVX_Vector hvx_vec_gather_offsets_h(uint32_t stride) {
+ int16_t offsets[64] __attribute__((aligned(128)));
+ for (int k = 0; k < 64; k++) {
+ offsets[k] = k * stride;
+ }
+ return *(HVX_Vector *) offsets;
+}
+
+// Rows [0, nrows) of the transpose of ncols src vectors (src_stride apart) to dst.
+// src, dst in VTCM. Async: hvx_gather_sync() before use.
+// Use for partial tiles (small ncols). Fewest packets.
+static inline void hvx_transpose_32x32_w_gather(uint8_t * restrict dst, uint32_t dst_stride, const uint8_t * restrict src,
+ uint32_t src_stride, HVX_Vector offsets, uint32_t nrows, uint32_t ncols) {
+ const uint32_t mu = ncols * src_stride;
+
+ #pragma unroll(4)
+ for (uint32_t r = 0; r < nrows; r++) {
+ Q6_vgather_ARMVw((HVX_Vector *) (dst + r * dst_stride), (size_t) (src + r * sizeof(int32_t)), mu, offsets);
+ }
+}
+
+// 16-bit version of hvx_transpose_32x32_w_gather.
+static inline void hvx_transpose_64x64_h_gather(uint8_t * restrict dst, uint32_t dst_stride, const uint8_t * restrict src,
+ uint32_t src_stride, HVX_Vector offsets, uint32_t nrows, uint32_t ncols) {
+ const uint32_t mu = ncols * src_stride;
+
+ #pragma unroll(4)
+ for (uint32_t r = 0; r < nrows; r++) {
+ Q6_vgather_ARMVh((HVX_Vector *) (dst + r * dst_stride), (size_t) (src + r * sizeof(int16_t)), mu, offsets);
+ }
+}
+
#endif /* HVX_BASE_H */