Commit da6c30ade for llama.cpp
commit da6c30ade8c30930dbf49fe57f9f02d727e99b15
Author: Trivikram Reddy <127072883+trivikram-reddy1@users.noreply.github.com>
Date: Thu Oct 29 01:57:18 2026 -0500
hexagon: improve DDR bw utilization in transposed CONCAT (#31277)
* hex-concat: double row block size for transposed CONCAT
* hex-concat: double buffer scratchpads
* hex-concat: chunked src1 rows in transposed concat
* hex-concat: pad src1 VTCM stride to spread gather lanes
* hex-vgather: apply similar vgather opts to SSM_CONV and GDN
---------
Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp
index e5b2cd05d..b9ae9f9db 100644
--- a/ggml/src/ggml-hexagon/htp-kparams.cpp
+++ b/ggml/src/ggml-hexagon/htp-kparams.cpp
@@ -2185,7 +2185,8 @@ static bool htp_kparams_solve_ssm_conv(
target_max_tile = (std::max)(target_max_tile, 32u);
target_max_tile = (std::min)(target_max_tile, 128u);
- uint32_t d_inner_tile = (uint32_t) (avail_for_tiles / (2 * (ncs + n_t) * sizeof(float)));
+ const size_t raw_stride_bytes = ncs * sizeof(float) + sizeof(float);
+ uint32_t d_inner_tile = (uint32_t) (avail_for_tiles / (2 * (raw_stride_bytes + n_t * sizeof(float))));
d_inner_tile = (d_inner_tile / 32) * 32;
if (d_inner_tile == 0) {
d_inner_tile = 32;
diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c
index 20369884b..8f3e199c2 100644
--- a/ggml/src/ggml-hexagon/htp/concat-ops.c
+++ b/ggml/src/ggml-hexagon/htp/concat-ops.c
@@ -47,87 +47,108 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void *
const uint32_t src0_ne0 = src0->ne[0];
const uint32_t src1_ne0 = src1->ne[0];
- const uint32_t row_end = cctx->row_start + cctx->nrows;
- const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread;
- const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end;
- if (start_i >= end_i) return;
+ // dst rows [row_begin, row_end) of this thread; a dst row is a src0 row followed by a src1 column
+ const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_thread;
+ const uint32_t row_end = MIN(row_begin + cctx->nrows_per_thread, cctx->row_start + cctx->nrows);
+ if (row_begin >= row_end) return;
dma_queue * dma_q = octx->ctx->dma[ith];
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];
- uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread;
- uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread;
+ const struct fastdiv_values * div_ne2 = &cctx->div_ne2;
+ const uint32_t ne2 = dst->ne[2];
- const uint32_t block_i = 32;
- const uint32_t spad1_stride = block_i * sizeof(float);
+ // tile: block_rows dst rows x block_cols src1 columns
+ const uint32_t block_rows = 128;
+ const uint32_t block_cols = 256;
+ // +1 element spreads the gather lanes across VTCM, so they do not serialize on one path
+ const uint32_t spad1_stride = block_rows * sizeof(float) + sizeof(float);
const HVX_Vector offsets = hvx_vec_gather_offsets_w(spad1_stride);
- const uint32_t src1_ne0_padded = hex_round_up(src1_ne0, 32);
- const uint32_t src0_row_bytes = src0_ne0 * sizeof(float);
- const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN);
- const uint32_t src0_pre = src0_row_padded - src0_row_bytes;
- const uint32_t spad0_row_bytes = src0_row_padded + src1_ne0_padded * sizeof(float);
+ const uint32_t block_cols_padded = MIN(hex_round_up(src1_ne0, 32), block_cols);
+ const uint32_t src0_row_bytes = src0_ne0 * sizeof(float);
+ const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN);
+ const uint32_t src0_pre = src0_row_padded - src0_row_bytes;
+ const uint32_t spad0_stride = src0_row_padded + block_cols_padded * sizeof(float);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
-
- const struct fastdiv_values * div_ne2 = &cctx->div_ne2;
- const uint32_t ne2 = dst->ne[2];
+ // two slots per scratchpad: tile b gathers in slot b&1 while dst[b-1] and src1[b+1] DMA in the other
+ uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread;
+ uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread;
+ const uint32_t spad0_slot = block_rows * spad0_stride;
+ const uint32_t spad1_slot = block_cols_padded * spad1_stride;
- uint32_t p = 0;
- uint32_t i = start_i;
+ // tiles walk (plane, row, col) with col fastest; src0 goes out with the col == 0 tile only
+ uint32_t plane = 0;
+ uint32_t row = row_begin;
+ uint32_t col = 0;
+ uint32_t slot = 0;
+ bool dst_pending = false;
- const dma_addr_t src1_addr = concat_plane_addr(src1, p, div_ne2, ne2) + i * src1->nb[1];
- dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(end_i - i, block_i) * sizeof(float), src1_ne0);
+ const dma_addr_t src1_addr = concat_plane_addr(src1, plane, div_ne2, ne2) + row * src1->nb[1];
+ dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(row_end - row, block_rows) * sizeof(float), MIN(src1_ne0, block_cols));
- const dma_addr_t src0_addr = concat_plane_addr(src0, p, div_ne2, ne2) + i * src0->nb[1];
- dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, MIN(end_i - i, block_i));
+ const dma_addr_t src0_addr = concat_plane_addr(src0, plane, div_ne2, ne2) + row * src0->nb[1];
+ dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_stride, src0->nb[1], src0_row_bytes, MIN(row_end - row, block_rows));
- dma_queue_pop(dma_q); // src1
+ while (plane < cctx->nplanes) {
+ const uint32_t nrows = MIN(row_end - row, block_rows);
+ const uint32_t ncols = MIN(src1_ne0 - col, block_cols);
- while (p < cctx->nplanes) {
- const uint32_t current_block_i = MIN(end_i - i, block_i);
+ uint8_t * spad0 = spad0_base + slot * spad0_slot;
+ uint8_t * spad1 = spad1_base + slot * spad1_slot;
- htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
- for (uint32_t j = 0; j < src1_ne0; j += 32) {
- const uint8_t * src_ptr = spad1_base + j * spad1_stride;
- uint8_t * dst_ptr = spad0_base + src0_row_padded + j * sizeof(float);
- hvx_transpose_32x32_w_gather(dst_ptr, spad0_row_bytes, src_ptr, spad1_stride, offsets, current_block_i, MIN(src1_ne0 - j, 32));
+ dma_queue_pop(dma_q); // src1
+ if (col == 0) {
+ dma_queue_pop(dma_q); // src0
}
- hvx_gather_sync(spad0_base + src0_row_padded);
- htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
-
- uint32_t np = p;
- uint32_t ni = i + block_i;
- if (ni >= end_i) {
- ni = start_i;
- np++;
+
+ uint32_t next_plane = plane;
+ uint32_t next_row = row;
+ uint32_t next_col = col + block_cols;
+ if (next_col >= src1_ne0) {
+ next_col = 0;
+ next_row = row + block_rows;
+ if (next_row >= row_end) {
+ next_row = row_begin;
+ next_plane++;
+ }
}
- const bool has_next = np < cctx->nplanes;
- const uint32_t next_block_i = MIN(end_i - ni, block_i);
+ const bool has_next = next_plane < cctx->nplanes;
+ const uint32_t next_nrows = MIN(row_end - next_row, block_rows);
+ const uint32_t next_ncols = MIN(src1_ne0 - next_col, block_cols);
- // spad1 is free after the gather sync, prefetch next src1 ahead of the dst write
+ // other spad1 slot is free since the previous gather sync
if (has_next) {
- const dma_addr_t nsrc1_addr = concat_plane_addr(src1, np, div_ne2, ne2) + ni * src1->nb[1];
- dma_queue_push(dma_q, dma_make_data(spad1_base, nsrc1_addr), spad1_stride, src1->nb[0], next_block_i * sizeof(float), src1_ne0);
+ const dma_addr_t addr = concat_plane_addr(src1, next_plane, div_ne2, ne2) + next_row * src1->nb[1] + next_col * src1->nb[0];
+ dma_queue_push(dma_q, dma_make_data(spad1_base + (slot ^ 1) * spad1_slot, addr), spad1_stride, src1->nb[0], next_nrows * sizeof(float), next_ncols);
}
- dma_queue_pop(dma_q); // src0
-
- const dma_addr_t dst_addr = concat_plane_addr(dst, p, div_ne2, ne2) + i * dst->nb[1];
- dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base + src0_pre), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(float), current_block_i);
-
- if (has_next) {
- dma_queue_pop(dma_q); // next src1
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row);
+ for (uint32_t c = 0; c < ncols; c += 32) {
+ hvx_transpose_32x32_w_gather(spad0 + src0_row_padded + c * sizeof(float), spad0_stride, spad1 + c * spad1_stride, spad1_stride, offsets, nrows, MIN(ncols - c, 32));
}
- dma_queue_pop(dma_q); // dst
+ hvx_gather_sync(spad0 + src0_row_padded);
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row);
- // spad0 is free after the dst write
- if (has_next) {
- const dma_addr_t nsrc0_addr = concat_plane_addr(src0, np, div_ne2, ne2) + ni * src0->nb[1];
- dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, nsrc0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, next_block_i);
+ // other spad0 slot is free once dst[b-1] is written
+ if (dst_pending) {
+ dma_queue_pop(dma_q); // dst
}
+ if (has_next && next_col == 0) {
+ const dma_addr_t addr = concat_plane_addr(src0, next_plane, div_ne2, ne2) + next_row * src0->nb[1];
+ dma_queue_push(dma_q, dma_make_data(spad0_base + (slot ^ 1) * spad0_slot + src0_pre, addr), spad0_stride, src0->nb[1], src0_row_bytes, next_nrows);
+ }
+
+ // the col == 0 tile also carries the src0 columns in front
+ const uint32_t lead = (col == 0) ? src0_row_bytes : 0;
+ const dma_addr_t dst_addr = concat_plane_addr(dst, plane, div_ne2, ne2) + row * dst->nb[1] + src0_row_bytes + col * sizeof(float) - lead;
+ dma_queue_push(dma_q, dma_make_data(dst_addr, spad0 + src0_row_padded - lead), dst->nb[1], spad0_stride, lead + ncols * sizeof(float), nrows);
+ dst_pending = true;
- p = np;
- i = ni;
+ plane = next_plane;
+ row = next_row;
+ col = next_col;
+ slot ^= 1;
}
dma_queue_flush(dma_q);
}
@@ -143,87 +164,108 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void *
const uint32_t src0_ne0 = src0->ne[0];
const uint32_t src1_ne0 = src1->ne[0];
- const uint32_t row_end = cctx->row_start + cctx->nrows;
- const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread;
- const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end;
- if (start_i >= end_i) return;
+ // dst rows [row_begin, row_end) of this thread; a dst row is a src0 row followed by a src1 column
+ const uint32_t row_begin = cctx->row_start + ith * cctx->nrows_per_thread;
+ const uint32_t row_end = MIN(row_begin + cctx->nrows_per_thread, cctx->row_start + cctx->nrows);
+ if (row_begin >= row_end) return;
dma_queue * dma_q = octx->ctx->dma[ith];
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];
- uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread;
- uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread;
+ const struct fastdiv_values * div_ne2 = &cctx->div_ne2;
+ const uint32_t ne2 = dst->ne[2];
- const uint32_t block_i = 64;
- const uint32_t spad1_stride = block_i * sizeof(__fp16);
+ // tile: block_rows dst rows x block_cols src1 columns
+ const uint32_t block_rows = 128;
+ const uint32_t block_cols = 512;
+ // +1 element spreads the gather lanes across VTCM, so they do not serialize on one path
+ const uint32_t spad1_stride = block_rows * sizeof(__fp16) + sizeof(__fp16);
const HVX_Vector offsets = hvx_vec_gather_offsets_h(spad1_stride);
- const uint32_t src1_ne0_padded = hex_round_up(src1_ne0, 64);
- const uint32_t src0_row_bytes = src0_ne0 * sizeof(__fp16);
- const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN);
- const uint32_t src0_pre = src0_row_padded - src0_row_bytes;
- const uint32_t spad0_row_bytes = src0_row_padded + src1_ne0_padded * sizeof(__fp16);
+ const uint32_t block_cols_padded = MIN(hex_round_up(src1_ne0, 64), block_cols);
+ const uint32_t src0_row_bytes = src0_ne0 * sizeof(__fp16);
+ const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN);
+ const uint32_t src0_pre = src0_row_padded - src0_row_bytes;
+ const uint32_t spad0_stride = src0_row_padded + block_cols_padded * sizeof(__fp16);
- struct htp_thread_trace * tr = &octx->ctx->trace[ith];
-
- const struct fastdiv_values * div_ne2 = &cctx->div_ne2;
- const uint32_t ne2 = dst->ne[2];
+ // two slots per scratchpad: tile b gathers in slot b&1 while dst[b-1] and src1[b+1] DMA in the other
+ uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread;
+ uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread;
+ const uint32_t spad0_slot = block_rows * spad0_stride;
+ const uint32_t spad1_slot = block_cols_padded * spad1_stride;
- uint32_t p = 0;
- uint32_t i = start_i;
+ // tiles walk (plane, row, col) with col fastest; src0 goes out with the col == 0 tile only
+ uint32_t plane = 0;
+ uint32_t row = row_begin;
+ uint32_t col = 0;
+ uint32_t slot = 0;
+ bool dst_pending = false;
- const dma_addr_t src1_addr = concat_plane_addr(src1, p, div_ne2, ne2) + i * src1->nb[1];
- dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(end_i - i, block_i) * sizeof(__fp16), src1_ne0);
+ const dma_addr_t src1_addr = concat_plane_addr(src1, plane, div_ne2, ne2) + row * src1->nb[1];
+ dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(row_end - row, block_rows) * sizeof(__fp16), MIN(src1_ne0, block_cols));
- const dma_addr_t src0_addr = concat_plane_addr(src0, p, div_ne2, ne2) + i * src0->nb[1];
- dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, MIN(end_i - i, block_i));
+ const dma_addr_t src0_addr = concat_plane_addr(src0, plane, div_ne2, ne2) + row * src0->nb[1];
+ dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_stride, src0->nb[1], src0_row_bytes, MIN(row_end - row, block_rows));
- dma_queue_pop(dma_q); // src1
+ while (plane < cctx->nplanes) {
+ const uint32_t nrows = MIN(row_end - row, block_rows);
+ const uint32_t ncols = MIN(src1_ne0 - col, block_cols);
- while (p < cctx->nplanes) {
- const uint32_t current_block_i = MIN(end_i - i, block_i);
+ uint8_t * spad0 = spad0_base + slot * spad0_slot;
+ uint8_t * spad1 = spad1_base + slot * spad1_slot;
- htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
- for (uint32_t j = 0; j < src1_ne0; j += 64) {
- const uint8_t * src_ptr = spad1_base + j * spad1_stride;
- uint8_t * dst_ptr = spad0_base + src0_row_padded + j * sizeof(__fp16);
- hvx_transpose_64x64_h_gather(dst_ptr, spad0_row_bytes, src_ptr, spad1_stride, offsets, current_block_i, MIN(src1_ne0 - j, 64));
+ dma_queue_pop(dma_q); // src1
+ if (col == 0) {
+ dma_queue_pop(dma_q); // src0
}
- hvx_gather_sync(spad0_base + src0_row_padded);
- htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
-
- uint32_t np = p;
- uint32_t ni = i + block_i;
- if (ni >= end_i) {
- ni = start_i;
- np++;
+
+ uint32_t next_plane = plane;
+ uint32_t next_row = row;
+ uint32_t next_col = col + block_cols;
+ if (next_col >= src1_ne0) {
+ next_col = 0;
+ next_row = row + block_rows;
+ if (next_row >= row_end) {
+ next_row = row_begin;
+ next_plane++;
+ }
}
- const bool has_next = np < cctx->nplanes;
- const uint32_t next_block_i = MIN(end_i - ni, block_i);
+ const bool has_next = next_plane < cctx->nplanes;
+ const uint32_t next_nrows = MIN(row_end - next_row, block_rows);
+ const uint32_t next_ncols = MIN(src1_ne0 - next_col, block_cols);
- // spad1 is free after the gather sync, prefetch next src1 ahead of the dst write
+ // other spad1 slot is free since the previous gather sync
if (has_next) {
- const dma_addr_t nsrc1_addr = concat_plane_addr(src1, np, div_ne2, ne2) + ni * src1->nb[1];
- dma_queue_push(dma_q, dma_make_data(spad1_base, nsrc1_addr), spad1_stride, src1->nb[0], next_block_i * sizeof(__fp16), src1_ne0);
+ const dma_addr_t addr = concat_plane_addr(src1, next_plane, div_ne2, ne2) + next_row * src1->nb[1] + next_col * src1->nb[0];
+ dma_queue_push(dma_q, dma_make_data(spad1_base + (slot ^ 1) * spad1_slot, addr), spad1_stride, src1->nb[0], next_nrows * sizeof(__fp16), next_ncols);
}
- dma_queue_pop(dma_q); // src0
-
- const dma_addr_t dst_addr = concat_plane_addr(dst, p, div_ne2, ne2) + i * dst->nb[1];
- dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base + src0_pre), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(__fp16), current_block_i);
-
- if (has_next) {
- dma_queue_pop(dma_q); // next src1
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row);
+ for (uint32_t c = 0; c < ncols; c += 64) {
+ hvx_transpose_64x64_h_gather(spad0 + src0_row_padded + c * sizeof(__fp16), spad0_stride, spad1 + c * spad1_stride, spad1_stride, offsets, nrows, MIN(ncols - c, 64));
}
- dma_queue_pop(dma_q); // dst
+ hvx_gather_sync(spad0 + src0_row_padded);
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row);
- // spad0 is free after the dst write
- if (has_next) {
- const dma_addr_t nsrc0_addr = concat_plane_addr(src0, np, div_ne2, ne2) + ni * src0->nb[1];
- dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, nsrc0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, next_block_i);
+ // other spad0 slot is free once dst[b-1] is written
+ if (dst_pending) {
+ dma_queue_pop(dma_q); // dst
}
+ if (has_next && next_col == 0) {
+ const dma_addr_t addr = concat_plane_addr(src0, next_plane, div_ne2, ne2) + next_row * src0->nb[1];
+ dma_queue_push(dma_q, dma_make_data(spad0_base + (slot ^ 1) * spad0_slot + src0_pre, addr), spad0_stride, src0->nb[1], src0_row_bytes, next_nrows);
+ }
+
+ // the col == 0 tile also carries the src0 columns in front
+ const uint32_t lead = (col == 0) ? src0_row_bytes : 0;
+ const dma_addr_t dst_addr = concat_plane_addr(dst, plane, div_ne2, ne2) + row * dst->nb[1] + src0_row_bytes + col * sizeof(__fp16) - lead;
+ dma_queue_push(dma_q, dma_make_data(dst_addr, spad0 + src0_row_padded - lead), dst->nb[1], spad0_stride, lead + ncols * sizeof(__fp16), nrows);
+ dst_pending = true;
- p = np;
- i = ni;
+ plane = next_plane;
+ row = next_row;
+ col = next_col;
+ slot ^= 1;
}
dma_queue_flush(dma_q);
}
diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.h b/ggml/src/ggml-hexagon/htp/concat-ops.h
index a5d93284f..1f0744434 100644
--- a/ggml/src/ggml-hexagon/htp/concat-ops.h
+++ b/ggml/src/ggml-hexagon/htp/concat-ops.h
@@ -53,13 +53,17 @@ static inline void htp_concat_transposed_vtcm_layout_build(
uint32_t type_size,
uint32_t n_threads) {
- uint32_t block_i = (type_size == 4) ? 32 : 64;
- uint32_t spad1_stride = block_i * type_size;
- uint32_t src1_ne0_padded = hex_round_up(src1_ne0, block_i);
- uint32_t spad0_row_bytes = hex_round_up(src0_ne0 * type_size, 128) + src1_ne0_padded * type_size;
+ // src1_ne0 is processed in chunks of block_cols tokens, so VTCM does not grow with the batch
+ uint32_t block_rows = 128;
+ uint32_t block_cols = (type_size == 4) ? 256 : 512;
+ // +1 element spreads the gather lanes across VTCM, so they do not serialize on one path
+ uint32_t spad1_stride = block_rows * type_size + type_size;
+ uint32_t block_cols_padded = MIN(hex_round_up(src1_ne0, (type_size == 4) ? 32 : 64), block_cols);
+ uint32_t spad0_row_bytes = hex_round_up(src0_ne0 * type_size, 128) + block_cols_padded * type_size;
- L->src0_spad_size_per_thread = block_i * spad0_row_bytes;
- L->src1_spad_size_per_thread = src1_ne0_padded * spad1_stride;
+ // double-buffered: two slots per thread
+ L->src0_spad_size_per_thread = 2 * block_rows * spad0_row_bytes;
+ L->src1_spad_size_per_thread = 2 * block_cols_padded * spad1_stride;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_spad0, (size_t) n_threads * L->src0_spad_size_per_thread);
diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c
index 5252ac352..a70f41185 100644
--- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c
+++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c
@@ -1456,9 +1456,10 @@ static inline void gdn_dma_push_chunk_gb(
const dma_addr_t g_dma = g->data + (uint64_t) iv3 * g->nb[3] + (uint64_t) t_chunk * g->nb[2] + (uint64_t) iv1 * g->nb[1];
const dma_addr_t beta_dma = beta->data + (uint64_t) iv3 * beta->nb[3] + (uint64_t) t_chunk * beta->nb[2] + (uint64_t) iv1 * beta->nb[1];
const uint32_t row_bytes = n_batch * sizeof(float);
+ const uint32_t dst_stride = (n_batch > 1) ? row_bytes + sizeof(float) : row_bytes;
- dma_queue_push(dma_q, dma_make_data(vtcm_g_raw, g_dma), row_bytes, g->nb[2], row_bytes, chunk_size);
- dma_queue_push(dma_q, dma_make_data(vtcm_b_raw, beta_dma), row_bytes, beta->nb[2], row_bytes, chunk_size);
+ dma_queue_push(dma_q, dma_make_data(vtcm_g_raw, g_dma), dst_stride, g->nb[2], row_bytes, chunk_size);
+ dma_queue_push(dma_q, dma_make_data(vtcm_b_raw, beta_dma), dst_stride, beta->nb[2], row_bytes, chunk_size);
}
static inline void gdn_pack_s_col_tiles(
@@ -1707,19 +1708,16 @@ static void gdn_hvx_phase1a_worker(unsigned int n, unsigned int i, void * data)
hvx_vmem(head->b_f32[curr_buf] + 0) = hvx_vmem(bctx->vtcm_b_raw + 0);
hvx_vmem(head->b_f32[curr_buf] + 32) = hvx_vmem(bctx->vtcm_b_raw + 32);
} else {
- int32_t offsets[32] __attribute__((aligned(128)));
- for (int k = 0; k < 32; ++k) {
- offsets[k] = k * n_batch * sizeof(float);
- }
- HVX_Vector vv = *(const HVX_Vector *) offsets;
+ const uint32_t vtcm_stride = (n_batch + 1) * sizeof(float);
+ const HVX_Vector vv = hvx_vec_gather_offsets_w(vtcm_stride);
const size_t rt_g = (size_t) ((const uint8_t *) bctx->vtcm_g_raw + i * sizeof(float));
const size_t rt_b = (size_t) ((const uint8_t *) bctx->vtcm_b_raw + i * sizeof(float));
- const size_t mu = 64 * n_batch * sizeof(float);
+ const size_t mu = 64 * vtcm_stride;
Q6_vgather_ARMVw((HVX_Vector *) (head->g_f32[curr_buf] + 0), rt_g, mu, vv);
- Q6_vgather_ARMVw((HVX_Vector *) (head->g_f32[curr_buf] + 32), rt_g + 32 * n_batch * sizeof(float), mu, vv);
+ Q6_vgather_ARMVw((HVX_Vector *) (head->g_f32[curr_buf] + 32), rt_g + 32 * vtcm_stride, mu, vv);
Q6_vgather_ARMVw((HVX_Vector *) (head->b_f32[curr_buf] + 0), rt_b, mu, vv);
- Q6_vgather_ARMVw((HVX_Vector *) (head->b_f32[curr_buf] + 32), rt_b + 32 * n_batch * sizeof(float), mu, vv);
+ Q6_vgather_ARMVw((HVX_Vector *) (head->b_f32[curr_buf] + 32), rt_b + 32 * vtcm_stride, mu, vv);
}
const uint32_t t_chunk = bctx->c * 64;
diff --git a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h
index 0625c4d44..0d8818588 100644
--- a/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h
+++ b/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h
@@ -204,7 +204,7 @@ static inline void htp_gdn_hmx_vtcm_layout_build(
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_g_f32[1], bh * dma_scalar_sz, pipeline);
VTCM_LAYOUT_ALLOC(off, off_b_f32[0], bh * dma_scalar_sz);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_b_f32[1], bh * dma_scalar_sz, pipeline);
- const size_t raw_gb_sz = hex_round_up(bh * chunk_size * sizeof(float), 128);
+ const size_t raw_gb_sz = hex_round_up((bh + 1) * chunk_size * sizeof(float), 128);
VTCM_LAYOUT_ALLOC(off, off_g_raw[0], raw_gb_sz);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_g_raw[1], raw_gb_sz, pipeline);
VTCM_LAYOUT_ALLOC(off, off_b_raw[0], raw_gb_sz);
diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.c b/ggml/src/ggml-hexagon/htp/ssm-conv.c
index dae339e4d..4b06b18f2 100644
--- a/ggml/src/ggml-hexagon/htp/ssm-conv.c
+++ b/ggml/src/ggml-hexagon/htp/ssm-conv.c
@@ -173,11 +173,12 @@ static inline void hvx_ssm_conv_decode_4(const float * x, const float * w, float
// the lanes past the channel tail.
static inline void hvx_ssm_conv_transpose_block(const float * raw_block,
float * T,
+ uint32_t raw_stride_bytes,
uint32_t ncs,
uint32_t cb_n,
HVX_Vector vv) {
const size_t base = (size_t) raw_block;
- const uint32_t mu = cb_n * ncs * sizeof(float) - 1;
+ const uint32_t mu = cb_n * raw_stride_bytes - 1;
for (uint32_t t = 0; t < ncs; ++t) {
Q6_vgather_ARMVw((HVX_Vector *) (T + (size_t) t * VLEN_FP32), base + t * sizeof(float), mu, vv);
@@ -393,7 +394,8 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
// src0 spad holds two raw tiles (fetch of tile n+1 overlaps compute of tile n) plus
// the transposed block. dst spad holds two tiles so a writeback can stay in flight.
- const size_t src0_tile_raw_bytes = hex_round_up(d_inner_tile * ncs * sizeof(float), 128);
+ const size_t raw_stride_bytes = ncs * sizeof(float) + sizeof(float);
+ const size_t src0_tile_raw_bytes = hex_round_up(d_inner_tile * raw_stride_bytes, 128);
const size_t dst_tile_bytes = hex_round_up(d_inner_tile * n_t * sizeof(float), 128);
float * src0_tile_raw[2] = { (float *) src0_spad_base, (float *) (src0_spad_base + src0_tile_raw_bytes) };
@@ -419,7 +421,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
(ir0 + tile_off_fetch) * src0_stride_inner_bytes; \
dma_queue_push(dma_q, \
dma_make_data((uint8_t *) src0_tile_raw[(c) & 1], fetch_ddr), \
- row_bytes, src0_stride_inner_bytes, row_bytes, cur_tile_n); \
+ raw_stride_bytes, src0_stride_inner_bytes, row_bytes, cur_tile_n); \
tile_off_fetch += d_inner_tile; \
if (tile_off_fetch >= d_inner_per_thread) { \
tile_off_fetch = 0; \
@@ -446,11 +448,7 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
const uint32_t C_TILE = VLEN_FP32;
// gather offsets: lane r reads channel r of the raw tile block
- uint32_t __attribute__((aligned(VLEN))) gather_off[VLEN_FP32];
- for (uint32_t r = 0; r < VLEN_FP32; ++r) {
- gather_off[r] = r * ncs * sizeof(float);
- }
- const HVX_Vector vv = *(const HVX_Vector *) gather_off;
+ const HVX_Vector vv = hvx_vec_gather_offsets_w(raw_stride_bytes);
uint32_t i3 = 0;
uint32_t tile_off = 0;
@@ -474,7 +472,8 @@ static void ssm_conv_thread_f32_prefill(unsigned int nth, unsigned int ith, void
const uint32_t cb_n = MIN(C_TILE, tile_n - cb);
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) tile_off);
- hvx_ssm_conv_transpose_block(raw + (size_t) cb * ncs, src0_T, ncs, cb_n, vv);
+ const float * raw_ch = (const float *) ((const uint8_t *) raw + (size_t) cb * raw_stride_bytes);
+ hvx_ssm_conv_transpose_block(raw_ch, src0_T, raw_stride_bytes, ncs, cb_n, vv);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) tile_off);
const float * restrict wp = src1_T + tile_off + cb;
diff --git a/ggml/src/ggml-hexagon/htp/ssm-conv.h b/ggml/src/ggml-hexagon/htp/ssm-conv.h
index 43b2575a2..cd4180857 100644
--- a/ggml/src/ggml-hexagon/htp/ssm-conv.h
+++ b/ggml/src/ggml-hexagon/htp/ssm-conv.h
@@ -54,11 +54,12 @@ static inline void htp_ssm_conv_vtcm_layout_build(
L->src0_bytes_per_thread = 2 * src0_tile_raw_bytes + src0_T_bytes;
L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * sizeof(float), 128);
} else {
- const uint32_t ncs = d_conv - 1 + n_t;
- const size_t src0_block_T = hex_round_up((size_t) ncs * 32 * sizeof(float), 128);
- const size_t src0_tile_raw = hex_round_up((size_t) d_inner_tile * ncs * sizeof(float), 128);
- L->src0_bytes_per_thread = 2 * src0_tile_raw + src0_block_T;
- L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * n_t * sizeof(float), 128);
+ const uint32_t ncs = d_conv - 1 + n_t;
+ const size_t raw_stride_bytes = ncs * sizeof(float) + sizeof(float);
+ const size_t src0_block_T = hex_round_up((size_t) ncs * 32 * sizeof(float), 128);
+ const size_t src0_tile_raw = hex_round_up((size_t) d_inner_tile * raw_stride_bytes, 128);
+ L->src0_bytes_per_thread = 2 * src0_tile_raw + src0_block_T;
+ L->dst_bytes_per_thread = 2 * hex_round_up((size_t) d_inner_tile * n_t * sizeof(float), 128);
}
L->src0_bytes = L->src0_bytes_per_thread * n_threads;