Commit 4d71b42fd for llama.cpp
commit 4d71b42fd87db24b8b7c808c151b713aa4459cfd
Author: Pascal <admin@serveurperso.com>
Date: Mon Oct 26 15:18:45 2026 +0100
ggml-cpu : rewrite concat as one row kernel for all types (#31179)
Every dst row comes from src0 or src1, or for dim 0 from the row of
src0 followed by the row of src1, so a single kernel covers every
type: a dense source row is one memcpy, quantized types included,
and a strided source is copied element by element with the element
size known at compile time.
Rows are split across threads instead of i2 alone, which is often 1,
and are copied in tiles of 32: for a transposed source the elements
are walked in the outer loop, so the rows of a tile read neighboring
addresses instead of jumping a full row stride per element.
Add transposed b cases to test_concat, the layout of the delta-net
conv input, and a perf case for it.
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index 5735f96f8..afeda2aff 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -1914,213 +1914,87 @@ void ggml_compute_forward_repeat_back(
}
// ggml_compute_forward_concat
-static void ggml_compute_forward_concat_any(
- const ggml_compute_params * params,
- ggml_tensor * dst) {
-
- const ggml_tensor * src0 = dst->src[0];
- const ggml_tensor * src1 = dst->src[1];
-
- const int ith = params->ith;
- const int nth = params->nth;
-
- GGML_TENSOR_BINARY_OP_LOCALS
-
- const int32_t dim = ggml_get_op_params_i32(dst, 0);
-
- GGML_ASSERT(dim >= 0 && dim < 4);
- GGML_ASSERT(ggml_is_contiguous_rows(src0));
- GGML_ASSERT(ggml_is_contiguous_rows(src1));
-
- int64_t o[4] = {0, 0, 0, 0};
- if (dim == 0) {
- GGML_ASSERT(src0->ne[0] % ggml_blck_size(src0->type) == 0);
- GGML_ASSERT(src1->ne[0] % ggml_blck_size(src1->type) == 0);
+// rows of dst copied together, so that a strided source is read along neighboring rows
+static constexpr int64_t GGML_CONCAT_TILE = 32;
- o[dim] = src0->ne[dim]/ggml_blck_size(src0->type);
- } else {
- o[dim] = src0->ne[dim];
- }
-
- // Region 1: copy rows from src0
- for (int i3 = 0; i3 < ne03; i3++) {
- for (int i2 = ith; i2 < ne02; i2 += nth) {
- for (int i1 = 0; i1 < ne01; i1++) {
- const char * x = (const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03;
- char * y = ( char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3;
- memcpy(y, x, ggml_row_size(src0->type, ne00));
- }
- }
- }
-
- // Region 2: copy rows from src1, offset into dst by o[]
- for (int i3 = 0; i3 < ne13; i3++) {
- for (int i2 = ith; i2 < ne12; i2 += nth) {
- for (int i1 = 0; i1 < ne11; i1++) {
- const char * x = (const char *) src1->data + i1*nb11 + i2*nb12 + i3*nb13;
- char * y = ( char *) dst->data + (i1 + o[1])*nb1 + (i2 + o[2])*nb2 + (i3 + o[3])*nb3 + o[0]*nb0;
- memcpy(y, x, ggml_row_size(src1->type, ne10));
- }
+// copy n_rows rows of n elements of es bytes from a source with element stride nb0, element by element
+template <size_t es>
+static void ggml_concat_copy_strided(char * y, size_t nby, const char * x, size_t nbx, size_t nb0, int64_t n, int64_t n_rows) {
+ for (int64_t i0 = 0; i0 < n; i0++) {
+ for (int64_t r = 0; r < n_rows; r++) {
+ memcpy(y + r*nby + i0*es, x + r*nbx + i0*nb0, es);
}
}
}
-static void ggml_compute_forward_concat_i8(
- const ggml_compute_params * params,
- ggml_tensor * dst) {
-
- const ggml_tensor * src0 = dst->src[0];
- const ggml_tensor * src1 = dst->src[1];
-
- GGML_ASSERT(ggml_type_size(src0->type) == sizeof(int8_t));
+// copy n_rows rows of n elements of src into dst, rows are nbx bytes apart in src and nby bytes apart in dst
+static void ggml_concat_copy_rows(char * y, size_t nby, const ggml_tensor * src, const char * x, size_t nbx, int64_t n, int64_t n_rows) {
+ const size_t es = ggml_type_size(src->type);
- const int ith = params->ith;
- const int nth = params->nth;
-
- GGML_TENSOR_BINARY_OP_LOCALS
-
- const int32_t dim = ggml_get_op_params_i32(dst, 0);
-
- GGML_ASSERT(dim >= 0 && dim < 4);
-
- int64_t o[4] = {0, 0, 0, 0};
- o[dim] = src0->ne[dim];
-
- const int8_t * x;
-
- // TODO: smarter multi-theading
- for (int i3 = 0; i3 < ne3; i3++) {
- for (int i2 = ith; i2 < ne2; i2 += nth) {
- for (int i1 = 0; i1 < ne1; i1++) {
- for (int i0 = 0; i0 < ne0; i0++) {
- if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
- x = (const int8_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);
- } else {
- x = (const int8_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
- }
-
- int8_t * y = (int8_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
-
- *y = *x;
- }
- }
+ if (src->nb[0] == es) {
+ const size_t row_size = ggml_row_size(src->type, n);
+ for (int64_t r = 0; r < n_rows; r++) {
+ memcpy(y + r*nby, x + r*nbx, row_size);
}
+ return;
}
-}
-
-static void ggml_compute_forward_concat_f16(
- const ggml_compute_params * params,
- ggml_tensor * dst) {
- const ggml_tensor * src0 = dst->src[0];
- const ggml_tensor * src1 = dst->src[1];
-
- GGML_ASSERT(ggml_type_size(src0->type) == sizeof(ggml_fp16_t));
-
- const int ith = params->ith;
- const int nth = params->nth;
-
- GGML_TENSOR_BINARY_OP_LOCALS
-
- const int32_t dim = ggml_get_op_params_i32(dst, 0);
-
- GGML_ASSERT(dim >= 0 && dim < 4);
-
- int64_t o[4] = {0, 0, 0, 0};
- o[dim] = src0->ne[dim];
-
- const ggml_fp16_t * x;
-
- // TODO: smarter multi-theading
- for (int i3 = 0; i3 < ne3; i3++) {
- for (int i2 = ith; i2 < ne2; i2 += nth) {
- for (int i1 = 0; i1 < ne1; i1++) {
- for (int i0 = 0; i0 < ne0; i0++) {
- if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
- x = (const ggml_fp16_t *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);
- } else {
- x = (const ggml_fp16_t *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
- }
-
- ggml_fp16_t * y = (ggml_fp16_t *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
-
- *y = *x;
- }
- }
- }
+ switch (es) {
+ case 1: ggml_concat_copy_strided<1>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
+ case 2: ggml_concat_copy_strided<2>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
+ case 4: ggml_concat_copy_strided<4>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
+ case 8: ggml_concat_copy_strided<8>(y, nby, x, nbx, src->nb[0], n, n_rows); break;
+ default: GGML_ABORT("unsupported element size");
}
}
-static void ggml_compute_forward_concat_f32(
+// each dst row comes from src0 or src1, or for dim 0 from the row of src0 followed by the row of src1
+void ggml_compute_forward_concat(
const ggml_compute_params * params,
ggml_tensor * dst) {
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
- GGML_ASSERT(ggml_type_size(src0->type) == sizeof(float));
-
- const int ith = params->ith;
- const int nth = params->nth;
-
GGML_TENSOR_BINARY_OP_LOCALS
const int32_t dim = ggml_get_op_params_i32(dst, 0);
GGML_ASSERT(dim >= 0 && dim < 4);
+ GGML_ASSERT(nb0 == ggml_type_size(dst->type));
+ GGML_ASSERT(!ggml_is_quantized(dst->type) || (nb00 == nb0 && nb10 == nb0));
+ GGML_ASSERT(dim != 0 || (ne00 % ggml_blck_size(dst->type) == 0 && ne10 % ggml_blck_size(dst->type) == 0));
- int64_t o[4] = {0, 0, 0, 0};
- o[dim] = src0->ne[dim];
+ const size_t y1 = dim == 0 ? ggml_row_size(dst->type, ne00) : 0;
- const float * x;
+ const int64_t nr = ne1*ne2*ne3;
+ const int64_t dr = (nr + params->nth - 1)/params->nth;
- // TODO: smarter multi-theading
- for (int i3 = 0; i3 < ne3; i3++) {
- for (int i2 = ith; i2 < ne2; i2 += nth) {
- for (int i1 = 0; i1 < ne1; i1++) {
- for (int i0 = 0; i0 < ne0; i0++) {
- if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
- x = (const float *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);
- } else {
- x = (const float *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
- }
+ const int64_t ir0 = dr*params->ith;
+ const int64_t ir1 = MIN(ir0 + dr, nr);
- float * y = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
+ for (int64_t ir = ir0; ir < ir1; ) {
+ int64_t i[4] = { 0, ir % ne1, (ir/ne1) % ne2, ir/(ne1*ne2) };
- *y = *x;
- }
- }
+ // a tile of consecutive rows sharing i2, i3 and their source
+ int64_t nt = MIN(MIN(ir1 - ir, ne1 - i[1]), GGML_CONCAT_TILE);
+ if (dim == 1 && i[1] < ne01) {
+ nt = MIN(nt, ne01 - i[1]);
}
- }
-}
-void ggml_compute_forward_concat(
- const ggml_compute_params * params,
- ggml_tensor * dst) {
+ char * y = (char *) dst->data + i[1]*nb1 + i[2]*nb2 + i[3]*nb3;
- const ggml_tensor * src0 = dst->src[0];
+ if (dim == 0 || i[dim] < src0->ne[dim]) {
+ ggml_concat_copy_rows(y, nb1, src0, (const char *) src0->data + i[1]*nb01 + i[2]*nb02 + i[3]*nb03, nb01, ne00, nt);
+ }
- switch (src0->type) {
- case GGML_TYPE_F16:
- case GGML_TYPE_BF16:
- case GGML_TYPE_I16:
- {
- ggml_compute_forward_concat_f16(params, dst);
- } break;
- case GGML_TYPE_I8:
- {
- ggml_compute_forward_concat_i8(params, dst);
- } break;
- case GGML_TYPE_F32:
- case GGML_TYPE_I32:
- {
- ggml_compute_forward_concat_f32(params, dst);
- } break;
- default:
- {
- ggml_compute_forward_concat_any(params, dst);
- }
+ if (dim == 0 || i[dim] >= src0->ne[dim]) {
+ i[dim] -= src0->ne[dim]; // i[0] does not address rows
+ ggml_concat_copy_rows(y + y1, nb1, src1, (const char *) src1->data + i[1]*nb11 + i[2]*nb12 + i[3]*nb13, nb11, ne10, nt);
+ }
+
+ ir += nt;
}
}
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index b6d2b4e55..011b74613 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -7126,7 +7126,7 @@ struct test_concat : public test_case {
const std::array<int64_t, 4> ne_a;
const int64_t ne_b_d;
const int dim;
- const int v; // view (1 << 0: non-cont a (first 3 dim), 1 << 1: non-cont b (first 3 dim), 1 << 2: non-cont a (last 2 dim), 1 << 3: non-cont b (last 2 dim))
+ const int v; // view (1 << 0: non-cont a (first 3 dim), 1 << 1: non-cont b (first 3 dim), 1 << 2: non-cont a (last 2 dim), 1 << 3: non-cont b (last 2 dim), 1 << 4: transposed b)
std::string vars() override {
return VARS_TO_STR5(type, ne_a, ne_b_d, dim, v);
@@ -7175,6 +7175,13 @@ struct test_concat : public test_case {
b = ggml_view_4d(ctx, b, ne_b[0], ne_b[1], ne_b[2], ne_b[3], b->nb[1], b->nb[2], b->nb[3], 0);
ggml_set_name(b, "view_of_b");
+ } else if (v & 16) {
+ auto ne = ne_b; std::swap(ne[0], ne[1]);
+ b = ggml_new_tensor(ctx, type, 4, ne.data());
+ ggml_set_name(b, "b");
+
+ b = ggml_transpose(ctx, b);
+ ggml_set_name(b, "transpose_of_b");
} else {
b = ggml_new_tensor(ctx, type, 4, ne_b.data());
ggml_set_name(b, "b");
@@ -11803,7 +11810,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 256, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // gemma4 E2B sliding
test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {512, 8, 512, 1}, 128, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, true, 0, true)); // gemma4 E4B global
- for (int v : { 0, 1, 2, 3 }) {
+ for (int v : { 0, 1, 2, 3, 16 }) {
for (int dim : { 0, 1, 2, 3, }) {
test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {11, 12, 13, 14}, 7, dim, v));
test_cases.emplace_back(new test_concat(GGML_TYPE_F16, {11, 12, 13, 14}, 7, dim, v));
@@ -12531,6 +12538,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
std::vector<std::unique_ptr<test_case>> test_cases;
+ // delta-net conv input: conv state followed by the transposed qkv of the ubatch
+ test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {3, 8192, 1, 1}, 512, 0, 16));
+
// SWIGLU at a 27B-class FFN width, fused [gate|up] vs split operands
// note: same bytes either way, so a backend that indexes them differently shows it here
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) {