Commit a30fb4924 for llama.cpp
commit a30fb4924dcfec81c1b2c20634bd47fa4403061a
Author: Michael Taylor <162068037+mctylr-gh@users.noreply.github.com>
Date: Fri Oct 16 04:28:50 2026 -0300
[SYCL]: fix memory errors in mul_mat, split buffer, host pool (#29889)
diff --git a/ggml/src/ggml-sycl/common.cpp b/ggml/src/ggml-sycl/common.cpp
index 894006949..f01d0ff95 100644
--- a/ggml/src/ggml-sycl/common.cpp
+++ b/ggml/src/ggml-sycl/common.cpp
@@ -115,7 +115,8 @@ void * ggml_sycl_malloc_device(size_t size, sycl::queue &q, ggml_sycl_mem_type t
#else
ze_device_mem_alloc_desc_t alloc_desc = {ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, nullptr, 0, 0};
#endif
- ze_result_t r = zeMemAllocDevice(ze_ctx, &alloc_desc, size, 64, ze_dev, &ptr);
+ // request SYCL_BUFFER_ALIGNMENT so tensor offsets aligned to it stay aligned in absolute terms
+ ze_result_t r = zeMemAllocDevice(ze_ctx, &alloc_desc, size, SYCL_BUFFER_ALIGNMENT, ze_dev, &ptr);
if (r == ZE_RESULT_SUCCESS && ptr) {
ggml_sycl_memtrace_add(type, ptr, size);
return ptr;
diff --git a/ggml/src/ggml-sycl/cpy.cpp b/ggml/src/ggml-sycl/cpy.cpp
index ef7413abd..8e1f1d50a 100644
--- a/ggml/src/ggml-sycl/cpy.cpp
+++ b/ggml/src/ggml-sycl/cpy.cpp
@@ -1255,6 +1255,8 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co
scope_op_debug_print scope_dbg_print(__func__, src1, /*num_src=*/0, debug_get_tensor_str("\tsrc0", src0));
const int64_t ne = ggml_nelements(src0);
GGML_ASSERT(ne == ggml_nelements(src1));
+ // the cpy kernels index in 32-bit
+ GGML_ASSERT(ne <= INT32_MAX);
GGML_TENSOR_BINARY_OP_LOCALS01;
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp
index 1cfc84126..b380cbedf 100644
--- a/ggml/src/ggml-sycl/ggml-sycl.cpp
+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp
@@ -899,6 +899,10 @@ static bool ggml_sycl_is_l0_discrete_gpu(int device) {
static void memcpy_host_forward(sycl::queue &q_dst, sycl::queue &q_src, void *ptr_dst,
const void *ptr_src, size_t size) {
char *host_buf = (char *)malloc(size);
+ if (host_buf == nullptr) {
+ GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory for host staging\n", __func__, size);
+ GGML_ABORT("fatal error");
+ }
q_src.memcpy(host_buf, (const char *)ptr_src, size).wait();
q_dst.memcpy((char *)ptr_dst, host_buf, size).wait();
free(host_buf);
@@ -1177,12 +1181,13 @@ bool is_bmg_g31_arch(int device) {
}
static size_t ggml_backend_sycl_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
- size_t max_alloc_size = dpct::get_current_device().get_max_mem_alloc_size();
+ ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context;
+ int device = ctx->device;
+ // use the buft device, the current device may differ on multi-GPU systems
+ size_t max_alloc_size = dpct::get_device(device).get_max_mem_alloc_size();
if (g_ggml_sycl_host_pinned_mem_2g) {
return std::min(max_alloc_size, (size_t) 2LL*1024*1024*1024);
} else {
- ggml_backend_sycl_buffer_type_context * ctx = (ggml_backend_sycl_buffer_type_context *)buft->context;
- int device = ctx->device;
if(is_bmg_g31_arch(device)) {
//Todo, it's workaround for BMG-G31, which has a known issue with large allocations.
//The max alloc size is reduced to 60% of the reported max alloc size.
@@ -1398,7 +1403,12 @@ ggml_backend_sycl_split_buffer_init_tensor(ggml_backend_buffer_t buffer,
ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};
ctx->tensor_extras.push_back(extra);
- ctx->streams.push_back(&(dpct::get_current_device().default_queue()));
+ // consumers index the queues by device id, so size the vector once with one queue per device
+ if (ctx->streams.empty()) {
+ for (int i = 0; i < ggml_sycl_info().device_count; ++i) {
+ ctx->streams.push_back(&(dpct::get_device(i).default_queue()));
+ }
+ }
for (int i = 0; i < ggml_sycl_info().device_count; ++i) {
int64_t row_low, row_high;
@@ -2089,8 +2099,6 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {
queue_ptr qptr;
int device;
- inline static int counter{ 0 };
-
struct ggml_sycl_buffer {
void * ptr = nullptr;
size_t size = 0;
@@ -2100,6 +2108,7 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {
static constexpr int MAX_POOL_SIZE{ 64 };
std::vector<ggml_sycl_buffer> buffer_pool = std::vector<ggml_sycl_buffer>(MAX_POOL_SIZE);
size_t pool_size = 0;
+ int counter = 0;
explicit ggml_sycl_pool_host(queue_ptr qptr_, int device_) : qptr(qptr_), device(device_) {}
@@ -2113,41 +2122,55 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {
b.size = 0;
}
}
- counter = 0;
}
void * alloc(size_t size, size_t * actual_size) override {
if (counter == MAX_POOL_SIZE) {
- ggml_sycl_buffer b = buffer_pool[0];
- void * ptr = b.ptr;
- *actual_size = b.size;
- counter = 1;
- return ptr;
- }
- ggml_sycl_buffer & b = buffer_pool[counter];
-
- if (b.ptr == nullptr) {
- void * ptr;
-
+ // the pool wrapped around, no slot is reserved for this allocation
+ void * ptr = nullptr;
SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr)));
if (!ptr) {
GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size);
return nullptr;
}
- pool_size += size;
+ pool_size += size;
*actual_size = size;
- counter = counter + 1;
+ counter = 0;
return ptr;
- } else {
- ++counter;
- b.size = size;
- return b.ptr;
}
+ ggml_sycl_buffer & b = buffer_pool[counter];
+ ++counter;
+
+ if (b.ptr != nullptr) {
+ if (b.size >= size) {
+ // hand out the cached buffer, free() puts it back into the pool
+ void * ptr = b.ptr;
+ *actual_size = b.size;
+ b.ptr = nullptr;
+ b.size = 0;
+ return ptr;
+ }
+ // the cached buffer is too small, replace it
+ SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(b.ptr, *qptr)));
+ pool_size -= b.size;
+ b.ptr = nullptr;
+ b.size = 0;
+ }
+
+ void * ptr;
+ SYCL_CHECK(CHECK_TRY_ERROR(ptr = (void *) sycl::malloc_host(size, *qptr)));
+ if (!ptr) {
+ GGML_LOG_ERROR("%s: can't allocate %zu Bytes of memory on host\n", __func__, size);
+ return nullptr;
+ }
+ pool_size += size;
+ *actual_size = size;
+ return ptr;
}
void free(void * ptr, size_t size) override {
// if the pool is not completed add the pointer to it in place of the first nullptr found.
- // Otherwise do nothing, pointers will be freed once the pool is deallocated.
+ // Otherwise free the buffer, all slots are cached.
for (int i = 0; i < MAX_POOL_SIZE; ++i) {
ggml_sycl_buffer & b = buffer_pool[i];
if (b.ptr == nullptr) {
@@ -2156,6 +2179,8 @@ struct ggml_sycl_pool_host : public ggml_sycl_pool {
return;
}
}
+ SYCL_CHECK(CHECK_TRY_ERROR(sycl::free(ptr, *qptr)));
+ pool_size -= size;
}
};
@@ -3154,8 +3179,8 @@ inline void ggml_sycl_op_mul_mat_sycl(
to_fp16_sycl(src1_ddf_i, src1_as_f16.get(), ne, stream);
}
const sycl::half *src1_ptr = src1->type == GGML_TYPE_F16
- ? (const sycl::half *)src1->data + src1_padded_row_size
- : src1_as_f16.get();
+ ? (const sycl::half *)src1_ddf_i
+ : src1_as_f16.get();
ggml_sycl_pool_alloc<sycl::half> src0_as_f16(ctx.pool());
if (src0->type != GGML_TYPE_F16) {
@@ -3252,6 +3277,8 @@ inline void ggml_sycl_op_sum(ggml_backend_sycl_context & ctx, ggml_tensor *dst)
float * dst_dd = static_cast<float *>(dst->data);
const int64_t ne = ggml_nelements(dst->src[0]);
+ // the kernel indexes in 32-bit
+ GGML_ASSERT(ne <= INT32_MAX);
sum_rows_f32_sycl(src0_dd, dst_dd, ne, 1, main_stream);
}
@@ -3266,6 +3293,8 @@ inline void ggml_sycl_op_sum_rows(ggml_backend_sycl_context & ctx, ggml_tensor *
const int64_t ncols = dst->src[0]->ne[0];
const int64_t nrows = ggml_nrows(dst->src[0]);
+ // the kernel indexes in 32-bit
+ GGML_ASSERT(ncols * nrows <= INT32_MAX);
sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream);
}
@@ -3282,6 +3311,8 @@ inline void ggml_sycl_op_mean(ggml_backend_sycl_context & ctx, ggml_tensor * dst
const int64_t ncols = dst->src[0]->ne[0];
const int64_t nrows = ggml_nrows(dst->src[0]);
+ // the kernel indexes in 32-bit
+ GGML_ASSERT(ncols * nrows <= INT32_MAX);
sum_rows_f32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream);
@@ -3351,6 +3382,8 @@ inline void ggml_sycl_op_argmax(ggml_backend_sycl_context & ctx, ggml_tensor * d
const int64_t ncols = dst->src[0]->ne[0];
const int64_t nrows = ggml_nrows(dst->src[0]);
+ // the kernel indexes in 32-bit
+ GGML_ASSERT(ncols * nrows <= INT32_MAX);
argmax_f32_i32_sycl(src0_dd, dst_dd, ncols, nrows, main_stream);
}
@@ -3366,6 +3399,8 @@ inline void ggml_sycl_op_diag_mask_inf(ggml_backend_sycl_context & ctx, ggml_ten
const int64_t ne00 = dst->src[0]->ne[0];
const int64_t ne01 = dst->src[0]->ne[1];
const int nrows0 = ggml_nrows(dst->src[0]);
+ // the kernel indexes in 32-bit
+ GGML_ASSERT(ne00 * nrows0 <= INT32_MAX);
const int n_past = ((int32_t *) dst->op_params)[0];
@@ -3444,6 +3479,9 @@ inline void ggml_sycl_op_scale(ggml_backend_sycl_context & ctx, ggml_tensor * ds
memcpy(&scale, (float *) dst->op_params + 0, sizeof(float));
memcpy(&bias, (float *) dst->op_params + 1, sizeof(float));
+ // the kernel indexes in 32-bit
+ GGML_ASSERT(ggml_nelements(dst->src[0]) <= INT32_MAX);
+
scale_f32_sycl(src0_dd, dst_dd, scale, bias, ggml_nelements(dst->src[0]), main_stream);
/*
DPCT1010:87: SYCL uses exceptions to report errors and does not use the
@@ -3524,7 +3562,6 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten
const size_t q8_1_bs = QK8_1;
ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;
- ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra;
const bool src0_is_contiguous = ggml_is_contiguous(src0);
const bool src1_is_contiguous = ggml_is_contiguous(src1);
@@ -3606,7 +3643,8 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten
queue_ptr stream = ctx.stream(i, 0);
if (src0_is_contiguous) {
- dev[i].src0_dd = (char *) src0->data;
+ // a split tensor keeps its rows in the per-device slices of the tensor extra
+ dev[i].src0_dd = split ? (char *) src0_extra->data_device[i] : (char *) src0->data;
} else {
dev[i].src0_dd = dev[i].src0_dd_alloc.alloc(ctx.pool(i), ggml_nbytes(src0));
}
@@ -3702,7 +3740,8 @@ static void ggml_sycl_op_mul_mat(ggml_backend_sycl_context & ctx, const ggml_ten
src1_ncols * src1_padded_col_size * q8_1_ts / q8_1_bs)
.wait()));
} else {
- const char * src1_ddf_i_source = (const char *) src1_extra->data_device[ctx.device] +
+ // src1 is not split, on the main device it lives at dev[ctx.device].src1_ddf
+ const char * src1_ddf_i_source = (const char *) dev[ctx.device].src1_ddf +
(i0 * ne11 + src1_col_0) * ne10 * ggml_type_size(src1->type);
SYCL_CHECK(
@@ -5269,6 +5308,7 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, gg
(int) ne11,
/*stride_col_y=*/src1_padded_cols / QK8_1,
/*stride_col_dst=*/(int) glu->ne[0], stream);
+ GGML_UNUSED(gate);
}
// Fused dense-FFN mat-vec for the {mul_mat(gate), mul_mat(up), GLU} subgraph at node_idx.
@@ -6860,7 +6900,9 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
case GGML_GLU_OP_GEGLU_ERF:
case GGML_GLU_OP_GEGLU_QUICK:
case GGML_GLU_OP_SWIGLU_CLAMP:
- return ggml_is_contiguous_1(op->src[0]);
+ // the fused GLU kernels only have f32/f16 instantiations
+ return ggml_is_contiguous_1(op->src[0]) &&
+ (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16);
default:
return false;
}
@@ -7123,7 +7165,8 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
case GGML_OP_RMS_NORM_BACK:
return ggml_is_contiguous(op->src[0]);
case GGML_OP_SCALE:
- return true;
+ // the scale kernel is f32 only
+ return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32;
case GGML_OP_CONT:
return true;
case GGML_OP_TRI:
@@ -7156,12 +7199,19 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
#endif
) &&
op->src[0]->type == op->type;
- case GGML_OP_CONV_3D:
+ case GGML_OP_CONV_3D: {
+ const int32_t * opts = (const int32_t *) op->op_params;
+ // zero kernel volume, channels or output makes the im2col GEMM degenerate (k == 0 or m == 0)
+ // and MKL BLAS rejects it; let those fall back to the CPU backend
+ const int64_t knl_n_total = op->src[0]->ne[0] * op->src[0]->ne[1] * op->src[0]->ne[2] * opts[9];
+ const int64_t patch_total = op->ne[0] * op->ne[1] * op->ne[2] * opts[10];
return op->type == GGML_TYPE_F32 &&
(op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
op->src[1]->type == GGML_TYPE_F32 &&
ggml_is_contiguous(op->src[0]) &&
- ggml_is_contiguous(op->src[1]);
+ ggml_is_contiguous(op->src[1]) &&
+ knl_n_total > 0 && patch_total > 0;
+ }
case GGML_OP_SUM:
case GGML_OP_SUM_ROWS:
case GGML_OP_MEAN:
diff --git a/ggml/src/ggml-sycl/roll.cpp b/ggml/src/ggml-sycl/roll.cpp
index 1e0518178..a95cee9a9 100644
--- a/ggml/src/ggml-sycl/roll.cpp
+++ b/ggml/src/ggml-sycl/roll.cpp
@@ -73,6 +73,8 @@ void ggml_sycl_roll(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
const ggml_tensor *src = dst->src[0];
GGML_ASSERT(src && src->type == GGML_TYPE_F32);
+ // the kernel indexes in 32-bit
+ GGML_ASSERT(dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3] <= INT32_MAX);
const int ne0 = (int) dst->ne[0];
const int ne1 = (int) dst->ne[1];
const int ne2 = (int) dst->ne[2];