Commit e707790c7 for llama.cpp
commit e707790c73590c2ff5303045b714afb0c5e22035
Author: Frosty40 <newjordan@gmail.com>
Date: Thu Oct 15 20:59:38 2026 -0500
sycl: support reordered Q8_0 MoE weights (#29337)
Preserve the existing MoE accumulation order.
Assisted-by: Codex
Assisted-by: Angel
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp
index 6759af0ed..1cfc84126 100644
--- a/ggml/src/ggml-sycl/ggml-sycl.cpp
+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp
@@ -4442,6 +4442,47 @@ static bool reorder_qw_q8_0(uint8_t * data_device, const int ncols, const int nr
return true;
}
+// Reorder each Q8_0 expert slice into a self-contained [qs][d] SoA layout.
+static bool reorder_qw_q8_0_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) {
+ GGML_ASSERT(expert_bytes % sizeof(block_q8_0) == 0);
+ const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q8_0));
+ const size_t total_bytes = expert_bytes * (size_t) n_expert;
+
+ sycl_reorder_temp_buffer tmp(stream, total_bytes);
+ if (!tmp) {
+ GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, total_bytes);
+ return false;
+ }
+ uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr);
+
+ sycl::event copy_event;
+ SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, total_bytes)));
+ if (!g_ggml_sycl_use_async_mem_op) {
+ copy_event.wait();
+ }
+
+ const int total_blocks = blocks_per_expert * (int) n_expert;
+ auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) {
+ const int gb = gb_;
+ const int e = gb / blocks_per_expert;
+ const int ib = gb % blocks_per_expert;
+ const block_q8_0 * x = (const block_q8_0 *) (tmp_buf + (size_t) e * expert_bytes);
+ uint8_t * base = data_device + (size_t) e * expert_bytes;
+
+ auto * qs_ptr = base;
+ auto * d_ptr = (sycl::half *) (qs_ptr + (size_t) blocks_per_expert * QK8_0);
+
+ for (int j = 0; j < QK8_0; ++j) {
+ *((int8_t *) qs_ptr + ib * QK8_0 + j) = x[ib].qs[j];
+ }
+ d_ptr[ib] = x[ib].d;
+ });
+ if (!g_ggml_sycl_use_async_mem_op) {
+ reorder_event.wait_and_throw();
+ }
+ return true;
+}
+
static bool reorder_qw_q4_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) {
GGML_ASSERT(size % sizeof(block_q4_K) == 0);
GGML_ASSERT(offset % sizeof(block_q4_K) == 0);
@@ -4966,6 +5007,8 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) {
if (src0->ne[2] > 1) {
GGML_ASSERT((size_t) size == (size_t) src0->ne[2] * src0->nb[2]);
switch (src0->type) {
+ case GGML_TYPE_Q8_0:
+ return reorder_qw_q8_0_moe(data_device, src0->nb[2], src0->ne[2], stream);
case GGML_TYPE_Q4_K:
return reorder_qw_q4_k_moe(data_device, src0->nb[2], src0->ne[2], stream);
case GGML_TYPE_Q5_K:
@@ -5051,8 +5094,8 @@ static void opt_for_reorder_id(ggml_backend_sycl_context * ctx, const ggml_tenso
if (!g_ggml_sycl_enable_optimize || !ctx->opt_feature.reorder) {
return;
}
- if (src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K &&
- src0->type != GGML_TYPE_MXFP4) {
+ if (src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_Q4_K &&
+ src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_MXFP4) {
return;
}
// The MXFP4 reorder kernels use 8-byte vector loads, so every expert slice must stay aligned.
@@ -5438,7 +5481,7 @@ static bool ggml_sycl_mul_mat_id_mmvq_fused(
const int n_experts_used = (int) n_ids_per_group;
const int nrows = (int) src0->ne[1];
- // Lazily reorder the (Q4_K) expert weights into a per-expert SoA layout, then run the reorder
+ // Lazily reorder supported expert weights into a per-expert SoA layout, then run the reorder
// GEMV. Placed after the bail checks so a non-dispatchable op does not pay the reorder cost.
opt_for_reorder_id(&ctx, src0);
const ggml_tensor_extra_gpu * src0_extra =
diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp
index eade8949f..e9057777c 100644
--- a/ggml/src/ggml-sycl/mmvq.cpp
+++ b/ggml/src/ggml-sycl/mmvq.cpp
@@ -3202,9 +3202,11 @@ static void mul_mat_vec_q_moe_reorder(
const auto sg = item_ct1.get_sub_group();
+ constexpr int vdr_mmvq = reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0 ?
+ VDR_Q8_0_Q8_1_MMVQ : block_traits::vdr_mmvq;
const int blocks_per_row = ncols / block_traits::qk;
- constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi);
- constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq;
+ constexpr int blocks_per_subgroup = ceil_div(vdr_mmvq * WARP_SIZE, block_traits::qi);
+ constexpr int block_elements_per_subgroup = block_traits::qi / vdr_mmvq;
const int nblocks = nrows * (ncols / block_traits::qk);
static_assert(blocks_per_subgroup > 0);
@@ -3223,12 +3225,24 @@ static void mul_mat_vec_q_moe_reorder(
#pragma unroll
for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) {
- const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);
- partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
+ const int iqs = elem + vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);
+ if constexpr (reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0) {
+ partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs, vdr_mmvq);
+ } else {
+ partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
+ }
}
}
- auto sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>());
+ float sum = partial_sum;
+ if constexpr (reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0) {
+ // Keep the accumulation order of the non-reordered MoE kernel.
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
+ sum += dpct::permute_sub_group_by_xor(sg, sum, mask);
+ }
+ } else {
+ sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>());
+ }
if (sg.leader()) {
dst[row] = sum;
}
@@ -3269,6 +3283,11 @@ bool ggml_sycl_mul_mat_vec_q_id_reorder(
size_t src1_row_stride,
dpct::queue_ptr stream) {
switch (src0_type) {
+ case GGML_TYPE_Q8_0:
+ launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0>>(
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
+ return true;
case GGML_TYPE_Q4_K:
launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>>(
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hpp
index d1b6092be..57360a004 100644
--- a/ggml/src/ggml-sycl/vecdotq.hpp
+++ b/ggml/src/ggml-sycl/vecdotq.hpp
@@ -483,7 +483,8 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0> {
__dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
- const sycl::half2 * q8_1_ds, const int & iqs) {
+ const sycl::half2 * q8_1_ds, const int & iqs,
+ const int vdr = q8_0_traits::vdr_mmvq) {
const uint8_t * base = static_cast<const uint8_t *>(vbq);
const int8_t * qs = reinterpret_cast<const int8_t *>(base + ibx_offset.first);
const ggml_half d = *reinterpret_cast<const ggml_half *>(base + d_offset.first);
@@ -492,14 +493,14 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0> {
int u[q8_0_traits::vdr_mmvq];
#pragma unroll
- for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) {
+ for (int i = 0; i < vdr; ++i) {
v[i] = get_int_from_int8(qs, iqs + i);
u[i] = get_int_from_int8_aligned(q8_1_quant_ptr, iqs + i);
}
int sumi = 0;
#pragma unroll
- for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) {
+ for (int i = 0; i < vdr; ++i) {
sumi = dpct::dp4a(v[i], u[i], sumi);
}