Commit bf48103ad for llama.cpp
commit bf48103ad164d919fbad6a28359c3f728e4ad942
Author: Anant Shrivastava <anant@anantshri.info>
Date: Thu Oct 29 11:27:53 2026 +0530
sycl : vectorize reordered-layout dequantization stores (Q4_0, K-quants, IQ3, MXFP4) (#30976)
* sycl : widen reordered q4_0 dequantization stores
The reordered-layout Q4_0 conversion kernel processed one qs byte per
work-item with two scalar stores, strided 16 elements apart. Give it
the same treatment the Q8_0 kernel got in #30775: a flat
sycl::range<1> launch with four qs bytes (8 outputs) per work-item
inside one 32-element block and two vec<dst_t,4> stores.
Same float math and rounding as the scalar kernel; output is
bit-identical. The existing k % QK4_0 == 0 assert guarantees whole
32-element blocks, so the k/8 launch covers every row exactly. The
now-unreferenced scalar reorder kernel is removed.
Arc Pro B70, primed prefill (prefill after decode on a
reorder-installed tensor): +6.4% to +7.3% pp512/pp2048 on Q4_0
models (tested up to 27B dense); fresh-layout prefill and decode
unchanged; PPL within run noise.
* sycl : vectorize reordered q4_K dequantization stores
The reordered-layout Q4_K conversion kernel already loads its
quantized inputs through sycl::vec but writes the dequantized
results with scalar stores. Build the results in a vec<dst_t,n>
register and emit two vector stores instead. Same per-element
math; output is bit-identical.
Arc Pro B70, primed prefill (prefill after decode on a
reorder-installed tensor): +14.7% to +15.3% pp512/pp2048 on
Q4_K-dominant models (tested up to 27B dense); fresh-layout
prefill, decode and PPL unchanged.
* sycl : remap reordered q5_K dequantization to vec4 loads/stores
The reordered-layout Q5_K conversion kernel gave each thread only
four outputs fed by scalar ql/qh byte loads plus a local-memory
scale broadcast, so the stores were never its bottleneck: widening
the stores alone measured within noise on mixed-quant models and
+0.5% on a q5_K-dominant model. Re-decompose to 32 threads per
block, each consuming four consecutive ql and qh bytes and emitting
two aligned vec<dst_t,4> stores; a 4-element run shares one scale
pair, so the per-element math and the output are identical.
Arc Pro B70, primed prefill on a q5_K-dominant 27B model: +3.6%
pp512/pp2048 over the scalar kernel; fresh-layout prefill, decode
and PPL unchanged. The GET_ROWS reorder path moves to the same
32-thread decomposition.
* sycl : remap reordered q6_K dequantization to vec4 stores
The reordered-layout Q6_K conversion kernel wrote four scalars per
thread, strided 32 elements apart, each pair of neighbors sharing
neither instruction stream nor store transaction. Re-decompose the
same 64-thread work-group so each thread emits one aligned
vec<dst_t,4>: a 4-element run shares one 16-element scale group,
one ql nibble source and one qh bit pair, so the per-element math
and the output are identical to the scalar kernel.
Arc Pro B70, primed prefill: +28.7% pp512/pp2048 on a Q6_K 27B
model; fresh-layout prefill, decode and PPL unchanged.
* sycl : vectorize reordered q3_K dequantization stores
Each thread of the reordered-layout Q3_K conversion kernel already
computes four consecutive outputs; build them in a vec<dst_t,4>
register and emit one vector store instead of four scalars. Same
per-element math; output is bit-identical.
Arc Pro B70, primed prefill: +16.8% pp512/pp2048 on a Q3_K-dominant
27B model; fresh-layout prefill, decode and PPL unchanged.
* sycl : vectorize reordered iq3_xxs/iq3_s dequantization stores
Both reordered-layout IQ3 conversion kernels write eight contiguous
outputs per thread as scalar stores (four from each grid lookup).
Build two vec<dst_t,4> registers and emit two vector stores. Same
per-element math; output is bit-identical.
Arc Pro B70, primed prefill: +20.3% to +21.1% pp512/pp2048 on
IQ3_XXS- and IQ3_S-dominant 27B models; fresh-layout prefill,
decode and PPL unchanged.
* sycl : remap reordered q2_K dequantization to vec4 stores
Re-decompose the reordered-layout Q2_K conversion kernel the same
way as Q6_K: one aligned vec<dst_t,4> per thread instead of four
scalars strided 32 apart. A 4-element run shares one scale byte and
one 2-bit lane, so the per-element math and the output are
identical to the scalar kernel.
Arc Pro B70, primed prefill: +13.1% pp512/pp2048 on a Q2_K-dominant
27B model; fresh-layout prefill, decode and PPL unchanged.
* sycl : widen reordered mxfp4 dequantization stores
Give the reordered-layout MXFP4 conversion kernel the same flat
sycl::range<1> launch as Q4_0: four qs bytes (8 outputs) per
work-item inside one 32-element block and two vec<dst_t,4> stores.
Same LUT lookups and per-block scale; output is bit-identical. The
k % QK_MXFP4 == 0 assert guarantees whole blocks, so the k/8 launch
covers every row exactly. The old scalar kernel is removed.
The reordered layout for MXFP4 is installed on the MoE expert path
(reorder_qw_mxfp4_moe); dense MXFP4 tensors keep the normal layout.
Arc Pro B70, MXFP4 MoE 20B model: +57% pp2048 / +61% pp512 prefill
(primed and fresh alike), decode unchanged, PPL within the model's
run-to-run noise.
diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cpp
index 02b287ebc..1e7edf3b8 100644
--- a/ggml/src/ggml-sycl/convert.cpp
+++ b/ggml/src/ggml-sycl/convert.cpp
@@ -158,15 +158,12 @@ static void dequantize_row_q4_0_sycl_reorder(const void *vx, dst_t *y, const int
dpct::has_capability_or_fail(stream->get_device(),
{sycl::aspect::fp16});
+ // flat range<1>, four qs bytes (8 outputs) per item: two vectorized stores.
+ // k % QK4_0 == 0 gives whole 32-element blocks, so the k/8 launch covers every row exactly.
GGML_ASSERT(k % QK4_0 == 0);
- const int64_t n_groups = ceil_div(k / 2, SYCL_DEQUANTIZE_BLOCK_SIZE);
- // decrease global range when it exceeds the max int; the kernel loops to cover the rest
- const int64_t wg_size = downsample_sycl_global_range(n_groups, SYCL_DEQUANTIZE_BLOCK_SIZE);
- stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_groups) *
- sycl::range<3>(1, 1, wg_size),
- sycl::range<3>(1, 1, wg_size)),
- [=](sycl::nd_item<3> item_ct1) {
- dequantize_block_q4_0_reorder(vx, y, k, item_ct1);
+ stream->parallel_for(sycl::range<1>((size_t) (k / 8)),
+ [=](sycl::id<1> idx) {
+ dequantize_block_q4_0_reorder_wide(vx, y, k, idx);
});
}
@@ -253,7 +250,7 @@ static void dequantize_row_q5_K_sycl_reorder(const void * vx, dst_t * y, const i
sycl::local_accessor<uint8_t, 1> scale_local_acc(sycl::range<1>(K_SCALE_SIZE), cgh);
cgh.parallel_for(
- sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 64), sycl::range<3>(1, 1, 64)),
+ sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * sycl::range<3>(1, 1, 32), sycl::range<3>(1, 1, 32)),
[=](sycl::nd_item<3> item_ct1) {
dequantize_block_q5_K_reorder(vx, y + item_ct1.get_group(2) * QK_K, item_ct1.get_group(2), get_pointer(scale_local_acc), item_ct1, nb);
});
@@ -531,13 +528,13 @@ static void dequantize_row_mxfp4_sycl(const void * vx, dst_t * y, const int64_t
template <typename dst_t>
static void dequantize_row_mxfp4_sycl_reorder(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) {
+ // flat range<1>, four qs bytes (8 outputs) per item: two vectorized stores.
+ // Each 32-element block holds QK_MXFP4/2 = 16 qs bytes = 4 items, so the
+ // k/8 launch covers the row exactly (k % QK_MXFP4 == 0 asserted).
GGML_ASSERT(k % QK_MXFP4 == 0);
- const int n_warp = (k / QK_MXFP4 + WARP_SIZE - 1) / WARP_SIZE;
- stream->parallel_for(
- sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) * sycl::range<3>(1, 1, WARP_SIZE),
- sycl::range<3>(1, 1, WARP_SIZE)),
- [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
- dequantize_block_mxfp4_reorder(vx, y, k, item_ct1);
+ stream->parallel_for(sycl::range<1>((size_t) (k / 8)),
+ [=](sycl::id<1> idx) {
+ dequantize_block_mxfp4_reorder_wide(vx, y, k, idx);
});
}
diff --git a/ggml/src/ggml-sycl/dequantize.hpp b/ggml/src/ggml-sycl/dequantize.hpp
index ae709efa1..eb94771d6 100644
--- a/ggml/src/ggml-sycl/dequantize.hpp
+++ b/ggml/src/ggml-sycl/dequantize.hpp
@@ -817,25 +817,34 @@ static void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restri
}
}
+// four qs bytes per item, all inside one 32-element block: two vec<dst_t,4> stores.
+// float math, same products as the scalar kernel; nibble order kept (lo at +0, hi at +16).
template<typename dst_t>
-static void dequantize_block_q4_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k,
- const sycl::nd_item<3> &item_ct1) {
- // one thread per qs byte, like dequantize_block<dequantize_q4_0>; the reorder layout makes the byte reads contiguous
- // grid-stride loop because the launcher may shrink the work-group to keep the global range below INT_MAX
- // float math, not dequantize_q4_0_reorder: that helper computes in half with GGML_SYCL_F16, which changes f32 output
- const sycl::half * d_ptr = (const sycl::half *) ((const uint8_t *) vx + k/2);
- const int64_t stride = item_ct1.get_local_range(2) * item_ct1.get_group_range(2);
-
- for (int64_t iq = item_ct1.get_global_id(2); iq < k/2; iq += stride) {
- const int64_t ib = iq / (QK4_0/2);
- const int64_t iqs = iq % (QK4_0/2);
- const int vq = ((const uint8_t *) vx)[iq];
- const float d = float(d_ptr[ib]);
-
- dst_t * y = yy + ib*QK4_0 + iqs;
- y[0] = d * ((vq & 0xF) - 8);
- y[QK4_0/2] = d * ((vq >> 4) - 8);
- }
+static void dequantize_block_q4_0_reorder_wide(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k,
+ const sycl::id<1> idx) {
+ const int64_t i = (int64_t) idx[0];
+ const int64_t ib = i / 4;
+ const int64_t iqs = 4 * (i % 4);
+
+ const uint8_t * qs = (const uint8_t *) vx;
+ const sycl::half * d_ptr = (const sycl::half *) ((const uint8_t *) vx + k/2);
+
+ const float d = float(d_ptr[ib]);
+ const uint8_t * q = qs + ib*(QK4_0/2) + iqs;
+
+ sycl::vec<float, 4> lo, hi;
+ lo[0] = d * ((q[0] & 0xF) - 8);
+ lo[1] = d * ((q[1] & 0xF) - 8);
+ lo[2] = d * ((q[2] & 0xF) - 8);
+ lo[3] = d * ((q[3] & 0xF) - 8);
+ hi[0] = d * ((q[0] >> 4) - 8);
+ hi[1] = d * ((q[1] >> 4) - 8);
+ hi[2] = d * ((q[2] >> 4) - 8);
+ hi[3] = d * ((q[3] >> 4) - 8);
+
+ dst_t * yb = yy + ib*QK4_0 + iqs;
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb) = lo.convert<dst_t, sycl::rounding_mode::automatic>();
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(yb + QK4_0/2) = hi.convert<dst_t, sycl::rounding_mode::automatic>();
}
// Dequantize Q8_0 from reorder layout: [all qs (k bytes)][all d values]
@@ -963,19 +972,25 @@ static void dequantize_block_q2_K_reorder(const void * __restrict__ vx, dst_t *
const ggml_half2 * dm = reinterpret_cast<const ggml_half2 *>(base + dm_offset);
const int64_t tid = item_ct1.get_local_id(2);
- const int64_t n = tid / 32;
- const int64_t l = tid - 32 * n;
- const int64_t is = 8 * n + l / 16;
+ const int64_t n = tid / 32; // 128-element chunk: 0 or 1
+ const int64_t eh0 = 4 * (tid - 32 * n); // base element within the chunk
- const uint8_t q = qs[32 * n + l];
- dst_t * y = yy + 128 * n;
+ const int shift = 2 * (eh0 / 32); // 2-bit lane: 0/2/4/6
+ const uint8_t * q4 = qs + 32 * n + (eh0 & 31);
+ const uint8_t sc0 = scales[8 * n + eh0 / 16];
const float dall = (*dm)[0];
const float dmin = (*dm)[1];
- y[l+ 0] = dall * (scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (scales[is+0] >> 4);
- y[l+32] = dall * (scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (scales[is+2] >> 4);
- y[l+64] = dall * (scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (scales[is+4] >> 4);
- y[l+96] = dall * (scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (scales[is+6] >> 4);
+
+ // remapped decomposition: one aligned vec<dst_t,4> per thread instead of four
+ // scalars strided 32 apart; a 4-run shares one scale byte and one 2-bit lane,
+ // so the per-element math and output are identical to the scalar kernel.
+ sycl::vec<dst_t, 4> v;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ v[j] = dall * (sc0 & 0xF) * ((q4[j] >> shift) & 3) - dmin * (sc0 >> 4);
+ }
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(yy + 128 * n + eh0) = v;
#else
GGML_UNUSED(vx);
GGML_UNUSED(yy);
@@ -1085,9 +1100,12 @@ static void dequantize_block_q3_K_reorder(const void * __restrict__ vx, dst_t *
const uint8_t * q = qs + 32 * n;
const uint8_t * hm = hmask;
- for (int l = l0; l < l0 + 4; ++l) {
- y[l] = dl * ((int8_t) ((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4));
+ sycl::vec<dst_t, 4> v;
+#pragma unroll
+ for (int jj = 0; jj < 4; ++jj) {
+ v[jj] = dl * ((int8_t) ((q[l0 + jj] >> shift) & 3) - ((hm[l0 + jj] & m) ? 0 : 4));
}
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + l0) = v;
#else
GGML_UNUSED(vx);
GGML_UNUSED(yy);
@@ -1173,10 +1191,14 @@ inline void dequantize_q4_K_common(dst_t * __restrict__ y, const uint8_t * __res
const float m2 = dmin * m;
sycl::vec<uint8_t, n> q_vec = vec_aligned_load<uint8_t, n>(qs_ptr + 32 * il + n * ir);
+
+ sycl::vec<dst_t, n> lo, hi;
for (int l = 0; l < n; ++l) {
- y[l + 0] = d1 * (q_vec[l] & 0xF) - m1;
- y[l + 32] = d2 * (q_vec[l] >> 4) - m2;
+ lo[l] = d1 * (q_vec[l] & 0xF) - m1;
+ hi[l] = d2 * (q_vec[l] >> 4) - m2;
}
+ *reinterpret_cast<sycl::vec<dst_t, n> *>(y + 0) = lo;
+ *reinterpret_cast<sycl::vec<dst_t, n> *>(y + 32) = hi;
}
template <typename dst_t>
@@ -1260,13 +1282,15 @@ static void dequantize_block_q5_K_reorder(const void * __restrict__ vx, dst_t *
uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t n_blocks) {
#if QK_K == 256
- // assume 64 threads
+ // remapped decomposition: 32 threads, each emitting two aligned vec<dst_t,4>
+ // from four consecutive ql/qh bytes; a 4-element run shares one scale pair,
+ // so the per-element math and output are identical to the 64-thread kernel.
const int64_t tid = item_ct1.get_local_id(2);
- const int64_t il = tid / 16; // 0...3
- const int64_t ir = tid % 16; // 0...15
+ const int64_t il = tid / 8; // 0...3
+ const int64_t ir = tid % 8; // 0...7
const int64_t is = 2 * il;
- dst_t * y = yy + 64 * il + 2 * ir;
+ dst_t * y = yy + 64 * il + 4 * ir;
const uint8_t * base = static_cast<const uint8_t *>(vx);
@@ -1284,8 +1308,8 @@ static void dequantize_block_q5_K_reorder(const void * __restrict__ vx, dst_t *
const float dall = dm_values.x();
const float dmin = dm_values.y();
- const uint8_t * ql = qs_ptr + 32 * il + 2 * ir;
- const uint8_t * qh = qh_ptr + 2 * ir;
+ const uint8_t * ql = qs_ptr + 32 * il + 4 * ir; // four consecutive bytes
+ const uint8_t * qh = qh_ptr + 4 * ir; // four consecutive bytes
if (tid < K_SCALE_SIZE) {
scales_local[tid] = scales_ptr[tid];
@@ -1299,12 +1323,18 @@ static void dequantize_block_q5_K_reorder(const void * __restrict__ vx, dst_t *
get_scale_min_k4(is + 1, scales_local, sc, m);
const float d2 = dall * sc; const float m2 = dmin * m;
- uint8_t hm = 1 << (2 * il);
- y[ 0] = d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1;
- y[ 1] = d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1;
- hm <<= 1;
- y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2;
- y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2;
+ const uint8_t hm_lo = 1 << (2 * il);
+ const uint8_t hm_hi = 1 << (2 * il + 1);
+
+ sycl::vec<dst_t, 4> lo, hi;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ lo[j] = d1 * ((ql[j] & 0xF) + (qh[j] & hm_lo ? 16 : 0)) - m1;
+ hi[j] = d2 * ((ql[j] >> 4) + (qh[j] & hm_hi ? 16 : 0)) - m2;
+ }
+
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + 0) = lo;
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + 32) = hi;
#else
GGML_UNUSED(ib); GGML_UNUSED(tid); GGML_UNUSED(yy); GGML_UNUSED(scales_local); GGML_UNUSED(n_blocks);
GGML_ABORT("Q5_K reorder dequantize not supported for QK_K != 256");
@@ -1360,11 +1390,13 @@ static void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restri
template <typename dst_t>
static void dequantize_block_q6_K_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, const int64_t ib,
const sycl::nd_item<3> & item_ct1, int64_t n_blocks) {
-
+ // remapped decomposition: each of the 64 threads emits one aligned vec<dst_t,4>
+ // instead of four scalars strided 32 apart. The four elements of a run share one
+ // 16-element scale group, one ql nibble source and one qh bit pair, so the
+ // per-element math and the output are identical to the scalar kernel.
const int64_t tid = item_ct1.get_local_id(2);
- const int64_t ip = tid / 32; // ip is 0 or 1
- const int64_t il = tid - 32 * ip; // 0...32
- const int64_t is = 8 * ip + il / 16;
+ const int64_t ip = tid / 32; // 128-element half: 0 or 1
+ const int64_t eh0 = 4 * (tid - 32 * ip); // base element within the half
const uint8_t * base_ptr = static_cast<const uint8_t *>(vx);
const auto ql_offset = ib * (QK_K / 2);
@@ -1376,16 +1408,18 @@ static void dequantize_block_q6_K_reorder(const void * __restrict__ vx, dst_t *
const uint8_t * scales_ptr = base_ptr + base_scales_offset;
const ggml_half * d = (const ggml_half *) (base_ptr + base_d_offset) + ib;
- dst_t * y = yy + 128 * ip + il;
-
- const uint8_t * ql = ql_ptr + 64 * ip + il;
- const uint8_t qh = *(qh_ptr + 32 * ip + il);
- const int8_t * sc = reinterpret_cast<const int8_t *>(scales_ptr + is);
+ const int shift = eh0 >= 64 ? 4 : 0; // low or high ql nibble
+ const int qb = 2 * (eh0 / 32); // qh bit pair: 0/2/4/6
+ const int8_t sc0 = reinterpret_cast<const int8_t *>(scales_ptr + 8 * ip + eh0 / 16)[0];
+ const uint8_t * ql4 = ql_ptr + 64 * ip + (eh0 & 63);
+ const uint8_t * qh4 = qh_ptr + 32 * ip + (eh0 & 31);
- y[0] = *d * sc[0] * ((int8_t) ((ql[0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32);
- y[32] = *d * sc[2] * ((int8_t) ((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32);
- y[64] = *d * sc[4] * ((int8_t) ((ql[0] >> 4) | (((qh >> 4) & 3) << 4)) - 32);
- y[96] = *d * sc[6] * ((int8_t) ((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32);
+ sycl::vec<dst_t, 4> v;
+#pragma unroll
+ for (int j = 0; j < 4; ++j) {
+ v[j] = *d * sc0 * ((int8_t) (((ql4[j] >> shift) & 0xF) | ((((qh4[j] >> qb) & 3)) << 4)) - 32);
+ }
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(yy + 128 * ip + eh0) = v;
}
template<typename dst_t>
@@ -1521,11 +1555,14 @@ static void dequantize_block_iq3_xxs_reorder(const void * __restrict__ vx, dst_t
const float dl = (float) d[i] * (0.5f + (aux32 >> 28)) * 0.5f;
const uint8_t signs = ksigns_iq2xs[(aux32 >> (7 * il)) & 127];
dst_t * y = yy + 32 * ib + 8 * il;
+ sycl::vec<dst_t, 4> lo, hi;
#pragma unroll
for (int j = 0; j < 4; ++j) {
- y[j+0] = dl * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);
- y[j+4] = dl * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);
+ lo[j] = dl * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);
+ hi[j] = dl * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);
}
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + 0) = lo;
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + 4) = hi;
#else
GGML_UNUSED(vx);
GGML_UNUSED(yy);
@@ -1594,11 +1631,14 @@ dequantize_block_iq3_s_reorder(const void * __restrict__ vx, dst_t * __restrict_
const float dl = (float) d * (1 + 2 * ((scales[ib/2] >> (4 * (ib%2))) & 0xf));
const uint8_t signs = signs_base[4 * ib + il];
dst_t * y = yy + 32 * ib + 8 * il;
+ sycl::vec<dst_t, 4> lo, hi;
#pragma unroll
for (int j = 0; j < 4; ++j) {
- y[j+0] = dl * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);
- y[j+4] = dl * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);
+ lo[j] = dl * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);
+ hi[j] = dl * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);
}
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + 0) = lo;
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + 4) = hi;
#else
GGML_UNUSED(vx);
GGML_UNUSED(yy);
@@ -1737,24 +1777,28 @@ static void dequantize_block_mxfp4(const void * __restrict__ vx, dst_t * __restr
}
}
-// Reordered MXFP4 ([qs...][e...], see ggml_sycl_reordered::block_q_t<MXFP4>): one work-item per block.
-template <typename dst_t>
-static void dequantize_block_mxfp4_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k,
- const sycl::nd_item<3> & item_ct1) {
- const int64_t ib = (int64_t) item_ct1.get_group(2) * WARP_SIZE + item_ct1.get_local_id(2);
- if (ib >= k / QK_MXFP4) {
- return;
- }
+// Reordered MXFP4 ([qs...][e...], see ggml_sycl_reordered::block_q_t<MXFP4>).
+// four qs bytes per item, all inside one 32-element block: two vec<dst_t,4> stores.
+// same LUT lookups and per-block scale; bit-identical output.
+template<typename dst_t>
+static void dequantize_block_mxfp4_reorder_wide(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k,
+ const sycl::id<1> idx) {
+ const int64_t i = (int64_t) idx[0];
+ const int64_t ib = i / 4;
+ const int64_t iq = 4 * (i % 4); // first of four qs bytes in the block
const uint8_t * qs = (const uint8_t *) vx + ib * (QK_MXFP4 / 2);
const float d = ggml_sycl_e8m0_to_fp32(((const uint8_t *) vx)[k / 2 + ib]) * 0.5f;
- dst_t * y = yy + ib * QK_MXFP4;
+ dst_t * y = yy + ib * QK_MXFP4 + iq;
+ sycl::vec<dst_t, 4> lo, hi;
#pragma unroll
- for (int j = 0; j < QK_MXFP4 / 2; ++j) {
- y[j] = d * kvalues_mxfp4[qs[j] & 0xf];
- y[j + QK_MXFP4 / 2] = d * kvalues_mxfp4[qs[j] >> 4];
+ for (int j = 0; j < 4; ++j) {
+ lo[j] = d * kvalues_mxfp4[qs[iq + j] & 0xf];
+ hi[j] = d * kvalues_mxfp4[qs[iq + j] >> 4];
}
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(y) = lo;
+ *reinterpret_cast<sycl::vec<dst_t, 4> *>(y + QK_MXFP4 / 2) = hi;
}
diff --git a/ggml/src/ggml-sycl/getrows.cpp b/ggml/src/ggml-sycl/getrows.cpp
index 36f6fcd30..7b53e3dcc 100644
--- a/ggml/src/ggml-sycl/getrows.cpp
+++ b/ggml/src/ggml-sycl/getrows.cpp
@@ -657,7 +657,7 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
case GGML_TYPE_Q5_K:
if (src0_reordered) {
const void * vx = dst->src[0]->data;
- get_rows_sycl_reorder_k<3, 64, K_SCALE_SIZE>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(),
+ get_rows_sycl_reorder_k<3, 32, K_SCALE_SIZE>(dst->src[0], dst->src[1], dst, src1_i32, (float *)dst->data, ctx.stream(),
[=](float * y, int64_t ib, uint8_t * scales_local, const sycl::nd_item<3> & item_ct1, int64_t nb) {
dequantize_block_q5_K_reorder(vx, y, ib, scales_local, item_ct1, nb);
});