Commit 0b1281aee for llama.cpp
commit 0b1281aeef1838bf5848de7848a17e8d12747770
Author: Chanho Lee <ibear6954@gmail.com>
Date: Thu Oct 22 14:30:04 2026 +0900
webgpu : add Q2_0 support (#30914)
* webgpu : add Q2_0 row dequantization
* webgpu : add Q2_0 matrix multiplication
diff --git a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
index 6efff9b37..e01a41770 100644
--- a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
+++ b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp
@@ -1655,6 +1655,7 @@ class ggml_webgpu_shader_lib {
switch (key.src_type) {
case GGML_TYPE_Q1_0:
+ case GGML_TYPE_Q2_0:
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q5_0:
case GGML_TYPE_Q8_0:
@@ -1699,7 +1700,7 @@ class ggml_webgpu_shader_lib {
} else if ((key.src_type >= GGML_TYPE_Q4_0 && key.src_type <= GGML_TYPE_Q8_1) ||
key.src_type == GGML_TYPE_IQ4_NL || key.src_type == GGML_TYPE_MXFP4) {
defines.push_back("BLOCK_SIZE=32u");
- } else if (key.src_type == GGML_TYPE_NVFP4) {
+ } else if (key.src_type == GGML_TYPE_Q2_0 || key.src_type == GGML_TYPE_NVFP4) {
defines.push_back("BLOCK_SIZE=64u");
} else if (key.src_type >= GGML_TYPE_Q2_K) {
defines.push_back("BLOCK_SIZE=256u");
diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp
index 77fbb4f47..71d9ae2b9 100644
--- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp
+++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp
@@ -4395,6 +4395,7 @@ static bool ggml_backend_webgpu_device_supports_buft(ggml_backend_dev_t dev, ggm
static bool ggml_webgpu_supported_qtype(ggml_type type) {
switch (type) {
case GGML_TYPE_Q1_0:
+ case GGML_TYPE_Q2_0:
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
@@ -4500,6 +4501,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
case GGML_TYPE_F16:
case GGML_TYPE_BF16:
case GGML_TYPE_Q1_0:
+ case GGML_TYPE_Q2_0:
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl
index a3114a4be..4bf3c9347 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl
@@ -51,6 +51,20 @@ fn copy_elements(src_base: u32, dst_base: u32, offset: u32) {
}
#endif
+#ifdef Q2_0
+fn copy_elements(src_base: u32, dst_base: u32, offset: u32) {
+ let block_byte_base = (src_base + offset) * 18u;
+ let d = load_f16_as_f32_at_src(block_byte_base);
+ for (var j = 0u; j < 4u; j++) {
+ let q_packed = load_u32_at_src(block_byte_base + 2u + j * 4u);
+ for (var k = 0u; k < 16u; k++) {
+ let q = (q_packed >> (2u * k)) & 3u;
+ dst[dst_base + offset * 64u + j * 16u + k] = (f32(q) - 1.0) * d;
+ }
+ }
+}
+#endif
+
#ifdef Q4_0
fn copy_elements(src_base: u32, dst_base: u32, offset: u32) {
let block_byte_base = (src_base + offset) * 18; // Block stride: 18 bytes
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl
index 83f6fa172..1af6286ac 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl
@@ -111,6 +111,29 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
}
#endif // INIT_SRC0_SHMEM_Q1_0
+#ifdef INIT_SRC0_SHMEM_Q2_0
+const BLOCK_SIZE = 64u;
+const BLOCK_SIZE_BYTES = 18u;
+const NQ = 16u;
+
+fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u32) {
+ for (var i = thread_id * NQ; i < TILE_SRC0_SHMEM; i += TOTAL_WORKGROUP_SIZE * NQ) {
+ let global_m = offset_m + i / TILE_K;
+ let global_k = k_outer + i % TILE_K;
+ if (global_m < params.m && global_k < params.k) {
+ let block_idx = batch_offset + global_m * params.stride_01 + global_k / BLOCK_SIZE;
+ let block_byte_base = block_idx * BLOCK_SIZE_BYTES;
+ let d = load_f16_at_src0(block_byte_base);
+ let q_packed = load_u32_at_src0(block_byte_base + 2u + (global_k % BLOCK_SIZE) / 4u);
+ for (var k = 0u; k < NQ; k++) {
+ let q = (q_packed >> (2u * k)) & 3u;
+ shmem[i + k] = (f16(q) - 1.0) * d;
+ }
+ }
+ }
+}
+#endif // INIT_SRC0_SHMEM_Q2_0
+
// legacy-quants
#if defined(INIT_SRC0_SHMEM_Q4_0) || defined(INIT_SRC0_SHMEM_Q4_1) || defined(INIT_SRC0_SHMEM_Q5_0) || defined(INIT_SRC0_SHMEM_Q5_1) || defined(INIT_SRC0_SHMEM_Q8_0) || defined(INIT_SRC0_SHMEM_Q8_1) || defined(INIT_SRC0_SHMEM_MXFP4)
const BLOCK_SIZE = 32u;
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl
index 841777df9..2c116aad6 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl
@@ -119,6 +119,46 @@ fn accumulate_vec_dot(thread_id: u32, row_base: u32, src0_batch_offset: u32, src
}
#endif
+#ifdef MUL_ACC_Q2_0
+#define BLOCK_SIZE 64
+#define BLOCK_SIZE_BYTES 18
+#define THREADS_PER_BLOCK 4
+#define ELEMS_PER_THREAD (BLOCK_SIZE/THREADS_PER_BLOCK)
+fn accumulate_vec_dot(thread_id: u32, row_base: u32, src0_batch_offset: u32, src1_idx_base: u32) -> array<array<f32, OUTPUTS_PER_WG>, NUM_COLS> {
+ var acc: array<array<f32, OUTPUTS_PER_WG>, NUM_COLS>;
+
+ let num_blocks = params.k / BLOCK_SIZE;
+ let thread_within_block = thread_id % THREADS_PER_BLOCK;
+ for (var block = thread_id / THREADS_PER_BLOCK; block < num_blocks; block += WG_SIZE / THREADS_PER_BLOCK) {
+ let x_base = src1_idx_base + block * BLOCK_SIZE + thread_within_block * ELEMS_PER_THREAD;
+ var x_block: array<array<f32, ELEMS_PER_THREAD>, NUM_COLS>;
+ for (var col = 0u; col < NUM_COLS; col++) {
+ for (var i = 0u; i < ELEMS_PER_THREAD; i++) {
+ x_block[col][i] = f32(SRC1[x_base + col * params.stride_11 + i]);
+ }
+ }
+ for (var row = 0u; row < OUTPUTS_PER_WG; row++) {
+ let output_row = row_base + row;
+ if (output_row < params.m) {
+ let block_byte_base = (src0_batch_offset + output_row * params.stride_01 + block) * BLOCK_SIZE_BYTES;
+ let d = f32(load_f16_at_src0(block_byte_base));
+ let q_packed = load_u32_at_src0(block_byte_base + 2u + 4u * thread_within_block);
+ for (var col = 0u; col < NUM_COLS; col++) {
+ var row_sum = 0.0;
+ for (var i = 0u; i < ELEMS_PER_THREAD; i++) {
+ let q = (q_packed >> (2u * i)) & 3u;
+ row_sum += (f32(q) - 1.0) * x_block[col][i];
+ }
+ acc[col][row] += d * row_sum;
+ }
+ }
+ }
+ }
+
+ return acc;
+}
+#endif
+
#ifdef MUL_ACC_Q4_0
#define BLOCK_SIZE 32
#define BLOCK_SIZE_BYTES 18