Commit 2bc46baa9 for llama.cpp
commit 2bc46baa9207d098d03741ac394e7faaf1a955b6
Author: bri-prism <288398250+bri-prism@users.noreply.github.com>
Date: Thu Nov 5 02:03:44 2026 -0800
vulkan: FWHT kernels for block widths above 512 (#29772)
* vulkan: add F16 input to the FWHT
The Vulkan FWHT accepts F32 input only. This adds an F16 source variant of the
shader, for both the subgroup and the shared-memory paths, so the kernel reads
an F16 source directly instead of requiring a converted copy.
The F16 shader needs shader-float16, so the pipelines are only created when the
device reports it. A null pipeline makes ggml_vk_can_use_fwht fall back, which
is the same path an unsupported width already takes.
supports_op and the dispatch both already route through ggml_vk_can_use_fwht,
so there is a single place asking whether an op qualifies; nothing to
consolidate here the way the CUDA/Metal ports needed.
test-backend-ops on an M5 Pro (MoltenVK, fp16: 1): MUL_MAT 1128/1128,
including all 24 Hadamard cases (18 existing F32, 6 new F16).
Assisted-by: Codex
* vulkan: FWHT kernels for block widths above 512
The Vulkan FWHT covers block widths 64 to 512; wider blocks fall back to the
dense matmul. Extend the width list to 8192 and pick the shader per width: the
subgroup shader keeps n/subgroup_size values per invocation, so widths above
512 (or a subgroup narrow enough to exceed 64 values per invocation) use the
shared-memory shader with one row per workgroup and a 256-wide block. The rows
per workgroup become a specialization constant so the dispatch matches.
Pipelines for a width are only created when the device's shared-memory and
workgroup-invocation limits allow it; otherwise that width keeps the fallback.
This is the Vulkan counterpart to #29100 (CUDA). Stacked on #29101; the last
commit is the one to review.
Assisted-by: Codex
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
index d3b1a8dd2..f529a13ae 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
@@ -486,6 +486,15 @@ static constexpr uint32_t num_topk_moe_pipelines = 10;
static constexpr uint32_t num_topk_pipelines = 11;
+// FWHT block widths with a dedicated pipeline. The subgroup shader keeps n/subgroup_size
+// values per invocation, so widths above fwht_max_subgroup_n, or a subgroup narrow enough to
+// exceed fwht_max_subgroup_el_w values, use the shared-memory shader with one row per
+// workgroup instead.
+static constexpr uint32_t num_fwht_pipelines = 8;
+static constexpr uint32_t fwht_max_subgroup_n = 512;
+static constexpr uint32_t fwht_max_subgroup_el_w = 64;
+static constexpr uint32_t fwht_rows = 4;
+
static constexpr std::initializer_list<ggml_op> topk_moe_early_softmax_norm{ GGML_OP_SOFT_MAX, GGML_OP_RESHAPE, GGML_OP_ARGSORT,
GGML_OP_VIEW, GGML_OP_GET_ROWS, GGML_OP_RESHAPE,
GGML_OP_SUM_ROWS, GGML_OP_CLAMP, GGML_OP_DIV,
@@ -935,7 +944,9 @@ struct vk_device_struct {
vk_pipeline pipeline_sum_rows_f32;
vk_pipeline pipeline_cross_entropy_loss_f32, pipeline_cross_entropy_loss_f32_wg512;
vk_pipeline pipeline_cross_entropy_loss_back_f32, pipeline_cross_entropy_loss_back_f32_wg512;
- vk_pipeline pipeline_fwht_f32[4];
+ vk_pipeline pipeline_fwht_f32[num_fwht_pipelines];
+ vk_pipeline pipeline_fwht_f16[num_fwht_pipelines];
+ uint32_t fwht_rows_per_wg[num_fwht_pipelines] = {};
vk_pipeline pipeline_cumsum_f32;
vk_pipeline pipeline_cumsum_small_f32;
vk_pipeline pipeline_cumsum_multipass1_f32;
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index d18ea9ab2..175b14a3a 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -3592,21 +3592,31 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
// Intel Windows driver in range [32.0.101.8509, 32.0.101.8860) will crash when using fwht kernels so we gate that here
const bool can_use_fwht = device->driver_id != vk::DriverId::eIntelProprietaryWindows ||
!ggml_vk_intel_windows_driver_in_range(device->properties.driverVersion, 101, 8509, 101, 8860);
- if (can_use_fwht && device->subgroup_basic && device->subgroup_shuffle) {
+ if (can_use_fwht) {
+ const bool use_subgroup = device->subgroup_basic && device->subgroup_shuffle;
+ const uint32_t sg = std::max(device->subgroup_size, 1u);
int idx = 0;
- for (uint32_t n : {64, 128, 256, 512}) {
- if (device->subgroup_size <= n) {
- ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_f32", fwht_f32_len, fwht_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { device->subgroup_size, n }, 1, true, true, device->subgroup_size);
+ for (uint32_t n : {64, 128, 256, 512, 1024, 2048, 4096, 8192}) {
+ const bool wide = n > fwht_max_subgroup_n || n / sg > fwht_max_subgroup_el_w;
+ if (use_subgroup && !wide) {
+ if (device->subgroup_size <= n) {
+ ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_f32", fwht_f32_len, fwht_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { device->subgroup_size, n, fwht_rows }, 1, true, true, device->subgroup_size);
+ ggml_vk_create_pipeline(device, device->pipeline_fwht_f16[idx], "fwht_f16", fwht_f16_len, fwht_f16_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { device->subgroup_size, n, fwht_rows }, 1, true, true, device->subgroup_size);
+ device->fwht_rows_per_wg[idx] = fwht_rows;
+ }
+ } else {
+ // wide blocks trade rows per workgroup for a smaller register block
+ const uint32_t block_size = wide ? std::min(n, 256u) : std::min(sg, n);
+ const uint32_t rows = wide ? 1u : fwht_rows;
+ if ((uint64_t)rows * n * sizeof(float) <= device->properties.limits.maxComputeSharedMemorySize &&
+ block_size * rows <= device->properties.limits.maxComputeWorkGroupInvocations) {
+ ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_shmem_f32", fwht_shmem_f32_len, fwht_shmem_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { block_size, n, rows }, 1);
+ ggml_vk_create_pipeline(device, device->pipeline_fwht_f16[idx], "fwht_shmem_f16", fwht_shmem_f16_len, fwht_shmem_f16_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { block_size, n, rows }, 1);
+ device->fwht_rows_per_wg[idx] = rows;
+ }
}
++idx;
}
- } else if (can_use_fwht) {
- int idx = 0;
- for (uint32_t n : {64, 128, 256, 512}) {
- const uint32_t block_size = std::min(device->subgroup_size, n);
- ggml_vk_create_pipeline(device, device->pipeline_fwht_f32[idx], "fwht_shmem_f32", fwht_shmem_f32_len, fwht_shmem_f32_data, "main", 2, sizeof(vk_op_fwht_push_constants), {1, 1, 1}, { block_size, n }, 1);
- ++idx;
- }
}
const uint32_t cumsum_elem_per_thread = (device->vendor_id == VK_VENDOR_ID_AMD || device->vendor_id == VK_VENDOR_ID_INTEL) ? 2 : 4;
@@ -7211,6 +7221,10 @@ static int ggml_vk_fwht_pipeline_idx(int64_t n) {
case 128: return 1;
case 256: return 2;
case 512: return 3;
+ case 1024: return 4;
+ case 2048: return 5;
+ case 4096: return 6;
+ case 8192: return 7;
default: return -1;
}
}
@@ -7224,12 +7238,18 @@ bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml_tensor
return false;
}
- const int idx = ggml_vk_fwht_pipeline_idx(src1->ne[0]);
- if (idx < 0 || ctx->device->pipeline_fwht_f32[idx] == nullptr) {
+ if (dst->type != GGML_TYPE_F32) {
+ return false;
+ }
+
+ if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) {
return false;
}
- if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
+ const bool src_f16 = src1->type == GGML_TYPE_F16;
+
+ const int idx = ggml_vk_fwht_pipeline_idx(src1->ne[0]);
+ if (idx < 0 || (src_f16 ? ctx->device->pipeline_fwht_f16[idx] : ctx->device->pipeline_fwht_f32[idx]) == nullptr) {
return false;
}
@@ -7243,9 +7263,10 @@ bool ggml_vk_can_use_fwht(const ggml_backend_vk_context * ctx, const ggml_tensor
void ggml_vk_fwht(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src, ggml_tensor * dst) {
const int idx = ggml_vk_fwht_pipeline_idx(src->ne[0]);
- vk_pipeline pipeline = ctx->device->pipeline_fwht_f32[idx];
+ vk_pipeline pipeline = src->type == GGML_TYPE_F16 ? ctx->device->pipeline_fwht_f16[idx] : ctx->device->pipeline_fwht_f32[idx];
- const uint32_t rows_per_workgroup = 4;
+ const uint32_t rows_per_workgroup = ctx->device->fwht_rows_per_wg[idx];
+ GGML_ASSERT(rows_per_workgroup > 0);
const uint32_t n_rows = (uint32_t)ggml_nrows(src);
const uint32_t max_workgroups_x = ctx->device->properties.limits.maxComputeWorkGroupCount[0];
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp b/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp
index a2069964a..d279d4306 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp
@@ -5,11 +5,13 @@
#extension GL_KHR_shader_subgroup_basic : enable
#extension GL_KHR_shader_subgroup_shuffle : enable
#endif
+#extension GL_EXT_shader_16bit_storage : require
layout(constant_id = 0) const uint BLOCK_SIZE = 32;
layout(constant_id = 1) const uint N = 128;
+layout(constant_id = 2) const uint ROWS = 4;
-layout(local_size_x_id = 0, local_size_y = 4, local_size_z = 1) in;
+layout(local_size_x_id = 0, local_size_y_id = 2, local_size_z = 1) in;
layout(push_constant) uniform parameter
{
@@ -19,13 +21,13 @@ layout(push_constant) uniform parameter
float scale;
};
-layout(binding = 0, std430) readonly buffer A { float data_a[]; };
+layout(binding = 0, std430) readonly buffer A { A_TYPE data_a[]; };
layout(binding = 1, std430) writeonly buffer D { float data_d[]; };
const uint EL_W = N / BLOCK_SIZE;
#ifdef FWHT_SHMEM
-shared float shmem[4 * N];
+shared float shmem[ROWS * N];
#endif
void main() {
@@ -54,7 +56,7 @@ void main() {
[[unroll]]
for (uint i = 0; i < EL_W; ++i) {
- reg[i] = row < n_rows ? data_a[src_offset + row_offset + i * BLOCK_SIZE + tid] * scale : 0.0;
+ reg[i] = row < n_rows ? float(data_a[src_offset + row_offset + i * BLOCK_SIZE + tid]) * scale : 0.0;
}
#ifdef FWHT_SHMEM
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
index 3af09fb48..dba9d8ca8 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
@@ -1110,8 +1110,10 @@ void process_shaders() {
string_to_spv("sum_rows_f32", "sum_rows.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
string_to_spv("cross_entropy_loss_f32", "cross_entropy_loss.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}));
string_to_spv("cross_entropy_loss_back_f32", "cross_entropy_loss_back.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}));
- string_to_spv("fwht_f32", "fwht.comp", {});
- string_to_spv("fwht_shmem_f32", "fwht.comp", {{"FWHT_SHMEM", "1"}});
+ string_to_spv("fwht_f32", "fwht.comp", {{"A_TYPE", "float"}});
+ string_to_spv("fwht_shmem_f32", "fwht.comp", {{"A_TYPE", "float"}, {"FWHT_SHMEM", "1"}});
+ string_to_spv("fwht_f16", "fwht.comp", {{"A_TYPE", "float16_t"}});
+ string_to_spv("fwht_shmem_f16", "fwht.comp", {{"A_TYPE", "float16_t"}, {"FWHT_SHMEM", "1"}});
string_to_spv("count_equal_i32", "count_equal.comp", merge_maps(base_dict, {{"A_TYPE", "int"}, {"B_TYPE", "int"}, {"D_TYPE", "int"}}));
string_to_spv("dsv4_hc_comb_f32", "dsv4_hc_comb.comp", {});
string_to_spv("dsv4_hc_pre_f32", "dsv4_hc_pre.comp", {});