Commit 61d447234 for llama.cpp
commit 61d447234482cc0807cab652d763e0fdb5f5d816
Author: Jeff Bolz <jbolz@nvidia.com>
Date: Thu Nov 5 06:31:44 2026 -0600
vulkan: Implement test-fusion interface (#30769)
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-common.h b/ggml/src/ggml-vulkan/ggml-vulkan-common.h
index 0ac0f8b83..b5491fc4b 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-common.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-common.h
@@ -245,6 +245,12 @@ bool ggml_backend_vk_comm_allreduce_tensor(void * comm_ctx, ggml_tensor ** tenso
#endif
// debug
+void ggml_vk_count_fusion(const vk_device & device, const char * label);
+void * ggml_backend_vk_fusion_get(ggml_backend_dev_t dev);
+void ggml_backend_vk_fusion_stats_init(void * finfo);
+void ggml_backend_vk_fusion_stats_reset(void * finfo);
+int ggml_backend_vk_fusion_stats_get(void * finfo, const char ** labels, uint64_t * counts, int n);
+void ggml_backend_vk_fusion_set_enabled(void * finfo, bool enabled);
int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op);
#ifdef GGML_VULKAN_RUN_TESTS
template <typename X_TYPE, typename Y_TYPE>
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp
index 97747e866..5fa919511 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp
@@ -12,6 +12,58 @@ uint32_t vk_perf_logger_frequency = 1;
std::string vk_pipeline_stats_filter;
+void ggml_vk_count_fusion(const vk_device & device, const char * label) {
+ if (!device->fusion_stats_enabled.load(std::memory_order_relaxed)) {
+ return;
+ }
+ std::lock_guard<std::mutex> lock(device->fusion_mutex);
+ device->fusion_counts[label]++;
+}
+
+void * ggml_backend_vk_fusion_get(ggml_backend_dev_t dev) {
+ auto * ctx = (ggml_backend_vk_device_context *) dev->context;
+ return ggml_vk_get_device(ctx->device).get();
+}
+
+void ggml_backend_vk_fusion_stats_init(void * finfo) {
+ auto * device = (vk_device_struct *) finfo;
+ device->fusion_stats_enabled.store(true, std::memory_order_relaxed);
+}
+
+void ggml_backend_vk_fusion_stats_reset(void * finfo) {
+ auto * device = (vk_device_struct *) finfo;
+ std::lock_guard<std::mutex> lock(device->fusion_mutex);
+ // Keep labels and their order stable between fused and unfused runs.
+ for (auto & entry : device->fusion_counts) {
+ entry.second = 0;
+ }
+}
+
+int ggml_backend_vk_fusion_stats_get(void * finfo, const char ** labels, uint64_t * counts, int n) {
+ auto * device = (vk_device_struct *) finfo;
+ std::lock_guard<std::mutex> lock(device->fusion_mutex);
+ if (labels == nullptr) {
+ return (int) device->fusion_counts.size();
+ }
+ int i = 0;
+ for (const auto & entry : device->fusion_counts) {
+ if (i >= n) {
+ break;
+ }
+ labels[i] = entry.first.c_str();
+ if (counts) {
+ counts[i] = entry.second;
+ }
+ i++;
+ }
+ return i;
+}
+
+void ggml_backend_vk_fusion_set_enabled(void * finfo, bool enabled) {
+ auto * device = (vk_device_struct *) finfo;
+ device->disable_fusion = !enabled;
+}
+
void vk_memory_logger::log_allocation(vk_buffer_ref buf_ref, size_t size) {
if (!vk_memory_logger_enabled) {
return;
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
index f529a13ae..abc01ebe2 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
@@ -1018,6 +1018,9 @@ struct vk_device_struct {
ggml_backend_buffer_type buffer_type;
bool disable_fusion;
+ std::atomic<bool> fusion_stats_enabled {false};
+ std::mutex fusion_mutex;
+ std::map<std::string, uint64_t> fusion_counts;
bool disable_descriptor_reuse;
std::atomic<uint64_t> buffer_destroy_count {};
bool disable_host_visible_vidmem;
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index fc0ff83dd..1e203695b 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -4964,8 +4964,7 @@ vk_device ggml_vk_get_device(size_t idx) {
device->disable_descriptor_reuse = getenv("GGML_VK_DISABLE_DESCRIPTOR_REUSE") != nullptr;
- device->add_rms_fusion = !device->disable_fusion &&
- device->subgroup_arithmetic &&
+ device->add_rms_fusion = device->subgroup_arithmetic &&
device->vendor_id != VK_VENDOR_ID_INTEL;
device->partials_binding_alignment =
std::max(4u, (uint32_t)device->properties.limits.minStorageBufferOffsetAlignment);
@@ -12448,11 +12447,12 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in
cgraph->nodes[next_node_idx]->op == GGML_OP_RMS_NORM &&
cgraph->nodes[next_node_idx]->src[0] == cgraph->nodes[next_node_idx - 1] &&
ggml_nrows(cgraph->nodes[next_node_idx]) == 1 &&
- ctx->device->add_rms_fusion) {
+ ctx->device->add_rms_fusion && !ctx->device->disable_fusion) {
uint32_t size = ggml_vk_rms_partials_size(ctx, cgraph->nodes[node_idx]);
ctx->do_add_rms_partials_offset_calculation = true;
if (ctx->prealloc_size_add_rms_partials_offset + size <= ctx->prealloc_size_add_rms_partials) {
ctx->do_add_rms_partials = true;
+ ggml_vk_count_fusion(ctx->device, "ADD_RMS_NORM_PARTIALS");
}
}
}
@@ -14860,6 +14860,16 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
}
if (enqueued) {
+ if (fusion_string && ctx->device->fusion_stats_enabled.load(std::memory_order_relaxed)) {
+ std::string label = fusion_string;
+ if (cgraph->nodes[i]->op == GGML_OP_ADD) {
+ label += "_" + std::to_string(ctx->num_additional_fused_ops + 1);
+ }
+ if (ctx->fused_topk_moe_scale) {
+ label += "_SCALE";
+ }
+ ggml_vk_count_fusion(ctx->device, label.c_str());
+ }
++submitted_nodes;
#ifndef GGML_VULKAN_CHECK_RESULTS
@@ -16428,9 +16438,22 @@ static void * ggml_backend_vk_reg_get_proc_address(ggml_backend_reg_t reg, const
if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) {
return (void *) ggml_backend_vk_comm_allreduce_tensor;
}
-#else
- UNUSED(name);
#endif
+ if (strcmp(name, "ggml_backend_fusion_get") == 0) {
+ return (void *) ggml_backend_vk_fusion_get;
+ }
+ if (strcmp(name, "ggml_backend_fusion_stats_init") == 0) {
+ return (void *) ggml_backend_vk_fusion_stats_init;
+ }
+ if (strcmp(name, "ggml_backend_fusion_stats_reset") == 0) {
+ return (void *) ggml_backend_vk_fusion_stats_reset;
+ }
+ if (strcmp(name, "ggml_backend_fusion_stats_get") == 0) {
+ return (void *) ggml_backend_vk_fusion_stats_get;
+ }
+ if (strcmp(name, "ggml_backend_fusion_set_enabled") == 0) {
+ return (void *) ggml_backend_vk_fusion_set_enabled;
+ }
return nullptr;
}