Commit b56f34ab1 for llama.cpp

commit b56f34ab130b35c038ffa0acc81cb998a8b5f87c
Author: ynankani <ynankani@nvidia.com>
Date:   Thu Oct 1 11:23:52 2026 +0000

    CUDA: Handle compute type for NVFP4 on cublass path (#29173)

    * CUDA: Handle compute type for NVFP4 on cublass path

    Signed-off-by: ynankani <ynankani@nvidia.com>

    * Use BF16 compute type for quantized models if HW allows

    Signed-off-by: ynankani <ynankani@nvidia.com>

    * Set acc prec to bf16 for nvfp4 as it needs atleast bf16 range

    Signed-off-by: ynankani <ynankani@nvidia.com>

    * Update ggml/src/ggml-cuda/ggml-cuda.cu

    Co-authored-by: Johannes Gäßler <johannesg@5d6.de>

    * preserve op_params for per-expert matmul

    Signed-off-by: ynankani <ynankani@nvidia.com>

    ---------

    Signed-off-by: ynankani <ynankani@nvidia.com>
    Co-authored-by: Johannes Gäßler <johannesg@5d6.de>

diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index ebbb5c106..f9fb46c2a 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -1616,6 +1616,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const

 static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
     const int cc = ggml_cuda_info().devices[ctx.device].cc;
+    const ggml_prec prec = (ggml_prec) ggml_get_op_params_i32(dst, 0);
     ggml_type compute_type = src0->type;
     if (ggml_is_quantized(compute_type)) {
         compute_type = fast_fp16_hardware_available(cc) ? GGML_TYPE_F16 : GGML_TYPE_F32;
@@ -1629,7 +1630,10 @@ static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml
             compute_type = GGML_TYPE_F32;
         }
     }
-    if (dst->op_params[0] == GGML_PREC_F32) {
+    // F16 is the only compute type that can not satisfy a request for BF16
+    if (prec == GGML_PREC_BF16 && compute_type == GGML_TYPE_F16) {
+        compute_type = fast_bf16_hardware_available(cc) ? GGML_TYPE_BF16 : GGML_TYPE_F32;
+    } else if (prec == GGML_PREC_F32) {
         compute_type = GGML_TYPE_F32;
     }

@@ -2033,6 +2037,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *

         ggml_tensor dst_slice;
         memset(&dst_slice, 0, sizeof(dst_slice));
+        memcpy(dst_slice.op_params, dst->op_params, sizeof(dst_slice.op_params));
         dst_slice.buffer = dst->buffer;
         dst_slice.type   = type_dst_sorted;
         dst_slice.ne[0]  = ne0;
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index 2df063f95..ae477c08d 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -1527,6 +1527,10 @@ ggml_tensor * llm_graph_context::build_lora_mm(
         prec_policy->apply(res);
     }

+    if (w->type == GGML_TYPE_NVFP4) {
+        ggml_prec_set_acc(res, GGML_PREC_BF16);
+    }
+
     if (w_s) {
         res = ggml_mul(ctx0, res, w_s);
     }
@@ -1563,6 +1567,10 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
         prec_policy->apply(res);
     }

+    if (w->type == GGML_TYPE_NVFP4) {
+        ggml_prec_set_acc(res, GGML_PREC_BF16);
+    }
+
     if (w_s) {
         const int64_t n_expert = w_s->ne[0];
         const int64_t n_tokens = cur->ne[2];