Commit f851e244d for llama.cpp
commit f851e244d0dc56074ab701c4467edb116840cb88
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Tue Oct 13 15:24:34 2026 +0300
cuda : add GGML_CUDA_GRAPH_DEBUG and read env vars once (#30327)
* ggml-cuda: gate CUDA graph debug logs behind GGML_CUDA_GRAPH_DEBUG
* ggml-cuda: cache env var lookups in static const instead of re-reading per call
* ggml-cuda: read env vars exactly once via ggml_cuda_env_flag/ggml_cuda_env_int
* ggml-cuda: consolidate env flag reads into single static accessors
* ggml-cuda : drop env helper comments
Assisted-by: pi:llama.cpp/Qwen3.8-27B
* cont : remove NDEBUG gates for CUDA graph logs
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index f4537d447..9fe618f85 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -138,10 +138,54 @@ int ggml_cuda_get_device() {
return id;
}
+static bool ggml_cuda_env_flag(const char * name) {
+ const char * env = getenv(name);
+ return env != nullptr && std::atoi(env) != 0;
+}
+
+static int ggml_cuda_env_int(const char * name, int default_value) {
+ const char * env = getenv(name);
+ return env != nullptr ? std::atoi(env) : default_value;
+}
+
+static bool ggml_cuda_env_set(const char * name) {
+ return getenv(name) != nullptr;
+}
+
+static bool ggml_cuda_unified_memory_enabled() {
+ static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_ENABLE_UNIFIED_MEMORY");
+ return enabled;
+}
+
+static bool ggml_cuda_pinned_memory_disabled() {
+ static const bool disabled = ggml_cuda_env_flag("GGML_CUDA_NO_PINNED");
+ return disabled;
+}
+
+static bool ggml_cuda_register_host_enabled() {
+ static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_REGISTER_HOST");
+ return enabled;
+}
+
+static bool ggml_cuda_fusion_disabled() {
+ static const bool disabled = ggml_cuda_env_flag("GGML_CUDA_DISABLE_FUSION");
+ return disabled;
+}
+
+static bool ggml_cuda_p2p_enabled() {
+ static const bool enabled = ggml_cuda_env_set("GGML_CUDA_P2P");
+ return enabled;
+}
+
+static int ggml_cuda_op_offload_min_batch() {
+ static const int min_batch = ggml_cuda_env_int("GGML_OP_OFFLOAD_MIN_BATCH", 32);
+ return min_batch;
+}
+
static cudaError_t ggml_cuda_device_malloc(void ** ptr, size_t size, int device) {
ggml_cuda_set_device(device);
cudaError_t err;
- if (getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") != nullptr) {
+ if (ggml_cuda_unified_memory_enabled()) {
err = cudaMallocManaged(ptr, size);
#if defined(GGML_USE_HIP)
if (err == hipSuccess) {
@@ -383,7 +427,7 @@ static ggml_cuda_device_info ggml_cuda_init() {
// configure logging to stdout
// CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr));
- if (getenv("GGML_CUDA_P2P") != nullptr) {
+ if (ggml_cuda_p2p_enabled()) {
for (int id = 0; id < info.physical_device_count; ++id) {
CUDA_CHECK(cudaSetDevice(id));
for (int id_other = 0; id_other < info.physical_device_count; ++id_other) {
@@ -598,7 +642,7 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool {
CU_CHECK(cuMemRelease(handle));
// VMM Bug fix for P2P access if GGML_CUDA_P2P is set, or if NCCL build
- bool use_peer_access = getenv("GGML_CUDA_P2P") != nullptr;
+ bool use_peer_access = ggml_cuda_p2p_enabled();
#if defined(GGML_USE_NCCL)
use_peer_access = true;
#endif // defined(GGML_USE_NCCL)
@@ -1272,7 +1316,7 @@ static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buff
}
static void * ggml_cuda_host_malloc(size_t size) {
- if (getenv("GGML_CUDA_NO_PINNED") != nullptr) {
+ if (ggml_cuda_pinned_memory_disabled()) {
return nullptr;
}
@@ -2641,6 +2685,11 @@ static bool ggml_cuda_is_view_or_noop(const ggml_tensor * t) {
}
#ifdef USE_CUDA_GRAPH
+static bool ggml_cuda_graph_debug_enabled() {
+ static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_GRAPH_DEBUG");
+ return enabled;
+}
+
static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) {
bool use_cuda_graph = true;
@@ -2660,9 +2709,9 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) {
// the mul_mat_id fallback path synchronizes the stream, so we cannot use CUDA graphs
// ref: https://github.com/ggml-org/llama.cpp/pull/18958
use_cuda_graph = false;
-#ifndef NDEBUG
- GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__);
-#endif
+ if (ggml_cuda_graph_debug_enabled()) {
+ GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__);
+ }
}
}
@@ -2686,7 +2735,9 @@ static bool ggml_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctx
if (cgraph->uid != 0 &&
cgraph->uid == graph->uid) {
- GGML_LOG_DEBUG("CUDA Graph id %zu reused\n", cgraph->uid);
+ if (ggml_cuda_graph_debug_enabled()) {
+ GGML_LOG_DEBUG("CUDA Graph id %zu reused\n", cgraph->uid);
+ }
GGML_ASSERT((int)graph->node_props.size() == cgraph->n_nodes);
return false;
}
@@ -2733,9 +2784,9 @@ static void ggml_cuda_graph_update_executable(ggml_backend_cuda_context * cuda_c
#endif // CUDART_VERSION >= 12000
if (stat == cudaErrorGraphExecUpdateFailure) {
-#ifndef NDEBUG
- GGML_LOG_DEBUG("%s: CUDA graph update failed\n", __func__);
-#endif
+ if (ggml_cuda_graph_debug_enabled()) {
+ GGML_LOG_DEBUG("%s: CUDA graph update failed\n", __func__);
+ }
// The pre-existing graph exec cannot be updated due to violated constraints
// so instead clear error and re-instantiate
@@ -3605,8 +3656,7 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph,
// try and fuse nodes and return the number of nodes to skip
static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) {
- static bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION"));
- if (disable_fusion) {
+ if (ggml_cuda_fusion_disabled()) {
return 0;
}
@@ -4620,7 +4670,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co
if (graph->graph == nullptr) {
if (ggml_cuda_info().devices[cuda_ctx->device].cc < GGML_CUDA_CC_VOLTA) {
- if (!graph->disable_due_to_gpu_arch) {
+ if (!graph->disable_due_to_gpu_arch && ggml_cuda_graph_debug_enabled()) {
GGML_LOG_DEBUG("%s: disabling CUDA graphs due to GPU architecture\n", __func__);
}
graph->disable_due_to_gpu_arch = true;
@@ -4655,7 +4705,9 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,
// Warmup: need at least 2 calls with no property change on the 2nd call
if (!properties_changed) {
graph->warmup_complete = true;
- GGML_LOG_DEBUG("%s: CUDA graph warmup complete\n", __func__);
+ if (ggml_cuda_graph_debug_enabled()) {
+ GGML_LOG_DEBUG("%s: CUDA graph warmup complete\n", __func__);
+ }
use_cuda_graph = true;
cuda_graph_update_required = true;
}
@@ -4665,7 +4717,9 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,
if (properties_changed) {
// Properties changed - reset warmup, execute directly until stable again
graph->warmup_complete = false;
- GGML_LOG_DEBUG("%s: CUDA graph warmup reset\n", __func__);
+ if (ggml_cuda_graph_debug_enabled()) {
+ GGML_LOG_DEBUG("%s: CUDA graph warmup reset\n", __func__);
+ }
} else {
use_cuda_graph = true;
cuda_graph_update_required = graph->instance == nullptr;
@@ -4718,7 +4772,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev
static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) {
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
- static const bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION"));
+ const bool disable_fusion = ggml_cuda_fusion_disabled();
auto add_alloc_deps = [&](size_t start, size_t last_node) {
@@ -5138,7 +5192,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot
}
bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
- if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
+ if (!ggml_cuda_register_host_enabled()) {
return false;
}
@@ -5161,7 +5215,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
}
void ggml_backend_cuda_unregister_host_buffer(void * buffer) {
- if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
+ if (!ggml_cuda_register_host_enabled()) {
return;
}
@@ -5288,8 +5342,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *
CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device)));
// Check if UMA is explicitly enabled via environment variable
- bool uma_env = getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") != nullptr;
- bool is_uma = prop.integrated > 0 || uma_env;
+ bool is_uma = prop.integrated > 0 || ggml_cuda_unified_memory_enabled();
if (is_uma) {
// For UMA systems (like DGX Spark), use system memory info
@@ -5330,7 +5383,7 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back
props->device_id = ctx->pci_bus_id.empty() ? nullptr : ctx->pci_bus_id.c_str();
ggml_backend_cuda_device_get_memory(dev, &props->memory_free, &props->memory_total);
- bool host_buffer = getenv("GGML_CUDA_NO_PINNED") == nullptr;
+ const bool host_buffer = !ggml_cuda_pinned_memory_disabled();
#ifdef GGML_CUDA_NO_PEER_COPY
bool events = false;
#else
@@ -6026,7 +6079,7 @@ ggml_backend_reg_t ggml_backend_cuda_reg() {
std::lock_guard<std::mutex> lock(mutex);
if (!initialized) {
ggml_backend_cuda_reg_context * ctx = new ggml_backend_cuda_reg_context;
- const int min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32;
+ const int min_batch_size = ggml_cuda_op_offload_min_batch();
const ggml_cuda_device_info & info = ggml_cuda_info();
const bool virtual_devices = info.device_count > info.physical_device_count;