Commit 2c192444c for llama.cpp
commit 2c192444c968dab466b778724d3fd2ae6019482c
Author: Aman Gupta <amangupta052@gmail.com>
Date: Thu Nov 5 16:31:48 2026 +0800
fit : account for skipped NextN layers when setting n_gpu_layers (#31569)
* fit : account for skipped NextN layers when setting n_gpu_layers
Without MTP, fit counts n_layer + 1 layers while the model loader counts
n_layer_all + 1 for n_gpu_layers, including the NextN layers which hold no
weights. A fit result that offloads all layers then leaves layer 0 on the CPU
("offloaded 65/66 layers" for Qwen3.8-27B on two GPUs).
Add the skipped NextN layers to n_gpu_layers and to the tensor split of the
device that holds the output layer so the loader assigns the layers that
fit measured.
Fixes #31531
* fit : pass hp_ngl_skip by reference like the other hparams
* fit : add hp_ngl_skip to common_get_device_memory_data
diff --git a/common/fit.cpp b/common/fit.cpp
index 55b457e10..10d57f956 100644
--- a/common/fit.cpp
+++ b/common/fit.cpp
@@ -61,6 +61,7 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
const llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
uint32_t & hp_ngl,
+ uint32_t & hp_ngl_skip,
uint32_t & hp_n_ctx_train,
uint32_t & hp_n_expert,
ggml_log_level log_level) {
@@ -152,6 +153,8 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
if (mparams->load_mtp) {
hp_ngl += llama_model_n_layer_nextn(model);
}
+ // without MTP the NextN layers hold no weights but the model loader still counts them for n_gpu_layers
+ hp_ngl_skip = mparams->load_mtp ? 0 : llama_model_n_layer_nextn(model);
hp_n_ctx_train = llama_model_n_ctx_train(model);
hp_n_expert = llama_model_n_expert(model);
@@ -240,11 +243,12 @@ common_device_memory_data_vec common_get_device_memory_data(
const llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
uint32_t & hp_ngl,
+ uint32_t & hp_ngl_skip,
uint32_t & hp_n_ctx_train,
uint32_t & hp_n_expert,
ggml_log_level log_level) {
std::vector<llama_device_memory_data> impl = common_get_device_memory_data_impl(
- path_model, mparams, cparams, devs, hp_ngl, hp_n_ctx_train, hp_n_expert, log_level);
+ path_model, mparams, cparams, devs, hp_ngl, hp_ngl_skip, hp_n_ctx_train, hp_n_expert, log_level);
common_device_memory_data_vec ret(impl.size());
for (size_t i = 0; i < impl.size(); i++) {
@@ -270,6 +274,7 @@ static void common_params_fit_impl(
std::vector<ggml_backend_dev_t> devs;
uint32_t hp_ngl = 0; // hparams.n_gpu_layers
+ uint32_t hp_nsk = 0; // number of skipped NextN layers, counted by the model loader but not by hp_ngl
uint32_t hp_nct = 0; // hparams.n_ctx_train
uint32_t hp_nex = 0; // hparams.n_expert
@@ -291,6 +296,7 @@ static void common_params_fit_impl(
if (dmds_extra.empty() || n_ctx_extra != cparams->n_ctx) {
std::vector<ggml_backend_dev_t> devs_extra;
uint32_t ngl_extra = 0;
+ uint32_t nsk_extra = 0;
uint32_t nct_extra = 0;
uint32_t nex_extra = 0;
@@ -302,7 +308,7 @@ static void common_params_fit_impl(
dmds_t measured;
try {
measured = common_get_device_memory_data_impl(
- extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nct_extra, nex_extra, log_level);
+ extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nsk_extra, nct_extra, nex_extra, log_level);
} catch (const std::runtime_error & e) {
// the extra model is optional, fit the main model alone rather than giving up
LOG_WRN("%s: failed to measure the memory of the extra model, fitting without it: %s\n", __func__, e.what());
@@ -342,7 +348,7 @@ static void common_params_fit_impl(
// step 1: get data for default parameters and check whether any changes are necessary in the first place
LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__);
- dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
+ dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
// saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min:
const uint32_t n_ctx_max = (uint32_t) std::min<uint64_t>(uint64_t(hp_nct) * n_streams, UINT32_MAX);
@@ -354,7 +360,7 @@ static void common_params_fit_impl(
if (n_streams > 1) {
LOG_TRC("%s: context size unset and KV cache not unified -> using %" PRIu32 " for %" PRIu32 " sequences:\n",
__func__, n_ctx_max, n_streams);
- dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
+ dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
}
}
add_extra_memory(dmds_full);
@@ -493,7 +499,7 @@ static void common_params_fit_impl(
int64_t sum_projected_used_min_ctx = 0;
cparams->n_ctx = n_ctx_min_total;
- dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
+ dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
add_extra_memory(dmds_min_ctx);
if (nd == 0) {
sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total();
@@ -640,6 +646,20 @@ static void common_params_fit_impl(
assert(uint32_t(mparams.n_gpu_layers) <= hp_ngl + 1);
uint32_t il0 = hp_ngl + 1 - mparams.n_gpu_layers; // start index for tensor buft overrides
+ // the skipped NextN layers come right before the output layer, put them on the same device as the output layer
+ // so that the layers assigned by the model loader match the layers above
+ if (hp_nsk > 0 && mparams.n_gpu_layers > 0) {
+ mparams.n_gpu_layers += hp_nsk;
+ if (nd > 1) {
+ // the last devices have no layers if not even the output layer fits
+ size_t id_last = nd - 1;
+ while (ngl_per_device[id_last].n_layer == 0) {
+ id_last--;
+ }
+ tensor_split[id_last] += hp_nsk;
+ }
+ }
+
mparams.tensor_split = tensor_split;
size_t itbo = 0;
@@ -675,7 +695,7 @@ static void common_params_fit_impl(
set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy);
dmds_t dmd_nl = common_get_device_memory_data_impl(
- path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
+ path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
add_extra_memory(dmd_nl);
LOG_TRC("%s: memory for test allocation by device:\n", func_name);
@@ -704,7 +724,7 @@ static void common_params_fit_impl(
LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__);
dmds_t dmds_cpu_moe = common_get_device_memory_data_impl(
- path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
+ path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, log_level);
add_extra_memory(dmds_cpu_moe);
for (size_t id = 0; id < nd; id++) {
@@ -1185,10 +1205,11 @@ void common_fit_print(
llama_context_params * cparams) {
std::vector<ggml_backend_dev_t> devs;
uint32_t hp_ngl = 0; // hparams.n_gpu_layers
+ uint32_t hp_nsk = 0; // number of skipped NextN layers, counted by the model loader but not by hp_ngl
uint32_t hp_nct = 0; // hparams.n_ctx_train
uint32_t hp_nex = 0; // hparams.n_expert
- auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
+ auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nsk, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
GGML_ASSERT(dmd.size() == devs.size() + 1);
for (size_t id = 0; id < devs.size(); id++) {
diff --git a/common/fit.h b/common/fit.h
index 47478bba1..b2be5e4aa 100644
--- a/common/fit.h
+++ b/common/fit.h
@@ -104,6 +104,7 @@ common_device_memory_data_vec common_get_device_memory_data(
const llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
uint32_t & hp_ngl,
+ uint32_t & hp_ngl_skip,
uint32_t & hp_n_ctx_train,
uint32_t & hp_n_expert,
ggml_log_level log_level);