Commit b9f3599fb for llama.cpp
commit b9f3599fb1c429e89248f042699ea2e062a32e57
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Tue Oct 13 17:28:15 2026 +0300
llama : do not map foreign ranges into a backend buffer (#30405)
* llama : do not map foreign ranges into a backend buffer
A buffer created from a host pointer pins everything that it wraps. When
several contexts share the same file (lazy tensors, MoE tensors kept on the
CPU with -cmoe), the single buffer that spans the first and the last tensor
of a context also wraps the ranges owned by the other contexts, pinning data
that the backend never reads.
For models with per-layer token embeddings this pins the whole lazy tensor
into the GPU buffer (27 GiB for Qwen3.8-Flash-Next IQ4_NL), defeating the
point of reading it lazily.
Instead, compute the file ranges of each context and subtract the ranges
owned by the other contexts, then create one buffer per range. Weights that
are mapped in more than one context (e.g. token_embd.weight) keep being part
of the range of every context that uses them.
Assisted-by: pi:llama.cpp/Qwen3.8-Flash-Next
* llama : enable prefetch on macOS
The prefetch hint of the tensors marked as prefetchable is a no-op on macOS,
so their pages are faulted in one by one while the graph runs. madvise
(MADV_WILLNEED) behaves as expected here, so add the platform to the guard.
With the lazy per-layer token embeddings of Qwen3.8-Flash-Next IQ4_NL read
from a cold file cache, prompt processing goes from 356 / 375 / 475 t/s to
735 / 749 / 788 t/s for pp512 / pp2048 / pp4096.
Assisted-by: pi:llama.cpp/Qwen3.8-Flash-Next
* llama : use a plain buffer list per context
Assisted-by: pi:llama.cpp/Qwen3.8-Flash-Next
diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
index 045489df7..80a719670 100644
--- a/src/llama-mmap.cpp
+++ b/src/llama-mmap.cpp
@@ -835,7 +835,7 @@ const bool llama_mlock::SUPPORTED = false;
#endif
void llama_prefetch(llama_memory_ranges mr) {
-#if defined(__linux__) || (defined(_WIN32) && _WIN32_WINNT >= 0x602)
+#if defined(__linux__) || defined(__APPLE__) || (defined(_WIN32) && _WIN32_WINNT >= 0x602)
if (mr.empty()) {
return;
}
@@ -874,6 +874,7 @@ void llama_prefetch(llama_memory_ranges mr) {
};
#else
auto prefetch = [&]() {
+ // TODO: in a follow-up, use posix_madvise(.., POSIX_MADV_WILLNEED) to match the rest of this file
if (madvise((void *) begin, end - begin, MADV_WILLNEED) != 0) {
LLAMA_LOG_WARN("llama_prefetch: madvise(MADV_WILLNEED) failed: %s\n", strerror(errno));
return false;
diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp
index 6cc0b2028..bc93c6f9e 100644
--- a/src/llama-model-loader.cpp
+++ b/src/llama-model-loader.cpp
@@ -13,6 +13,7 @@
#include <cstring>
#include <future>
#include <regex>
+#include <unordered_set>
static const size_t kiB = 1024;
static const size_t MiB = 1024*kiB;
@@ -1446,21 +1447,80 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps
}
}
-void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const {
- GGML_ASSERT(!mappings.empty());
- const auto & mapping = mappings.at(idx);
+using mapping_ranges = std::vector<llama_model_loader::mapping_range>;
+
+// merge a list of [first, last) ranges into a sorted, non-overlapping list
+static mapping_ranges merge_ranges(mapping_ranges ranges) {
+ std::sort(ranges.begin(), ranges.end(), [](const auto & a, const auto & b) { return a.first < b.first; });
+
+ mapping_ranges res;
+ for (const auto & range : ranges) {
+ if (!res.empty() && range.first <= res.back().last) {
+ res.back().last = std::max(res.back().last, range.last);
+ } else {
+ res.push_back(range);
+ }
+ }
+
+ return res;
+}
+
+// subtract the sorted, non-overlapping ranges 'sub' from 'a'
+static mapping_ranges sub_ranges(const mapping_ranges & a, const mapping_ranges & sub) {
+ mapping_ranges res;
+
+ for (const auto & x : a) {
+ size_t cur = x.first;
+ for (const auto & y : sub) {
+ if (y.last <= cur || y.first >= x.last) {
+ continue;
+ }
+ if (y.first > cur) {
+ res.push_back({ cur, y.first });
+ }
+ cur = std::max(cur, y.last);
+ }
+ if (cur < x.last) {
+ res.push_back({ cur, x.last });
+ }
+ }
+
+ return res;
+}
+
+std::vector<llama_model_loader::mapping_range> llama_model_loader::get_mapping_ranges(uint32_t idx, ggml_context * ctx) const {
+ GGML_ASSERT(idx < mappings.size());
+
+ // data of this context, and data of the other contexts that share the same file
+ // a weight that is mapped in several contexts (e.g. the token embedding) is not foreign data
+ std::unordered_set<const llama_tensor_weight *> own;
+ mapping_ranges own_ranges;
+ mapping_ranges other;
- *first = mapping->size();
- *last = 0;
- *addr = mapping->addr();
for (ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor; tensor = ggml_get_next_tensor(ctx, tensor)) {
const auto * weight = get_weight(ggml_get_name(tensor));
if (!weight || weight->idx != idx) {
continue;
}
- *first = std::min(*first, weight->offs);
- *last = std::max(*last, weight->offs + ggml_nbytes(tensor));
+ own.insert(weight);
+ own_ranges.push_back({ weight->offs, weight->offs + ggml_nbytes(tensor) });
+ }
+
+ for (const auto & it : ctx_map) {
+ if (!it.second || it.second.get() == ctx) {
+ continue;
+ }
+
+ for (ggml_tensor * tensor = ggml_get_first_tensor(it.second.get()); tensor; tensor = ggml_get_next_tensor(it.second.get(), tensor)) {
+ const auto * weight = get_weight(ggml_get_name(tensor));
+ if (!weight || weight->idx != idx || own.count(weight)) {
+ continue;
+ }
+ other.push_back({ weight->offs, weight->offs + ggml_nbytes(tensor) });
+ }
}
+
+ return sub_ranges(merge_ranges(std::move(own_ranges)), merge_ranges(std::move(other)));
}
void llama_model_loader::unmap_weight(const llama_tensor_weight & w) const {
@@ -1492,7 +1552,7 @@ const void * llama_model_loader::load_data_range(const llama_tensor_weight & w,
bool llama_model_loader::load_all_data(
struct ggml_context * ctx,
- llama_buf_map & bufs,
+ std::vector<ggml_backend_buffer_t> & bufs,
llama_mlocks * lmlocks,
llama_progress_callback progress_callback,
void * progress_callback_user_data) {
@@ -1528,7 +1588,7 @@ bool llama_model_loader::load_all_data(
}
// When not using mmaped io use async uploads from pinned memory to GPU memory.
// First determine if the backend supports the necessary features for async uploads.
- auto * buf = bufs.count(0) ? bufs.at(0) : nullptr;
+ auto * buf = bufs.empty() ? nullptr : bufs[0];
if (!buf) {
LLAMA_LOG_DEBUG("%s: no buffer found for async uploads\n", func);
return nullptr;
@@ -1599,7 +1659,7 @@ bool llama_model_loader::load_all_data(
if (upload_backend) {
LLAMA_LOG_DEBUG("%s: using async uploads for device %s, buffer type %s, backend %s\n", __func__,
ggml_backend_dev_name(ggml_backend_get_device(upload_backend)),
- ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs.at(0))),
+ ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs[0])),
ggml_backend_name(upload_backend));
}
@@ -1640,11 +1700,17 @@ bool llama_model_loader::load_all_data(
if (from_mapping) {
const auto & mapping = mappings.at(weight->idx);
+ uint8_t * data = (uint8_t *) mapping->addr() + weight->offs;
+
+ // the tensors of a file can be spread across several buffers - find the one containing this tensor's data
ggml_backend_buffer_t buf_mmap = nullptr;
- if (bufs.count(weight->idx)) {
- buf_mmap = bufs.at(weight->idx);
+ for (const auto & buf : bufs) {
+ const uint8_t * base = (const uint8_t *) ggml_backend_buffer_get_base(buf);
+ if (base && base <= data && data + n_size <= base + ggml_backend_buffer_get_size(buf)) {
+ buf_mmap = buf;
+ break;
+ }
}
- uint8_t * data = (uint8_t *) mapping->addr() + weight->offs;
if (check_tensors) {
validation_result.emplace_back(std::async(std::launch::async, [cur, data, n_size] {
diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h
index 9e51d0ce7..ed59079a4 100644
--- a/src/llama-model-loader.h
+++ b/src/llama-model-loader.h
@@ -16,8 +16,6 @@
#include <stdexcept>
#include <unordered_map>
-using llama_buf_map = std::unordered_map<uint32_t, ggml_backend_buffer_t>;
-
// lists of buffer types used for each layer
using buft_list_t = std::vector<std::pair<ggml_backend_dev_t, ggml_backend_buffer_type_t>>;
@@ -242,7 +240,16 @@ struct llama_model_loader {
void init_mappings(bool prefetch = true, llama_mlocks * mlock_mmaps = nullptr);
- void get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const;
+ // byte range within a mapped file
+ struct mapping_range {
+ size_t first; // inclusive
+ size_t last; // exclusive
+ };
+
+ // sorted, non-overlapping ranges of file 'idx' that back the tensors of 'ctx'
+ // ranges owned by other contexts are excluded, so that a buffer wrapping them won't pin
+ // data that this context never reads (lazy tensors, -cmoe tensors, ...)
+ std::vector<mapping_range> get_mapping_ranges(uint32_t idx, ggml_context * ctx) const;
// release a weight's mmap pages
void unmap_weight(const llama_tensor_weight & w) const;
@@ -254,7 +261,7 @@ struct llama_model_loader {
// Returns false if cancelled by progress_callback
bool load_all_data(
struct ggml_context * ctx,
- llama_buf_map & bufs,
+ std::vector<ggml_backend_buffer_t> & bufs,
llama_mlocks * lmlocks,
llama_progress_callback progress_callback,
void * progress_callback_user_data);
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 7072d029c..aeab7b75b 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -1875,12 +1875,13 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
pimpl->mappings.reserve(ml.mappings.size());
// create the backend buffers
- std::vector<std::pair<ggml_context *, llama_buf_map>> ctx_buf_maps;
- ctx_buf_maps.reserve(ml.ctx_map.size());
+ // note: the buffers are owned by pimpl->ctxs_bufs, this is a raw-pointer view used to load the data
+ std::vector<std::pair<ggml_context *, std::vector<ggml_backend_buffer_t>>> ctx_bufs;
+ ctx_bufs.reserve(ml.ctx_map.size());
- // Ensure we have enough capacity for the maximum backend buffer we will potentially create
- const size_t n_max_backend_buffer = ml.ctx_map.size() * ml.files.size();
- pimpl->ctxs_bufs.reserve(n_max_backend_buffer);
+ // reserve hint: each context creates one buffer per file, or more when its tensors are spread
+ // across disjoint ranges of a file (see get_mapping_ranges)
+ pimpl->ctxs_bufs.reserve(ml.ctx_map.size() * ml.files.size());
for (auto & [ctx_key, ctx_ptr] : ml.ctx_map) {
ggml_backend_buffer_type_t buft = ctx_key.buft;
@@ -1891,9 +1892,6 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
continue;
}
- llama_buf_map buf_map;
- buf_map.reserve(n_max_backend_buffer);
-
// check if it is possible to use buffer_from_host_ptr with this buffer type
ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft);
if (!dev) {
@@ -1915,24 +1913,24 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
if ((ml.use_mmap || is_lazy_mapped) && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft) {
GGML_ASSERT(!ml.no_alloc);
+ const size_t max_size = ggml_get_max_tensor_size(ctx);
+
for (uint32_t idx = 0; idx < ml.files.size(); idx++) {
- // only the mmap region containing the tensors in the model is mapped to the backend buffer
- // this is important for metal with apple silicon: if the entire model could be mapped to a metal buffer,
- // then we could just use metal for all layers
- // this allows using partial offloading when the model size exceeds the metal buffer size, but not the RAM size
- void * addr = nullptr;
- size_t first, last; // NOLINT
- ml.get_mapping_range(&first, &last, &addr, idx, ctx);
- if (first >= last) {
- continue;
- }
- const size_t max_size = ggml_get_max_tensor_size(ctx);
- ggml_backend_buffer_t buf = ggml_backend_dev_buffer_from_host_ptr(dev, (char *) addr + first, last - first, max_size);
- if (buf == nullptr) {
- throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft)));
+ // map the mmap regions that contain the tensors of this context
+ // this is important for metal with apple silicon: a backend buffer pins everything it
+ // wraps, so the regions owned by the other contexts (lazy tensors read on demand
+ // from disk, MoE tensors kept on the CPU, etc) are left out and stay out of RAM
+ // this also allows using partial offloading when the model size exceeds the metal buffer
+ // size, but not the RAM size
+ void * addr = ml.mappings.at(idx)->addr();
+
+ for (const auto & range : ml.get_mapping_ranges(idx, ctx)) {
+ ggml_backend_buffer_t buf = ggml_backend_dev_buffer_from_host_ptr(dev, (char *) addr + range.first, range.last - range.first, max_size);
+ if (buf == nullptr) {
+ throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft)));
+ }
+ bufs.emplace_back(buf);
}
- bufs.emplace_back(buf);
- buf_map.emplace(idx, buf);
}
} else {
ggml_backend_buffer_t buf;
@@ -1954,20 +1952,22 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
mlock_buf->grow_to(ggml_backend_buffer_get_size(buf));
}
bufs.emplace_back(buf);
- for (uint32_t idx = 0; idx < ml.files.size(); idx++) {
- buf_map.emplace(idx, buf);
- }
}
+ std::vector<ggml_backend_buffer_t> buf_ptrs;
+ buf_ptrs.reserve(bufs.size());
+
for (auto & buf : bufs) {
// indicate that this buffer contains weights
// this is used by ggml_backend_sched to improve op scheduling: ops that use a weight are preferably scheduled to the backend that contains the weight
ggml_backend_buffer_set_usage(buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
+
+ buf_ptrs.emplace_back(buf.get());
}
pimpl->ctxs_bufs.emplace_back(std::move(ctx_ptr), std::move(bufs));
- ctx_buf_maps.emplace_back(ctx, buf_map);
+ ctx_bufs.emplace_back(ctx, std::move(buf_ptrs));
}
if (llama_supports_gpu_offload()) {
@@ -2000,15 +2000,15 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
// without mmap, load non-host buffers first: their tensors go through a staging buffer, which is cheapest while the fewest weights are resident
if (!ml.use_mmap) {
- std::stable_partition(ctx_buf_maps.begin(), ctx_buf_maps.end(), [](const auto & ctx_buf_map) {
- const auto & buf_map = ctx_buf_map.second;
- return !buf_map.empty() && !ggml_backend_buffer_is_host(buf_map.begin()->second);
+ std::stable_partition(ctx_bufs.begin(), ctx_bufs.end(), [](const auto & x) {
+ const auto & bufs = x.second;
+ return !bufs.empty() && !ggml_backend_buffer_is_host(bufs[0]);
});
}
// load tensor data
- for (auto & [ctx, buf_map] : ctx_buf_maps) {
- if (!ml.load_all_data(ctx, buf_map, use_mlock ? &pimpl->mlock_mmaps : NULL, params.progress_callback, params.progress_callback_user_data)) {
+ for (auto & [ctx, bufs] : ctx_bufs) {
+ if (!ml.load_all_data(ctx, bufs, use_mlock ? &pimpl->mlock_mmaps : NULL, params.progress_callback, params.progress_callback_user_data)) {
return false;
}
}