Commit 61ad80bf9 for llama.cpp
commit 61ad80bf93de0ac03a29f83f5f3e9f596bf40c2d
Author: Pascal <admin@serveurperso.com>
Date: Thu Oct 15 15:25:19 2026 +0200
server: fix the off-by-two in the SWA reuse threshold (#30580)
The first evaluated token attends to the positions above
pos - n_swa, so a SWA state starting at pos - n_swa + 1 holds the
whole window, but the reuse threshold asked for two more cells.
A saved SWA state keeps only the cells inside its window, so a
state read back from a slot file or from the RAM prompt cache was
always rejected and the slot rolled back to a checkpoint.
The SWA threshold now matches the window, while a state without SWA
keeps the exact bound. The test harness gains override_kv, used to
give the SWA test model a window smaller than the test context.
[no release]
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index c20167b90..a18b7542c 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -3784,8 +3784,12 @@ private:
// ref: https://github.com/ggml-org/llama.cpp/pull/24110
const bool has_new_tokens = (n_past < slot.task->n_tokens());
- // the largest pos_min required for a checkpoint to be useful
- const auto pos_min_thold = std::max(0, pos_next - n_swa - (has_new_tokens ? 0 : 1));
+ // the first evaluated token attends to the positions > pos_eval - n_swa, so a SWA state
+ // starting at pos_eval - n_swa + 1 is enough, while a state without SWA must end before pos_eval
+ const llama_pos pos_eval = pos_next - (has_new_tokens ? 0 : 1);
+
+ // a state or a checkpoint is useful when its pos_min is below this threshold
+ const auto pos_min_thold = std::max(0, n_swa > 0 ? pos_eval - n_swa + 2 : pos_eval);
if (n_past > 0 && n_past <= slot.prompt.n_tokens()) {
const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id);
diff --git a/tools/server/tests/unit/test_slot_save.py b/tools/server/tests/unit/test_slot_save.py
index 793f1a0a7..488b63701 100644
--- a/tools/server/tests/unit/test_slot_save.py
+++ b/tools/server/tests/unit/test_slot_save.py
@@ -838,3 +838,89 @@ def test_slot_restore_checkpoints_draft_kv_type_change(swa_server, ctkd_restore)
})
assert res.status_code == 200
assert res.body["timings"]["prompt_n"] == n_live
+
+
+# a SWA window smaller than the prompt, so a saved state holds only the cells inside the window
+@pytest.fixture
+def swa_window_server(tmp_path):
+ swa = ServerPreset.tinygemma3()
+ swa.override_kv = "gemma3.attention.sliding_window=int:64"
+ swa.slot_save_path = str(tmp_path)
+ swa.temperature = 0.0
+ swa.n_predict = 8
+ return swa
+
+
+SWA_PROMPT = "The quick brown fox jumps over the lazy dog. " * 30
+
+
+def tokenize(server, text):
+ res = server.make_request("POST", "/tokenize", data={"content": text})
+ assert res.status_code == 200
+ return res.body["tokens"]
+
+
+# the first turn and the exact token continuation of it, so the next turn extends the saved state
+def swa_turns(server, id_slot):
+ res = server.make_request("POST", "/completion", data={
+ "prompt": tokenize(server, SWA_PROMPT),
+ "id_slot": id_slot,
+ "cache_prompt": True,
+ "return_tokens": True,
+ })
+ assert res.status_code == 200
+ return tokenize(server, SWA_PROMPT) + res.body["tokens"] + tokenize(server, " The end.")
+
+
+def swa_next(server, id_slot, prompt):
+ res = server.make_request("POST", "/completion", data={
+ "prompt": prompt,
+ "id_slot": id_slot,
+ "cache_prompt": True,
+ })
+ assert res.status_code == 200
+ return res.body["timings"]
+
+
+# a restored SWA state holds the window the next token needs, so it is reused as is
+def test_slot_restore_swa_window(swa_window_server):
+ server = swa_window_server
+ server.cache_ram = 0
+ server.start()
+
+ prompt = {id_slot: swa_turns(server, id_slot) for id_slot in [0, 1]}
+ assert prompt[0] == prompt[1]
+
+ res = server.make_request("POST", "/slots/0?action=save", data={"filename": "swa_slot0.bin"})
+ assert res.status_code == 200
+ res = server.make_request("POST", "/slots/0?action=erase")
+ assert res.status_code == 200
+ res = server.make_request("POST", "/slots/0?action=restore", data={"filename": "swa_slot0.bin"})
+ assert res.status_code == 200
+
+ timings = {id_slot: swa_next(server, id_slot, prompt[id_slot]) for id_slot in [0, 1]}
+
+ assert timings[0]["cache_n"] == timings[1]["cache_n"]
+ assert timings[0]["prompt_n"] == timings[1]["prompt_n"]
+
+
+# a SWA state loaded back from the RAM prompt cache is reused as is
+def test_prompt_cache_swa_window(swa_window_server):
+ server = swa_window_server
+ server.n_slots = 1
+
+ def run(other_prompt_first):
+ server.start()
+ prompt = swa_turns(server, 0)
+ if other_prompt_first:
+ res = server.make_request("POST", "/completion", data={"prompt": "Once upon a time " * 60, "cache_prompt": True})
+ assert res.status_code == 200
+ timings = swa_next(server, 0, prompt)
+ server.stop()
+ return timings
+
+ direct = run(False)
+ cached = run(True)
+
+ assert cached["cache_n"] == direct["cache_n"]
+ assert cached["prompt_n"] == direct["prompt_n"]
diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py
index b3207d387..1c84f5121 100644
--- a/tools/server/tests/utils.py
+++ b/tools/server/tests/utils.py
@@ -116,6 +116,7 @@ class ServerProcess:
media_path: str | None = None
sleep_idle_seconds: int | None = None
cache_ram: int | None = None
+ override_kv: str | None = None
no_cache_idle_slots: bool = False
log_path: str | None = None
ui_mcp_proxy: bool = False
@@ -286,6 +287,8 @@ class ServerProcess:
server_args.extend(["--sleep-idle-seconds", self.sleep_idle_seconds])
if self.cache_ram is not None:
server_args.extend(["--cache-ram", self.cache_ram])
+ if self.override_kv:
+ server_args.extend(["--override-kv", self.override_kv])
if self.no_cache_idle_slots:
server_args.append("--no-cache-idle-slots")
if self.ui_mcp_proxy: