Commit 8b5436102 for llama.cpp
commit 8b543610254aafe121942eaf79a23c3847177270
Author: Aldehir Rojas <hello@alde.dev>
Date: Fri Oct 9 10:06:22 2026 -0500
chat : refactor API (#30210)
diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp
index 9e3b6d640..1112738af 100644
--- a/common/chat-auto-parser-generator.cpp
+++ b/common/chat-auto-parser-generator.cpp
@@ -61,8 +61,7 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
data.prompt += data.generation_prompt;
}
- auto parser = autoparser.build_parser(inputs, parser_generation_prompt);
- data.parser = parser.save();
+ data.parser = autoparser.build_parser(inputs, parser_generation_prompt);
// Build grammar if tools are present
bool has_tools =
@@ -78,7 +77,7 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &
if (include_grammar) {
data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
// Set grammar triggers based on tool section markers (fall back to per-call markers)
diff --git a/common/chat.cpp b/common/chat.cpp
index c8f156a70..87f1e99a2 100644
--- a/common/chat.cpp
+++ b/common/chat.cpp
@@ -9,6 +9,7 @@
#include "json.h"
#include "log.h"
#include "parsers/parsers.h"
+#include "sampling.h"
#include "jinja/value.h"
#include "jinja/runtime.h"
@@ -112,38 +113,6 @@ const char * common_chat_role_to_string(common_chat_role role) {
return "";
}
-json common_chat_msg_delimiters::to_json() const {
- json result = json::array();
- for (const auto & d : delimiters) {
- result.push_back({
- { "role", common_chat_role_to_string(d.role) },
- { "delimiter", d.delimiter },
- });
- }
- return result;
-}
-
-common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimiters) {
- common_chat_msg_delimiters result;
-
- if (!delimiters.is_array()) {
- return result;
- }
-
- result.delimiters.reserve(delimiters.size());
- for (const auto & d : delimiters) {
- if (!d.is_object()) {
- continue;
- }
- result.delimiters.push_back({
- common_chat_role_from_string(d.value("role", std::string())),
- d.value("delimiter", std::string()),
- });
- }
-
- return result;
-}
-
void common_chat_msg_delimiters::tokenize(const llama_vocab * vocab) {
for (auto & d : delimiters) {
d.tokens = common_tokenize(vocab, d.delimiter, false, true);
@@ -620,8 +589,11 @@ std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & too
}
common_chat_continuation common_chat_continuation_parse(const common_json & value) {
- if (value.is_boolean() && value.get<bool>()) {
- return COMMON_CHAT_CONTINUATION_AUTO;
+ if (value.is_null()) {
+ return COMMON_CHAT_CONTINUATION_NONE;
+ }
+ if (value.is_boolean()) {
+ return value.get<bool>() ? COMMON_CHAT_CONTINUATION_AUTO : COMMON_CHAT_CONTINUATION_NONE;
}
if (value.is_string()) {
auto value_str = value.get<std::string>();
@@ -632,7 +604,7 @@ common_chat_continuation common_chat_continuation_parse(const common_json & valu
return COMMON_CHAT_CONTINUATION_CONTENT;
}
}
- return COMMON_CHAT_CONTINUATION_NONE;
+ throw std::invalid_argument("Invalid continue_final_message: expected a boolean, \"content\" or \"reasoning_content\"");
}
bool common_chat_verify_template(const std::string & tmpl, bool use_jinja) {
@@ -1087,41 +1059,55 @@ static json common_chat_extra_context() {
return ctx;
}
-std::optional<common_chat_params> common_chat_try_specialized_template(
- const common_chat_template & tmpl,
- const std::string & src,
- autoparser::generation_params & params) {
+static common_chat_params common_chat_params_init_lfm2_tokens(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
+ return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ true);
+}
+
+static common_chat_params common_chat_params_init_lfm2_5(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
+ return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ false);
+}
+
+// Older gemma4 templates need their tool responses rewritten before rendering
+static common_chat_params common_chat_params_init_gemma4_legacy(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {
+ auto adjusted = inputs;
+ workaround::convert_tool_responses_gemma4(adjusted.messages);
+ return common_chat_params_init_gemma4(tmpl, adjusted);
+}
+
+// Pick the dedicated handler for a template from its source, or null for the autoparser.
+// Order matters: the first match wins, and later checks assume the earlier ones did not match.
+static common_chat_params_init_fn common_chat_template_detect_params_init(const std::string & src) {
// Ministral/Mistral Large 3 - uses special reasoning structure fixes, can't use autoparser
// Note: Mistral Small 3.2 uses [CALL_ID] which Ministral doesn't have, so we can distinguish them
if (src.find("[SYSTEM_PROMPT]") != std::string::npos && src.find("[TOOL_CALLS]") != std::string::npos &&
src.find("[ARGS]") != std::string::npos && src.find("[CALL_ID]") == std::string::npos) {
LOG_DBG("Using specialized template: Ministral/Magistral Large 3\n");
- return common_chat_params_init_ministral_3(tmpl, params);
+ return common_chat_params_init_ministral_3;
}
// LLM-jp-4.1 - GPT-OSS dialect (spaces after special tokens, <|end|>-separated parallel calls)
if (src.find("chat_format=llm-jp-harmony-v1") != std::string::npos) {
LOG_DBG("Using specialized template: LLM-jp Harmony v1\n");
- return common_chat_params_init_llm_jp_harmony(tmpl, params);
+ return common_chat_params_init_llm_jp_harmony;
}
// GPT-OSS - has unique channel-based structure that needs dedicated handler
if (src.find("<|channel|>") != std::string::npos) {
LOG_DBG("Using specialized template: GPT-OSS\n");
- return common_chat_params_init_gpt_oss(tmpl, params);
+ return common_chat_params_init_gpt_oss;
}
// Muse Glimmer format using " to=<recipient>" recipients and <|eom|>/<|eot|> message terminators.
if (src.find("<atem:function_calls>") != std::string::npos && src.find("<|eom|>") != std::string::npos) {
LOG_DBG("Using specialized template: Muse Glimmer\n");
- return common_chat_params_init_muse_glimmer(tmpl, params);
+ return common_chat_params_init_muse_glimmer;
}
// Functionary v3.2 - uses recipient-based format with >>>recipient\n{content}
// Detection: template has ">>>all" for content and ">>>" prefix for tool calls
if (src.find(">>>all") != std::string::npos && src.find(">>>${recipient}") != std::string::npos) {
LOG_DBG("Using specialized template: Functionary v3.2\n");
- return common_chat_params_init_functionary_v3_2(tmpl, params);
+ return common_chat_params_init_functionary_v3_2;
}
// Kimi K2 Thinking - uses unique tool call ID format: functions.<name>:<index>
@@ -1129,14 +1115,14 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
if (src.find("<|tool_calls_section_begin|>") != std::string::npos &&
src.find("<|tool_call_begin|>") != std::string::npos) {
LOG_DBG("Using specialized template: Kimi K2 Thinking\n");
- return common_chat_params_init_kimi_k2(tmpl, params);
+ return common_chat_params_init_kimi_k2;
}
// Kimi K3 - the <|open|>/<|close|>/<|end_of_msg|> markers are unique to it
if (src.find("<|open|>") != std::string::npos && src.find("<|close|>") != std::string::npos &&
src.find("<|end_of_msg|>") != std::string::npos) {
LOG_DBG("Using specialized template: Kimi K3\n");
- return common_chat_params_init_kimi_k3(tmpl, params);
+ return common_chat_params_init_kimi_k3;
}
// K2 Horizon - <|ifm|im_start|> turns, <ifm|think*> reasoning picked by reasoning_effort and
@@ -1144,7 +1130,7 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
if (src.find("<|ifm|im_start|>") != std::string::npos &&
src.find("<ifm|tool_calls>") != std::string::npos) {
LOG_DBG("Using specialized template: K2 Horizon\n");
- return common_chat_params_init_k2_horizon(tmpl, params);
+ return common_chat_params_init_k2_horizon;
}
// Ling 3.0 / Bailing V3 - <role>X</role> sections with <arg_key>/<arg_value> tagged
@@ -1152,7 +1138,7 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
if (src.find("<role>ASSISTANT</role>") != std::string::npos &&
src.find("<arg_key>") != std::string::npos) {
LOG_DBG("Using specialized template: Ling 3.0 (Bailing V3)\n");
- return common_chat_params_init_ling3(tmpl, params);
+ return common_chat_params_init_ling3;
}
// Cohere2 MoE / North Code - marker-wrapped format with <|START_TEXT|> content and
@@ -1161,19 +1147,19 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
if (src.find("<|START_TEXT|>") != std::string::npos &&
src.find("<|START_ACTION|>") != std::string::npos) {
LOG_DBG("Using specialized template: Cohere2 MoE\n");
- return common_chat_params_init_cohere2moe(tmpl, params);
+ return common_chat_params_init_cohere2moe;
}
if (is_lfm2_template(src)) {
LOG_DBG("Using specialized template: LFM2\n");
- return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ true);
+ return common_chat_params_init_lfm2_tokens;
}
// LFM2.5 format detection: template uses plain "List of tools: [...]" with no special tokens
if (src.find("List of tools: [") != std::string::npos &&
src.find("<|tool_list_start|>") == std::string::npos) {
LOG_DBG("Using specialized template: LFM2.5\n");
- return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ false);
+ return common_chat_params_init_lfm2_5;
}
// GigaChatV3 format detection
@@ -1181,7 +1167,7 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
src.find("<|message_sep|>") != std::string::npos &&
src.find("<|function_call|>") == std::string::npos) {
LOG_DBG("Using specialized template: GigaChatV3\n");
- return common_chat_params_init_gigachat_v3(tmpl, params);
+ return common_chat_params_init_gigachat_v3;
}
// MiniMax-M3: the namespace token "]<]minimax[>[" collides with the autoparser's
@@ -1190,7 +1176,7 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
src.find("<tool_call>") != std::string::npos &&
src.find("<invoke name=") != std::string::npos) {
LOG_DBG("Using specialized template: MiniMax-M3\n");
- return common_chat_params_init_minimax_m3(tmpl, params);
+ return common_chat_params_init_minimax_m3;
}
// DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls.
@@ -1201,18 +1187,18 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
(src.find("function_calls") != std::string::npos ||
src.find("tool_calls") != std::string::npos)) {
LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n");
- return common_chat_params_init_deepseek_v3_2(tmpl, params);
+ return common_chat_params_init_deepseek_v3_2;
}
// Gemma4 format detection
if (src.find("'<|tool_call>call:'") != std::string::npos) {
+ LOG_DBG("Using specialized template: Gemma4\n");
if (src.find("{#- OpenAI Chat Completions:") == std::string::npos) {
- // apply workarounds if using the older gemma4 templates
LOG_WRN("%s: detected an outdated gemma4 chat template, applying compatibility workarounds. "
"Consider updating to the official template.\n", __func__);
- workaround::convert_tool_responses_gemma4(params.messages);
+ return common_chat_params_init_gemma4_legacy;
}
- return common_chat_params_init_gemma4(tmpl, params);
+ return common_chat_params_init_gemma4;
}
// MiniCPM5 - XML tool calls with <function name="..."><param name="...">...</param></function>
@@ -1220,14 +1206,14 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
src.find("<function name=\"") != std::string::npos &&
src.find("<param name=\"") != std::string::npos) {
LOG_DBG("Using specialized template: MiniCPM5\n");
- return common_chat_params_init_minicpm5(tmpl, params);
+ return common_chat_params_init_minicpm5;
}
// TranslateGemma - user content must follow a custom schema with language codes
if (src.find("[source_lang_code]") != std::string::npos &&
src.find("[target_lang_code]") != std::string::npos) {
LOG_DBG("Using specialized template: TranslateGemma\n");
- return common_chat_params_init_translate_gemma(tmpl, params);
+ return common_chat_params_init_translate_gemma;
}
// Qwen3-Coder XML tool calls, also used by Nemotron Nano 3, Qwen3.5 and StepFun-3.5-Flash
@@ -1237,10 +1223,51 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
// Exclude models that don't use \n between tags
src.find("'<tool_call><function=' ~ tool_call.name ~ '>'") == std::string::npos) {
LOG_DBG("Using specialized template: Qwen3-Coder\n");
- return common_chat_params_init_qwen3_coder(tmpl, params);
+ return common_chat_params_init_qwen3_coder;
+ }
+
+ return nullptr;
+}
+
+common_chat_template::common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token) {
+ jinja::lexer lexer;
+ auto lexer_res = lexer.tokenize(src);
+ this->prog = jinja::parse_from_tokens(lexer_res);
+
+ this->src = lexer_res.source;
+ this->bos_tok = bos_token;
+ this->eos_tok = eos_token;
+
+ this->caps = jinja::caps_get(prog);
+ // LOG_INF("%s: caps:\n%s\n", __func__, this->caps.to_string().c_str());
+
+ this->params_init = common_chat_template_detect_params_init(this->src);
+ if (this->params_init) {
+ return;
+ }
+
+ // The analysis depends only on the template, so run it once here instead of on every apply.
+ // A failure is kept for apply to report, so a bad template still loads like it did before.
+ try {
+ analysis = std::make_unique<autoparser::autoparser>();
+ analysis->analyze_template(*this);
+ } catch (const std::exception & e) {
+ analysis.reset();
+ analysis_error = e.what();
}
+}
+
+common_chat_template::~common_chat_template() = default;
+common_chat_template::common_chat_template(common_chat_template &&) = default;
+common_chat_template & common_chat_template::operator=(common_chat_template &&) = default;
- return std::nullopt;
+std::optional<common_chat_params> common_chat_try_specialized_template(
+ const common_chat_template & tmpl,
+ const autoparser::generation_params & params) {
+ if (!tmpl.params_init) {
+ return std::nullopt;
+ }
+ return tmpl.params_init(tmpl, params);
}
static common_chat_params common_chat_templates_apply_jinja(const struct common_chat_templates * tmpls,
@@ -1342,21 +1369,23 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
data.prompt = common_chat_template_direct_apply_impl(tmpl, params_copy);
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, params);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
- auto parser = build_chat_peg_parser([&data](common_chat_peg_builder &p) {
+ data.parser = build_chat_peg_parser([&data](common_chat_peg_builder &p) {
return p.literal(data.generation_prompt) << p.content(p.rest());
});
- data.parser = parser.save();
return data;
}
- if (auto result = common_chat_try_specialized_template(tmpl, src, params)) {
+ if (auto result = common_chat_try_specialized_template(tmpl, params)) {
return *result;
}
+ if (!tmpl.analysis) {
+ throw std::invalid_argument("Unable to generate parser for this template. Automatic parser generation failed: " + tmpl.analysis_error);
+ }
+
try {
LOG_DBG("%s: using differential autoparser\n", __func__);
- struct autoparser::autoparser autoparser;
- autoparser.analyze_template(tmpl);
+ const auto & autoparser = *tmpl.analysis;
auto auto_params = autoparser::peg_generator::generate_parser(tmpl, params, autoparser);
common_chat_msg_delimiters delimiters;
@@ -1377,8 +1406,7 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
auto_params.thinking_end_tags = {std::move(end_tag)};
}
}
- common_peg_arena arena;
- arena.load(auto_params.parser);
+ const auto & arena = auto_params.parser;
LOG_DBG("%s: generated parser:\n%s\n\nparser generation prompt: %s\n", __func__, arena.dump(arena.root()).c_str(), auto_params.generation_prompt.c_str());
return auto_params;
} catch (const std::exception & e) {
@@ -1525,9 +1553,10 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
const common_chat_input & input,
bool is_partial,
const common_chat_parser_params & params) {
- const common_peg_arena & parser = src_parser.empty() ?
- build_chat_peg_parser([](common_chat_peg_builder & p) { return p.content(p.rest()) + p.end(); }) :
- src_parser;
+ // both branches must be lvalues, a temporary here would copy the arena on every call
+ static const common_peg_arena content_only =
+ build_chat_peg_parser([](common_chat_peg_builder & p) { return p.content(p.rest()) + p.end(); });
+ const common_peg_arena & parser = src_parser.empty() ? content_only : src_parser;
if (src_parser.empty()) {
LOG_DBG("No parser definition detected, assuming pure content parser.");
@@ -1598,6 +1627,84 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars
return msg;
}
+common_chat_session::common_chat_session(const common_chat_templates * tmpls,
+ const llama_vocab * vocab,
+ const common_chat_templates_inputs & inputs,
+ const common_chat_session_params & params) {
+ auto applied = common_chat_templates_apply(tmpls, inputs);
+
+ templated = true;
+ prompt_text = std::move(applied.prompt);
+ result.role = "assistant";
+
+ grammar_text = std::move(applied.grammar);
+ grammar_lazy = applied.grammar_lazy;
+ stops = std::move(applied.additional_stops);
+ generation_prompt_text = applied.generation_prompt;
+ thinking_start = std::move(applied.thinking_start_tag);
+ thinking_ends = std::move(applied.thinking_end_tags);
+
+ parser_params.format = applied.format;
+ parser_params.generation_prompt = vocab ? common_chat_input_tokenize(vocab, applied.generation_prompt)
+ : common_chat_input(applied.generation_prompt);
+ parser_params.debug = params.debug;
+ parser_params.parser = std::move(applied.parser);
+
+ delimiters = std::move(applied.message_delimiters);
+
+ if (vocab) {
+ common_params_sampling resolved;
+ resolved.grammar_lazy = applied.grammar_lazy;
+ common_sampling_add_preserved_tokens(resolved, vocab, applied.preserved_tokens);
+ common_sampling_add_grammar_triggers(resolved, vocab, std::move(applied.grammar_triggers));
+ preserved_tokens = std::move(resolved.preserved_tokens);
+ grammar_triggers = std::move(resolved.grammar_triggers);
+
+ delimiters.tokenize(vocab);
+ } else {
+ grammar_triggers = std::move(applied.grammar_triggers);
+ }
+
+ if (inputs.continue_final_message != COMMON_CHAT_CONTINUATION_NONE && !params.echo) {
+ // start from the prefill so it is not emitted as part of the first delta
+ result = common_chat_parse(input, true, parser_params);
+ }
+}
+
+void common_chat_session::apply_sampling(common_params_sampling & sampling) const {
+ if (!templated) {
+ return;
+ }
+ if (!grammar_text.empty()) {
+ sampling.grammar = {COMMON_GRAMMAR_TYPE_TOOL_CALLS, grammar_text};
+ }
+ sampling.grammar_lazy = grammar_lazy;
+ sampling.generation_prompt = generation_prompt_text;
+ sampling.preserved_tokens.insert(preserved_tokens.begin(), preserved_tokens.end());
+ sampling.grammar_triggers.insert(sampling.grammar_triggers.end(), grammar_triggers.begin(), grammar_triggers.end());
+}
+
+const common_chat_msg & common_chat_session::feed(const common_chat_input & chunk) {
+ GGML_ASSERT(!finished && "feed() after finish()");
+ input.append(chunk);
+ auto msg = common_chat_parse(input, true, parser_params);
+ if (!msg.empty()) {
+ result = std::move(msg);
+ }
+ return result;
+}
+
+const common_chat_msg & common_chat_session::finish(const common_chat_input & chunk) {
+ GGML_ASSERT(!finished && "finish() called twice");
+ finished = true;
+ input.append(chunk);
+ auto msg = common_chat_parse(input, false, parser_params);
+ if (!msg.empty()) {
+ result = std::move(msg);
+ }
+ return result;
+}
+
std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
GGML_ASSERT(chat_templates != nullptr);
GGML_ASSERT(chat_templates->template_default != nullptr);
diff --git a/common/chat.h b/common/chat.h
index 4509a04cc..775334924 100644
--- a/common/chat.h
+++ b/common/chat.h
@@ -22,8 +22,16 @@ struct common_chat_templates;
namespace autoparser {
struct generation_params;
+struct autoparser;
} // namespace autoparser
+struct common_chat_params;
+struct common_chat_template;
+
+// Builds the prompt and parser for a template that has a dedicated handler (see common/parsers)
+using common_chat_params_init_fn = common_chat_params (*)(const common_chat_template & tmpl,
+ const autoparser::generation_params & inputs);
+
struct common_chat_tool_call {
std::string name;
std::string arguments;
@@ -54,19 +62,20 @@ struct common_chat_template {
std::string eos_tok;
std::string src;
chat_template_caps caps;
+ // Dedicated handler picked once from the source, null when the differential autoparser is used
+ common_chat_params_init_fn params_init = nullptr;
- common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token) {
- jinja::lexer lexer;
- auto lexer_res = lexer.tokenize(src);
- this->prog = jinja::parse_from_tokens(lexer_res);
+ // Differential analysis, run once here when there is no dedicated handler. Null when there
+ // is one, or when the analysis failed, in which case analysis_error says why.
+ std::unique_ptr<autoparser::autoparser> analysis;
+ std::string analysis_error;
- this->src = lexer_res.source;
- this->bos_tok = bos_token;
- this->eos_tok = eos_token;
+ common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token);
- this->caps = jinja::caps_get(prog);
- // LOG_INF("%s: caps:\n%s\n", __func__, this->caps.to_string().c_str());
- }
+ // autoparser is incomplete here, so these are defined where it is complete
+ ~common_chat_template();
+ common_chat_template(common_chat_template &&);
+ common_chat_template & operator=(common_chat_template &&);
const std::string & source() const { return src; }
const std::string & bos_token() const { return bos_tok; }
@@ -209,8 +218,6 @@ struct common_chat_msg_delimiters {
// split tokens into message spans. skips maps a start index to a length of a region to jump over without matching
common_chat_msg_spans split(const llama_tokens & tokens, const std::map<size_t, size_t> & skips = {}) const;
-
- common_json to_json() const;
};
struct common_chat_tool {
@@ -278,7 +285,7 @@ struct common_chat_params {
std::vector<common_grammar_trigger> grammar_triggers;
std::vector<std::string> preserved_tokens;
std::vector<std::string> additional_stops;
- std::string parser;
+ common_peg_arena parser;
common_chat_msg_delimiters message_delimiters;
};
@@ -310,16 +317,10 @@ common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const st
// per-message parsing syntax
// should be derived from common_chat_params
struct common_chat_parser_params {
- common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
- common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool parse_reasoning"
- // Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode)
- bool reasoning_in_content = false;
- common_chat_input generation_prompt;
- bool parse_tool_calls = true;
- bool is_continuation = false;
- bool echo = false; // Include assistant prefilled msg in output
- bool debug = false; // Enable debug output for PEG parser
- common_peg_arena parser = {};
+ common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
+ common_chat_input generation_prompt;
+ bool debug = false; // Enable debug output for PEG parser
+ common_peg_arena parser = {};
common_chat_parser_params() = default;
common_chat_parser_params(const common_chat_params & chat_params) {
format = chat_params.format;
@@ -365,6 +366,60 @@ const char * common_chat_format_name(common_chat_format format);
common_chat_msg common_chat_parse(const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);
+struct common_chat_session_params {
+ bool echo = false; // include the assistant prefill in the output when continuing a message
+ bool debug = false; // enable debug output for the PEG parser
+};
+
+class common_chat_session {
+ public:
+ common_chat_session() { result.role = "assistant"; }
+
+ common_chat_session(const common_chat_templates * tmpls,
+ const llama_vocab * vocab,
+ const common_chat_templates_inputs & inputs,
+ const common_chat_session_params & params = {});
+
+ const std::string & prompt() const { return prompt_text; }
+ common_chat_format format() const { return parser_params.format; }
+ const common_chat_msg & msg() const { return result; }
+ const common_peg_arena & parser() const { return parser_params.parser; }
+
+ const std::string & grammar() const { return grammar_text; }
+ const std::string & generation_prompt() const { return generation_prompt_text; }
+ const std::string & thinking_start_tag() const { return thinking_start; }
+ const std::vector<std::string> & thinking_end_tags() const { return thinking_ends; }
+ const std::vector<std::string> & additional_stops() const { return stops; }
+
+ const common_chat_msg_delimiters & message_delimiters() const { return delimiters; }
+
+ void apply_sampling(common_params_sampling & sampling) const;
+
+ bool has_template() const { return templated; }
+
+ const common_chat_msg & feed(const common_chat_input & chunk);
+
+ const common_chat_msg & finish(const common_chat_input & chunk = {});
+
+ private:
+ std::string prompt_text;
+ std::string grammar_text;
+ bool grammar_lazy = false;
+ std::vector<common_grammar_trigger> grammar_triggers;
+ std::set<llama_token> preserved_tokens;
+ std::vector<std::string> stops;
+ std::string generation_prompt_text;
+ std::string thinking_start;
+ std::vector<std::string> thinking_ends;
+
+ common_chat_parser_params parser_params;
+ common_chat_msg_delimiters delimiters;
+ common_chat_input input;
+ common_chat_msg result;
+ bool templated = false;
+ bool finished = false;
+};
+
// used by arg and server
const char * common_reasoning_format_name(common_reasoning_format format);
common_reasoning_format common_reasoning_format_from_name(const std::string & format);
@@ -401,8 +456,7 @@ std::string common_chat_template_generation_prompt(
std::optional<common_chat_params> common_chat_try_specialized_template(
const common_chat_template & tmpl,
- const std::string & src,
- autoparser::generation_params & params);
+ const autoparser::generation_params & params);
// specialized per-task preset
@@ -412,5 +466,3 @@ struct common_chat_prompt_preset {
};
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);
-
-common_chat_msg_delimiters common_chat_msg_delimiters_parse(const common_json & delimiters);
diff --git a/common/parsers/cohere2moe.cpp b/common/parsers/cohere2moe.cpp
index 59595368d..dde2da929 100644
--- a/common/parsers/cohere2moe.cpp
+++ b/common/parsers/cohere2moe.cpp
@@ -77,7 +77,7 @@ common_chat_params common_chat_params_init_cohere2moe(const common_chat_template
data.prompt += data.generation_prompt;
}
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal(GEN_PREFIX);
auto end = p.end();
@@ -124,12 +124,10 @@ common_chat_params common_chat_params_init_cohere2moe(const common_chat_template
return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end;
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
diff --git a/common/parsers/deepseek.cpp b/common/parsers/deepseek.cpp
index 88122eaab..61b2b66c6 100644
--- a/common/parsers/deepseek.cpp
+++ b/common/parsers/deepseek.cpp
@@ -145,7 +145,7 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
bool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal(GEN_PROMPT);
auto end = p.end();
@@ -256,12 +256,10 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
generation_prompt + reasoning + content_before_tools + tool_calls + end;
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = has_tools && !require_tools;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
diff --git a/common/parsers/functionary-v3-2.cpp b/common/parsers/functionary-v3-2.cpp
index 56f7643f2..3a1eff3d5 100644
--- a/common/parsers/functionary-v3-2.cpp
+++ b/common/parsers/functionary-v3-2.cpp
@@ -21,7 +21,7 @@ common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_te
data.prompt += data.generation_prompt;
}
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
// Functionary v3.2 format:
// - Normal content: >>>all\n{content}
// - Tool calls: >>>function_name\n{json_args}
@@ -76,13 +76,11 @@ common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_te
return generation_prompt + ret;
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
// Grammar trigger for when the model starts outputting a tool call
diff --git a/common/parsers/gemma4.cpp b/common/parsers/gemma4.cpp
index 2223089da..5ba264889 100644
--- a/common/parsers/gemma4.cpp
+++ b/common/parsers/gemma4.cpp
@@ -198,7 +198,7 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto start = p.rule("start", p.optional(p.literal("<|turn>model\n")));
if (extract_reasoning) {
@@ -290,12 +290,10 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &
return start + p.one_or_more(message);
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
diff --git a/common/parsers/gigachat-v3.cpp b/common/parsers/gigachat-v3.cpp
index 949942547..0d20f83f7 100644
--- a/common/parsers/gigachat-v3.cpp
+++ b/common/parsers/gigachat-v3.cpp
@@ -25,7 +25,7 @@ common_chat_params common_chat_params_init_gigachat_v3(
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
const auto *tool_call_start_prefix = "<|message_sep|>\n\nfunction call<|role_sep|>\n";
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto ret = p.eps();
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
// Build a choice of all available tools
@@ -60,13 +60,11 @@ common_chat_params common_chat_params_init_gigachat_v3(
return p.literal("assistant<|role_sep|>\n") + ret;
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
diff --git a/common/parsers/gpt-oss.cpp b/common/parsers/gpt-oss.cpp
index ed50524a9..b91c61d2a 100644
--- a/common/parsers/gpt-oss.cpp
+++ b/common/parsers/gpt-oss.cpp
@@ -45,8 +45,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
data.thinking_start_tag = "<|channel|>analysis<|message|>";
data.thinking_end_tags = {"<|end|>"};
- // These special tokens are required to parse properly, so we include them
- // even if parse_tool_calls is false.
+ // These special tokens are required to parse properly
data.preserved_tokens = {
"<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",
};
@@ -68,7 +67,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto start = p.rule("start", p.literal("<|start|>assistant"));
auto end = p.rule("end", p.literal("<|end|>"));
auto content = p.rule("message-content", p.until("<|end|>"));
@@ -138,12 +137,10 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
return p.zero_or_more(start + any) + start + (final_msg | unsolicited);
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
diff --git a/common/parsers/k2-horizon.cpp b/common/parsers/k2-horizon.cpp
index bd64f6890..b73734fbd 100644
--- a/common/parsers/k2-horizon.cpp
+++ b/common/parsers/k2-horizon.cpp
@@ -77,7 +77,7 @@ common_chat_params common_chat_params_init_k2_horizon(const common_chat_template
data.prompt += data.generation_prompt;
}
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal(GEN_PREFIX);
auto think_end = p.choice();
@@ -174,12 +174,10 @@ common_chat_params common_chat_params_init_k2_horizon(const common_chat_template
return generation_prompt + (reasoning << content << tool_calls);
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = !(has_response_format || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED);
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
if (data.grammar_lazy) {
diff --git a/common/parsers/kimi-k2.cpp b/common/parsers/kimi-k2.cpp
index 9992968a5..5e9d49ef9 100644
--- a/common/parsers/kimi-k2.cpp
+++ b/common/parsers/kimi-k2.cpp
@@ -48,7 +48,7 @@ common_chat_params common_chat_params_init_kimi_k2(const common_chat_template &
data.prompt += data.generation_prompt;
}
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
// Kimi K2 Thinking format:
// - Reasoning: <think>{reasoning}</think>
// - Content: text after reasoning
@@ -111,12 +111,10 @@ common_chat_params common_chat_params_init_kimi_k2(const common_chat_template &
return generation_prompt + reasoning + content_before_tools + tool_calls + end;
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
diff --git a/common/parsers/kimi-k3.cpp b/common/parsers/kimi-k3.cpp
index cf153f6b5..cee1c1671 100644
--- a/common/parsers/kimi-k3.cpp
+++ b/common/parsers/kimi-k3.cpp
@@ -66,7 +66,7 @@ common_chat_params common_chat_params_init_kimi_k3(const common_chat_template &
data.prompt += data.generation_prompt;
}
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto end = p.end();
auto start = p.optional(p.literal(MSG_START));
@@ -151,12 +151,10 @@ common_chat_params common_chat_params_init_kimi_k3(const common_chat_template &
return start + reasoning + response + tools + trailer + end;
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
diff --git a/common/parsers/lfm2.cpp b/common/parsers/lfm2.cpp
index 280788509..865929860 100644
--- a/common/parsers/lfm2.cpp
+++ b/common/parsers/lfm2.cpp
@@ -64,7 +64,7 @@ common_chat_params common_chat_params_init_lfm2(const common_chat_template &
data.prompt += data.generation_prompt;
}
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal(GEN_PROMPT);
auto end = p.end();
@@ -93,12 +93,10 @@ common_chat_params common_chat_params_init_lfm2(const common_chat_template &
return generation_prompt + reasoning + content + tool_calls + end;
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
diff --git a/common/parsers/ling3.cpp b/common/parsers/ling3.cpp
index 67d44d47a..5d8deb9af 100644
--- a/common/parsers/ling3.cpp
+++ b/common/parsers/ling3.cpp
@@ -80,7 +80,7 @@ common_chat_params common_chat_params_init_ling3(const common_chat_template &
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto end = p.end();
// the effective parse input is generation_prompt + model output, so the
@@ -185,12 +185,10 @@ common_chat_params common_chat_params_init_ling3(const common_chat_template &
return opener + reasoning + content + tools + tail + end;
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = !has_response_format && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
diff --git a/common/parsers/llm-jp-harmony.cpp b/common/parsers/llm-jp-harmony.cpp
index cafb82c12..aebceb126 100644
--- a/common/parsers/llm-jp-harmony.cpp
+++ b/common/parsers/llm-jp-harmony.cpp
@@ -48,8 +48,7 @@ common_chat_params common_chat_params_init_llm_jp_harmony(const common_chat_temp
data.thinking_start_tag = "<|channel|>analysis<|message|>";
data.thinking_end_tags = {"<|end|>"};
- // These special tokens are required to parse properly, so we include them
- // even if parse_tool_calls is false.
+ // These special tokens are required to parse properly
data.preserved_tokens = {
"<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",
};
@@ -71,7 +70,7 @@ common_chat_params common_chat_params_init_llm_jp_harmony(const common_chat_temp
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
// tokenizer space after special tokens; not p.space() since GBNF `space` allows one space only
auto sp = p.chars("[ ]", 0, -1);
auto channel_tag = p.literal("<|channel|>") + sp;
@@ -144,12 +143,10 @@ common_chat_params common_chat_params_init_llm_jp_harmony(const common_chat_temp
return p.zero_or_more(start + any) + start + final_msg;
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
diff --git a/common/parsers/minicpm5.cpp b/common/parsers/minicpm5.cpp
index 06f4cc8c8..45fad952b 100644
--- a/common/parsers/minicpm5.cpp
+++ b/common/parsers/minicpm5.cpp
@@ -46,7 +46,7 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template &
data.prompt += data.generation_prompt;
}
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal("<|im_start|>assistant\n");
auto reasoning = p.eps();
@@ -113,12 +113,10 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template &
return generation_prompt + reasoning + p.content(p.rest()) + p.end();
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
diff --git a/common/parsers/minimax-m3.cpp b/common/parsers/minimax-m3.cpp
index c38ab3e23..65a023959 100644
--- a/common/parsers/minimax-m3.cpp
+++ b/common/parsers/minimax-m3.cpp
@@ -56,7 +56,7 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
data.prompt += data.generation_prompt;
}
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START);
auto end = p.end();
@@ -213,12 +213,10 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template
return generation_prompt + reasoning + content_before_tools + tool_calls + end;
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
diff --git a/common/parsers/ministral3.cpp b/common/parsers/ministral3.cpp
index 9cc5248b5..83089d711 100644
--- a/common/parsers/ministral3.cpp
+++ b/common/parsers/ministral3.cpp
@@ -72,7 +72,7 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templat
data.prompt += data.generation_prompt;
}
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.eps();
auto reasoning =
extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps();
@@ -108,13 +108,11 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templat
return generation_prompt + (reasoning << p.content(p.rest()));
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
diff --git a/common/parsers/muse-glimmer.cpp b/common/parsers/muse-glimmer.cpp
index 6c7303701..f75af1cee 100644
--- a/common/parsers/muse-glimmer.cpp
+++ b/common/parsers/muse-glimmer.cpp
@@ -48,7 +48,7 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa
// Constrained grammar whenever tools are offered or a response format is requested.
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto start = p.rule("start", p.literal("<|start|>assistant"));
if (!extract_reasoning && !include_grammar) {
@@ -131,12 +131,10 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa
return p.zero_or_more(start + analysis) + start + final_msg;
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN,
diff --git a/common/parsers/qwen3-coder.cpp b/common/parsers/qwen3-coder.cpp
index 323b3da31..006ec6f98 100644
--- a/common/parsers/qwen3-coder.cpp
+++ b/common/parsers/qwen3-coder.cpp
@@ -71,7 +71,7 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
});
}
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal(GEN_PREFIX);
auto reasoning = p.eps();
@@ -174,13 +174,11 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat
return generation_prompt + (reasoning << p.content(p.rest()));
});
- data.parser = parser.save();
-
if (include_grammar) {
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
- parser.build_grammar(builder, data.grammar_lazy);
+ data.parser.build_grammar(builder, data.grammar_lazy);
});
if (data.grammar_lazy) {
diff --git a/common/parsers/translate-gemma.cpp b/common/parsers/translate-gemma.cpp
index c52778064..45b740e90 100644
--- a/common/parsers/translate-gemma.cpp
+++ b/common/parsers/translate-gemma.cpp
@@ -54,10 +54,9 @@ common_chat_params common_chat_params_init_translate_gemma(
data.prompt += data.generation_prompt;
}
- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
return p.literal(data.generation_prompt) << p.content(p.rest());
});
- data.parser = parser.save();
return data;
}
diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp
index 75a908a28..8033f7e81 100644
--- a/common/peg-parser.cpp
+++ b/common/peg-parser.cpp
@@ -1814,309 +1814,6 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo
}
}
-static common_json serialize_parser_variant(const common_peg_parser_variant & variant) {
- using json = common_json;
-
- return std::visit([](const auto & p) -> json {
- using T = std::decay_t<decltype(p)>;
-
- if constexpr (std::is_same_v<T, common_peg_epsilon_parser>) {
- return json{{"type", "epsilon"}};
- } else if constexpr (std::is_same_v<T, common_peg_start_parser>) {
- return json{{"type", "start"}};
- } else if constexpr (std::is_same_v<T, common_peg_end_parser>) {
- return json{{"type", "end"}};
- } else if constexpr (std::is_same_v<T, common_peg_literal_parser>) {
- return json{{"type", "literal"}, {"literal", p.literal}};
- } else if constexpr (std::is_same_v<T, common_peg_sequence_parser>) {
- return json{{"type", "sequence"}, {"children", p.children}};
- } else if constexpr (std::is_same_v<T, common_peg_choice_parser>) {
- return json{{"type", "choice"}, {"children", p.children}};
- } else if constexpr (std::is_same_v<T, common_peg_repetition_parser>) {
- return json{
- {"type", "repetition"},
- {"child", p.child},
- {"min_count", p.min_count},
- {"max_count", p.max_count}
- };
- } else if constexpr (std::is_same_v<T, common_peg_and_parser>) {
- return json{{"type", "and"}, {"child", p.child}};
- } else if constexpr (std::is_same_v<T, common_peg_not_parser>) {
- return json{{"type", "not"}, {"child", p.child}};
- } else if constexpr (std::is_same_v<T, common_peg_any_parser>) {
- return json{{"type", "any"}};
- } else if constexpr (std::is_same_v<T, common_peg_space_parser>) {
- return json{{"type", "space"}};
- } else if constexpr (std::is_same_v<T, common_peg_chars_parser>) {
- json ranges = json::array();
- for (const auto & range : p.ranges) {
- ranges.push_back({{"start", range.start}, {"end", range.end}});
- }
- return json{
- {"type", "chars"},
- {"pattern", p.pattern},
- {"ranges", ranges},
- {"negated", p.negated},
- {"min_count", p.min_count},
- {"max_count", p.max_count}
- };
- } else if constexpr (std::is_same_v<T, common_peg_string_parser>) {
- return json{{"type", "string"}, {"delimiter", std::string(1, p.delimiter)}};
- } else if constexpr (std::is_same_v<T, common_peg_until_parser>) {
- return json{{"type", "until"}, {"delimiters", p.delimiters}};
- } else if constexpr (std::is_same_v<T, common_peg_schema_parser>) {
- return json{
- {"type", "schema"},
- {"child", p.child},
- {"name", p.name},
- {"raw", p.raw}
- };
- } else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
- return json{
- {"type", "rule"},
- {"name", p.name},
- {"child", p.child},
- {"trigger", p.trigger}
- };
- } else if constexpr (std::is_same_v<T, common_peg_ref_parser>) {
- return json{{"type", "ref"}, {"name", p.name}};
- } else if constexpr (std::is_same_v<T, common_peg_atomic_parser>) {
- return json{{"type", "atomic"}, {"child", p.child}};
- } else if constexpr (std::is_same_v<T, common_peg_tag_parser>) {
- return json{
- {"type", "tag"},
- {"child", p.child},
- {"tag", p.tag}
- };
- } else if constexpr (std::is_same_v<T, common_peg_gbnf_parser>) {
- return json{{"type", "gbnf"}, {"child", p.child}, {"grammar", p.grammar}};
- } else if constexpr (std::is_same_v<T, common_peg_ac_parser>) {
- return json{{"type", "ac"}, {"child", p.child}, {"delimiters", p.delimiters}};
- }
- }, variant);
-}
-
-common_json common_peg_arena::to_json() const {
- auto parsers = common_json::array();
- for (const auto & parser : parsers_) {
- parsers.push_back(serialize_parser_variant(parser));
- }
- return common_json{
- {"parsers", parsers},
- {"rules", rules_},
- {"root", root_}
- };
-}
-
-static common_peg_parser_variant deserialize_parser_variant(const common_json & j) {
- if (!j.contains("type") || !j["type"].is_string()) {
- throw std::runtime_error("Parser variant JSON missing or invalid 'type' field");
- }
-
- std::string type = j["type"];
-
- if (type == "epsilon") {
- return common_peg_epsilon_parser{};
- }
- if (type == "start") {
- return common_peg_start_parser{};
- }
- if (type == "end") {
- return common_peg_end_parser{};
- }
- if (type == "literal") {
- if (!j.contains("literal") || !j["literal"].is_string()) {
- throw std::runtime_error("literal parser missing or invalid 'literal' field");
- }
- return common_peg_literal_parser{j["literal"]};
- }
- if (type == "sequence") {
- if (!j.contains("children") || !j["children"].is_array()) {
- throw std::runtime_error("sequence parser missing or invalid 'children' field");
- }
- return common_peg_sequence_parser{j["children"].get<std::vector<common_peg_parser_id>>()};
- }
- if (type == "choice") {
- if (!j.contains("children") || !j["children"].is_array()) {
- throw std::runtime_error("choice parser missing or invalid 'children' field");
- }
- return common_peg_choice_parser{j["children"].get<std::vector<common_peg_parser_id>>()};
- }
- if (type == "repetition") {
- if (!j.contains("child") || !j.contains("min_count") || !j.contains("max_count")) {
- throw std::runtime_error("repetition parser missing required fields");
- }
- return common_peg_repetition_parser{
- j["child"].get<common_peg_parser_id>(),
- j["min_count"].get<int>(),
- j["max_count"].get<int>()
- };
- }
- if (type == "and") {
- if (!j.contains("child")) {
- throw std::runtime_error("and parser missing 'child' field");
- }
- return common_peg_and_parser{j["child"].get<common_peg_parser_id>()};
- }
- if (type == "not") {
- if (!j.contains("child")) {
- throw std::runtime_error("not parser missing 'child' field");
- }
- return common_peg_not_parser{j["child"].get<common_peg_parser_id>()};
- }
- if (type == "any") {
- return common_peg_any_parser{};
- }
- if (type == "space") {
- return common_peg_space_parser{};
- }
- if (type == "chars") {
- if (!j.contains("pattern") || !j.contains("ranges") || !j.contains("negated") ||
- !j.contains("min_count") || !j.contains("max_count")) {
- throw std::runtime_error("chars parser missing required fields");
- }
- common_peg_chars_parser parser;
- parser.pattern = j["pattern"];
- parser.negated = j["negated"].get<bool>();
- parser.min_count = j["min_count"].get<int>();
- parser.max_count = j["max_count"].get<int>();
- for (const auto & range_json : j["ranges"]) {
- if (!range_json.contains("start") || !range_json.contains("end")) {
- throw std::runtime_error("char_range missing 'start' or 'end' field");
- }
- parser.ranges.push_back({
- range_json["start"].get<uint32_t>(),
- range_json["end"].get<uint32_t>()
- });
- }
- return parser;
- }
- if (type == "string") {
- if (!j.contains("delimiter")) {
- throw std::runtime_error("string parser missing delimiter field.");
- }
- std::string delimiter = j["delimiter"];
- if (delimiter.empty()) {
- throw std::runtime_error("string parser delimiter is empty.");
- }
- return common_peg_string_parser{delimiter[0]};
- }
- if (type == "until") {
- if (!j.contains("delimiters") || !j["delimiters"].is_array()) {
- throw std::runtime_error("until parser missing or invalid 'delimiters' field");
- }
- return common_peg_until_parser{j["delimiters"].get<std::vector<std::string>>()};
- }
- if (type == "schema") {
- if (!j.contains("child") || !j.contains("name") || !j.contains("raw")) {
- throw std::runtime_error("schema parser missing required fields");
- }
- common_peg_schema_parser parser;
- parser.child = j["child"].get<common_peg_parser_id>();
- parser.name = j["name"];
- parser.raw = j["raw"].get<bool>();
- return parser;
- }
- if (type == "rule") {
- if (!j.contains("name") || !j.contains("child") || !j.contains("trigger")) {
- throw std::runtime_error("rule parser missing required fields");
- }
- return common_peg_rule_parser{
- j["name"].get<std::string>(),
- j["child"].get<common_peg_parser_id>(),
- j["trigger"].get<bool>()
- };
- }
- if (type == "ref") {
- if (!j.contains("name") || !j["name"].is_string()) {
- throw std::runtime_error("ref parser missing or invalid 'name' field");
- }
- return common_peg_ref_parser{j["name"]};
- }
- if (type == "atomic") {
- if (!j.contains("child")) {
- throw std::runtime_error("tag parser missing required fields");
- }
- return common_peg_atomic_parser{
- j["child"].get<common_peg_parser_id>(),
- };
- }
- if (type == "tag") {
- if (!j.contains("child") || !j.contains("tag")) {
- throw std::runtime_error("tag parser missing required fields");
- }
- return common_peg_tag_parser{
- j["child"].get<common_peg_parser_id>(),
- j["tag"].get<std::string>(),
- };
- }
-
- if (type == "gbnf") {
- if (!j.contains("child") || !j.contains("grammar")) {
- throw std::runtime_error("gbnf parser missing required fields");
- }
- return common_peg_gbnf_parser{
- j["child"].get<common_peg_parser_id>(),
- j["grammar"].get<std::string>(),
- };
- }
-
- if (type == "ac") {
- if (!j.contains("child") || !j.contains("delimiters") || !j["delimiters"].is_array() || j["delimiters"].empty()) {
- throw std::runtime_error("ac parser requires 'child' and a non-empty 'delimiters' array");
- }
- return common_peg_ac_parser{
- j["child"].get<common_peg_parser_id>(),
- j["delimiters"].get<std::vector<std::string>>(),
- };
- }
-
- throw std::runtime_error("Unknown parser type: " + type);
-}
-
-common_peg_arena common_peg_arena::from_json(const common_json & j) {
- if (!j.contains("parsers") || !j["parsers"].is_array()) {
- throw std::runtime_error("JSON missing or invalid 'parsers' array");
- }
- if (!j.contains("rules") || !j["rules"].is_object()) {
- throw std::runtime_error("JSON missing or invalid 'rules' object");
- }
- if (!j.contains("root")) {
- throw std::runtime_error("JSON missing 'root' field");
- }
-
- common_peg_arena arena;
-
- const auto & parsers_json = j["parsers"];
- arena.parsers_.reserve(parsers_json.size());
- for (const auto & parser_json : parsers_json) {
- arena.parsers_.push_back(deserialize_parser_variant(parser_json));
- }
-
- arena.rules_ = j["rules"].get<std::unordered_map<std::string, common_peg_parser_id>>();
-
- for (const auto & [name, id] : arena.rules_) {
- if (id >= arena.parsers_.size()) {
- throw std::runtime_error("Rule '" + name + "' references invalid parser ID: " + std::to_string(id));
- }
- }
-
- arena.root_ = j["root"].get<common_peg_parser_id>();
- if (arena.root_ != COMMON_PEG_INVALID_PARSER_ID && arena.root_ >= arena.parsers_.size()) {
- throw std::runtime_error("Root references invalid parser ID: " + std::to_string(arena.root_));
- }
-
- return arena;
-}
-
-std::string common_peg_arena::save() const {
- return to_json().dump();
-}
-
-void common_peg_arena::load(const std::string & data) {
- *this = from_json(common_json::parse(data));
-}
-
common_peg_arena build_peg_parser(const std::function<common_peg_parser(common_peg_parser_builder & builder)> & fn) {
common_peg_parser_builder builder;
builder.set_root(fn(builder));
diff --git a/common/peg-parser.h b/common/peg-parser.h
index 816d007b0..9255d6eb9 100644
--- a/common/peg-parser.h
+++ b/common/peg-parser.h
@@ -357,12 +357,6 @@ class common_peg_arena {
std::string dump(common_peg_parser_id id) const;
- common_json to_json() const;
- static common_peg_arena from_json(const common_json & j);
-
- std::string save() const;
- void load(const std::string & data);
-
friend class common_peg_parser_builder;
private:
diff --git a/common/sampling.cpp b/common/sampling.cpp
index 6cc8872f2..9e16693fb 100644
--- a/common/sampling.cpp
+++ b/common/sampling.cpp
@@ -1050,3 +1050,41 @@ std::vector<common_sampler_type> common_sampler_types_from_chars(const std::stri
return samplers;
}
+
+void common_sampling_add_preserved_tokens(common_params_sampling & sampling, const llama_vocab * vocab, const std::vector<std::string> & tokens) {
+ GGML_ASSERT(vocab != nullptr);
+ for (const auto & t : tokens) {
+ auto ids = common_tokenize(vocab, t, false, true);
+ if (ids.size() == 1) {
+ sampling.preserved_tokens.insert(ids[0]);
+ }
+ }
+}
+
+void common_sampling_add_grammar_triggers(common_params_sampling & sampling, const llama_vocab * vocab, std::vector<common_grammar_trigger> triggers) {
+ GGML_ASSERT(vocab != nullptr);
+ for (auto & trigger : triggers) {
+ if (trigger.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {
+ const auto & word = trigger.value;
+ auto ids = common_tokenize(vocab, word, false, true);
+ if (ids.size() == 1) {
+ auto token = ids[0];
+ if (std::find(sampling.preserved_tokens.begin(), sampling.preserved_tokens.end(), (llama_token) token) == sampling.preserved_tokens.end()) {
+ throw std::runtime_error("Grammar trigger word should be marked as preserved token: " + word);
+ }
+ common_grammar_trigger token_trigger;
+ token_trigger.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN;
+ token_trigger.value = word;
+ token_trigger.token = token;
+ sampling.grammar_triggers.push_back(std::move(token_trigger));
+ } else {
+ sampling.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word});
+ }
+ } else {
+ sampling.grammar_triggers.push_back(std::move(trigger));
+ }
+ }
+ if (sampling.grammar_lazy && sampling.grammar_triggers.empty()) {
+ throw std::runtime_error("Error: no triggers set for lazy grammar!");
+ }
+}
diff --git a/common/sampling.h b/common/sampling.h
index 7ebae3df8..f19bd27c4 100644
--- a/common/sampling.h
+++ b/common/sampling.h
@@ -118,6 +118,12 @@ std::string common_sampler_type_to_str(enum common_sampler_type cnstr);
std::vector<enum common_sampler_type> common_sampler_types_from_names(const std::vector<std::string> & names);
std::vector<enum common_sampler_type> common_sampler_types_from_chars(const std::string & chars);
+// add the strings that are a single token in the vocab to the preserved tokens
+void common_sampling_add_preserved_tokens(common_params_sampling & sampling, const llama_vocab * vocab, const std::vector<std::string> & tokens);
+
+// add grammar triggers, a trigger word that is a single token becomes a token trigger and must be a preserved token
+void common_sampling_add_grammar_triggers(common_params_sampling & sampling, const llama_vocab * vocab, std::vector<common_grammar_trigger> triggers);
+
llama_sampler * llama_sampler_init_llg(const llama_vocab * vocab,
const char * grammar_kind, const char * grammar_data);
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index c44189ceb..5f9bd037f 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -245,7 +245,6 @@ llama_build_and_test(
peg-parser/test-basic.cpp
peg-parser/test-gbnf-generation.cpp
peg-parser/test-json-parser.cpp
- peg-parser/test-json-serialization.cpp
peg-parser/test-python-dict-parser.cpp
peg-parser/test-unicode.cpp
peg-parser/tests.h
diff --git a/tests/peg-parser/test-json-serialization.cpp b/tests/peg-parser/test-json-serialization.cpp
deleted file mode 100644
index da63a23bf..000000000
--- a/tests/peg-parser/test-json-serialization.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-#include "tests.h"
-
-void test_json_serialization(testing &t) {
- auto original = build_peg_parser([](common_peg_parser_builder & p) {
- return "<tool_call>" + p.json() + "</tool_call>";
- });
-
- auto json_serialized = original.to_json().dump();
-
- t.test("compare before/after", [&](testing &t) {
- auto deserialized = common_peg_arena::from_json(common_json::parse(json_serialized));
-
- // Test complex JSON
- std::string input = R"({"name": "test", "values": [1, 2, 3], "nested": {"a": true}})";
- common_peg_parse_context ctx1(input);
- common_peg_parse_context ctx2(input);
-
- auto result1 = original.parse(ctx1);
- auto result2 = deserialized.parse(ctx2);
-
- t.assert_equal("both_succeed", result1.success(), result2.success());
- t.assert_equal("same_end_pos", result1.end, result2.end);
- });
-
- t.bench("deserialize", [&]() {
- auto deserialized = common_peg_arena::from_json(common_json::parse(json_serialized));
- }, 100);
-}
diff --git a/tests/peg-parser/tests.h b/tests/peg-parser/tests.h
index 00e81815b..287da16d1 100644
--- a/tests/peg-parser/tests.h
+++ b/tests/peg-parser/tests.h
@@ -21,5 +21,4 @@ void test_basic(testing &t);
void test_json_parser(testing &t);
void test_gbnf_generation(testing &t);
void test_unicode(testing &t);
-void test_json_serialization(testing &t);
void test_python_dict_parser(testing &t);
diff --git a/tests/test-chat-auto-parser.cpp b/tests/test-chat-auto-parser.cpp
index 5aa948251..1c641b329 100644
--- a/tests/test-chat-auto-parser.cpp
+++ b/tests/test-chat-auto-parser.cpp
@@ -450,7 +450,7 @@ static int debug_single_template(const debug_options & opts) {
generation_params params = prepare_debug_params(opts, tools);
common_chat_params parser_data;
if (std::optional<common_chat_params> spec_tmpl =
- common_chat_try_specialized_template(chat_template, template_source, params)) {
+ common_chat_try_specialized_template(chat_template, params)) {
LOG_ERR("\n");
LOG_ERR("This template uses a specialized parser, analysis results will not be available.\n");
parser_data = *spec_tmpl;
@@ -484,8 +484,7 @@ static int debug_single_template(const debug_options & opts) {
if (!std::empty(parser_data.parser)) {
LOG_ERR("\n=== Generated Parser ===\n");
- common_peg_arena arena;
- arena.load(parser_data.parser);
+ const common_peg_arena & arena = parser_data.parser;
LOG_ERR("%s\n", arena.dump(arena.root()).c_str());
LOG_ERR("\n=== Generated Grammar ===\n");
diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp
index 41e4948c4..09f593692 100644
--- a/tests/test-chat.cpp
+++ b/tests/test-chat.cpp
@@ -1090,26 +1090,6 @@ struct peg_test_case {
std::vector<std::string> expect_rules;
};
-struct make_peg_parser {
- common_chat_params params_;
- common_peg_arena arena_;
- bool detailed_debug_;
-
- make_peg_parser(common_chat_templates * tmpls,
- const common_chat_templates_inputs & inputs,
- bool detailed_debug = false) {
- detailed_debug_ = detailed_debug;
- params_ = common_chat_templates_apply(tmpls, inputs);
- arena_.load(params_.parser);
- }
-
- common_chat_msg parse(const std::string & msg, bool is_partial) const {
- common_chat_parser_params parser_params(params_);
- parser_params.debug = detailed_debug_;
- return common_chat_peg_parse(arena_, common_chat_input(msg), is_partial, parser_params);
- }
-};
-
// Global template filter for --template flag
static std::string g_template_filter;
@@ -1161,14 +1141,19 @@ static void test_peg_parser(common_chat_templates * tmpls,
tc.expect.role = "assistant";
}
- auto parser = make_peg_parser(tmpls, tc.params, detailed_debug);
+ common_chat_session_params session_params;
+ session_params.debug = detailed_debug;
+ common_chat_session session(tmpls, nullptr, tc.params, session_params);
+ const auto & parser = session.parser();
+ common_params_sampling sampling;
+ session.apply_sampling(sampling);
if (detailed_debug) {
- LOG_DBG("Using parser: \n%s\n", parser.arena_.dump(parser.arena_.root()).c_str());
- LOG_DBG("Generation prompt: '%s'\n", parser.params_.generation_prompt.c_str());
+ LOG_DBG("Using parser: \n%s\n", parser.dump(parser.root()).c_str());
+ LOG_DBG("Generation prompt: '%s'\n", session.generation_prompt().c_str());
}
for (const auto & rule : tc.expect_rules) {
- if (!parser.arena_.has_rule(rule)) {
+ if (!parser.has_rule(rule)) {
LOG_ERR("Missing rule: %s\n", rule.c_str());
common_log_flush(common_log_main());
throw std::runtime_error("Test failed");
@@ -1179,12 +1164,14 @@ static void test_peg_parser(common_chat_templates * tmpls,
common_chat_msg msg_prev;
msg_accum.role = msg_prev.role = "assistant";
+ size_t fed = 0;
for (size_t i = 1; i <= tc.input.size(); ++i) {
auto is_partial = i < tc.input.size() || tc.is_partial;
// Use UTF-8 safe truncation to avoid corrupting multi-byte characters
size_t safe_len = utf8_truncate_safe_len(std::string_view(tc.input).substr(0, i));
- std::string prefix = tc.input.substr(0, safe_len);
- common_chat_msg msg_current = parser.parse(prefix, is_partial);
+ common_chat_input chunk(tc.input.substr(fed, safe_len - fed));
+ fed = safe_len;
+ const common_chat_msg & msg_current = is_partial ? session.feed(chunk) : session.finish(chunk);
for (const auto & diff : common_chat_msg_diff::compute_diffs(msg_prev, msg_current)) {
if (!diff.reasoning_content_delta.empty()) {
@@ -1222,30 +1209,33 @@ static void test_peg_parser(common_chat_templates * tmpls,
}
if (!tc.is_partial) {
- assert_msg_equals(tc.expect, parser.parse(tc.input, false), true);
+ if (tc.input.empty()) {
+ session.finish();
+ }
+ assert_msg_equals(tc.expect, session.msg(), true);
}
assert_msg_equals(tc.expect, msg_accum, true);
// A response format must be enforced by an eager grammar
if (!tc.params.json_schema.empty()) {
- if (parser.params_.grammar.empty()) {
+ if (session.grammar().empty()) {
throw std::runtime_error("json_schema is set but no grammar was produced");
}
- if (parser.params_.grammar_lazy) {
+ if (sampling.grammar_lazy) {
throw std::runtime_error("json_schema is set but the grammar is lazy");
}
}
// Test grammar if present in params
- if (!parser.params_.grammar.empty()) {
- auto grammar = build_grammar(parser.params_.grammar);
+ if (!session.grammar().empty()) {
+ auto grammar = build_grammar(session.grammar());
if (!grammar) {
- throw std::runtime_error("Failed to build grammar: " + parser.params_.grammar);
+ throw std::runtime_error("Failed to build grammar: " + session.grammar());
}
// In production, grammar triggers match against the full generated text
// including the generation prompt. All positions are in full_input coordinates.
- const auto & gen_prompt = parser.params_.generation_prompt;
+ const auto & gen_prompt = sampling.generation_prompt;
std::string full_input = gen_prompt + tc.input;
// Determine whether the reasoning-budget sampler path applies: tool-call grammar
@@ -1253,9 +1243,9 @@ static void test_peg_parser(common_chat_templates * tmpls,
// budget sampler inhibits grammar application while inside thinking blocks —
// triggers inside <think>...</think> are suppressed.
bool use_reasoning_budget_path = false;
- if (parser.params_.grammar_lazy && !parser.params_.thinking_end_tags.empty()) {
+ if (sampling.grammar_lazy && !session.thinking_end_tags().empty()) {
use_reasoning_budget_path = true;
- for (const auto & trigger : parser.params_.grammar_triggers) {
+ for (const auto & trigger : sampling.grammar_triggers) {
if (trigger.type != COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {
use_reasoning_budget_path = false;
break;
@@ -1270,8 +1260,8 @@ static void test_peg_parser(common_chat_templates * tmpls,
// Reasoning-budget path: simulate thinking-aware trigger detection.
// Walk through full_input tracking thinking state; only match triggers
// when outside thinking blocks.
- const auto & think_start = parser.params_.thinking_start_tag;
- const auto & think_ends = parser.params_.thinking_end_tags;
+ const auto & think_start = session.thinking_start_tag();
+ const auto & think_ends = session.thinking_end_tags();
bool in_thinking = false;
for (size_t i = 0; i < full_input.size(); ++i) {
@@ -1292,7 +1282,7 @@ static void test_peg_parser(common_chat_templates * tmpls,
continue;
}
// Outside thinking — check if any trigger word starts here
- for (const auto & trigger : parser.params_.grammar_triggers) {
+ for (const auto & trigger : sampling.grammar_triggers) {
if (full_input.compare(i, trigger.value.size(), trigger.value) == 0) {
if (earliest_trigger_pos == std::string::npos || i < earliest_trigger_pos) {
earliest_trigger_pos = i;
@@ -1314,7 +1304,7 @@ static void test_peg_parser(common_chat_templates * tmpls,
if (!use_reasoning_budget_path) {
// Legacy path: find triggers without thinking-awareness
- for (const auto & trigger : parser.params_.grammar_triggers) {
+ for (const auto & trigger : sampling.grammar_triggers) {
size_t pos = std::string::npos;
std::smatch match;
switch (trigger.type) {
@@ -1370,16 +1360,16 @@ static void test_peg_parser(common_chat_templates * tmpls,
// If the test expects tool calls and the grammar is lazy, the trigger must fire.
// Otherwise the grammar would never activate in production and tool calls wouldn't
// be constrained. A silent skip here would hide broken triggers.
- if (parser.params_.grammar_lazy && !tc.expect.tool_calls.empty() && !tc.is_partial
+ if (sampling.grammar_lazy && !tc.expect.tool_calls.empty() && !tc.is_partial
&& earliest_trigger_pos == std::string::npos) {
std::string trigger_desc;
- for (const auto & trigger : parser.params_.grammar_triggers) {
+ for (const auto & trigger : sampling.grammar_triggers) {
trigger_desc += "\n [type=" + std::to_string(trigger.type) + "] " + trigger.value;
}
throw std::runtime_error(
"Grammar trigger did not fire, but test expects tool calls (lazy grammar).\n"
">>> Input: " + full_input + "\n"
- ">>> Triggers (" + std::to_string(parser.params_.grammar_triggers.size()) + "):" + trigger_desc);
+ ">>> Triggers (" + std::to_string(sampling.grammar_triggers.size()) + "):" + trigger_desc);
}
// Determine the constrained portion of input to test against grammar.
@@ -1392,7 +1382,7 @@ static void test_peg_parser(common_chat_templates * tmpls,
auto constrain_from = std::max(earliest_trigger_pos, gen_prompt.size());
constrained = full_input.substr(constrain_from);
grammar_triggered = true;
- } else if (!parser.params_.grammar_lazy) {
+ } else if (!sampling.grammar_lazy) {
// For non-lazy grammars, the entire input should match
grammar_triggered = true;
}
@@ -1411,7 +1401,7 @@ static void test_peg_parser(common_chat_templates * tmpls,
std::to_string(result.matched_codepoints) + " codepoints): " +
(result.matched_prefix.size() > 100 ? result.matched_prefix.substr(0, 100) + "..." : result.matched_prefix) +
"\n\n>>> Expected next: " + result.expected_description +
- "\n\n>>> Grammar: " + parser.params_.grammar;
+ "\n\n>>> Grammar: " + session.grammar();
} else {
error_msg =
"Grammar match failed:\n\n"
@@ -1422,7 +1412,7 @@ static void test_peg_parser(common_chat_templates * tmpls,
(result.matched_prefix.size() > 100 ? result.matched_prefix.substr(0, 100) + "..." : result.matched_prefix) +
"\n\n>>> Failing character: " + result.failing_char +
"\n\n>>> Expected: " + result.expected_description +
- "\n\n>>> Grammar: " + parser.params_.grammar;
+ "\n\n>>> Grammar: " + session.grammar();
}
throw std::runtime_error(error_msg);
}
@@ -1437,7 +1427,7 @@ static void test_peg_parser(common_chat_templates * tmpls,
// Start from tc.expect but copy tool call arguments from the actual parser
// output, which preserves original JSON formatting (e.g. {"arg1":1} vs {"arg1": 1}).
auto reconstruction_msg = tc.expect;
- auto parsed_msg = parser.parse(tc.input, false);
+ const auto & parsed_msg = session.msg();
for (size_t i = 0; i < reconstruction_msg.tool_calls.size() && i < parsed_msg.tool_calls.size(); i++) {
reconstruction_msg.tool_calls[i].arguments = parsed_msg.tool_calls[i].arguments;
}
@@ -1446,7 +1436,7 @@ static void test_peg_parser(common_chat_templates * tmpls,
reconstruction_inputs.add_generation_prompt = false;
auto reconstruction_params = common_chat_templates_apply(tmpls, reconstruction_inputs);
- std::string expected_text = parser.params_.prompt + tc.input;
+ std::string expected_text = session.prompt() + tc.input;
bool match = reconstruction_params.prompt == expected_text ||
(reconstruction_params.prompt.size() > expected_text.size() &&
reconstruction_params.prompt.compare(0, expected_text.size(), expected_text) == 0);
@@ -4644,8 +4634,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
inputs.messages = { msg };
auto params = common_chat_templates_apply(tmpls.get(), inputs);
- common_peg_arena arena;
- arena.load(params.parser);
+ const common_peg_arena & arena = params.parser;
common_chat_parser_params pp(params);
// generation_prompt is non-empty for thinking models, so result.end
@@ -7791,6 +7780,77 @@ static void test_deepseek_v4_tool_result_ordering() {
}
}
+static void test_chat_session() {
+ LOG_DBG("%s\n", __func__);
+ auto tmpls = read_templates("models/templates/Qwen3.5-4B.jinja");
+
+ common_chat_templates_inputs inputs;
+ inputs.messages = { message_user };
+ inputs.tools = { special_function_tool };
+ inputs.reasoning_format = COMMON_REASONING_FORMAT_AUTO;
+
+ const std::string output =
+ "I'm\nthinking\n</think>\n\n"
+ "<tool_call>\n"
+ "<function=special_function>\n"
+ "<parameter=arg1>\n1\n</parameter>\n"
+ "</function>\n"
+ "</tool_call>";
+
+ // the session renders the prompt and parses the output fed to it in small chunks
+ {
+ common_chat_session session(tmpls.get(), nullptr, inputs);
+
+ assert_contains(session.prompt(), "\"name\": \"special_function\"");
+ assert_contains(session.prompt(), "<|im_start|>user\nHey there!<|im_end|>\n<|im_start|>assistant\n<think>\n");
+ assert_equals(false, session.grammar().empty());
+ assert_equals(std::string("<|im_start|>assistant\n<think>\n"), session.generation_prompt());
+
+ const std::string thinking = "I'm\nthinking\n</think>\n\n";
+ for (size_t i = 0; i < thinking.size(); i += 3) {
+ session.feed(common_chat_input(thinking.substr(i, 3)));
+ }
+ assert_msg_equals(simple_assist_msg("", "I'm\nthinking\n"), session.msg());
+
+ for (size_t i = thinking.size(); i < output.size(); i += 3) {
+ session.feed(common_chat_input(output.substr(i, 3)));
+ }
+ assert_msg_equals(simple_assist_msg("", "I'm\nthinking\n", "special_function", "{\"arg1\":1}"), session.finish());
+ }
+
+ // a copy does not see what is fed to the original
+ {
+ common_chat_session a(tmpls.get(), nullptr, inputs);
+ common_chat_session b = a;
+ a.feed(common_chat_input(output));
+ assert_equals(true, b.msg().empty());
+ b.feed(common_chat_input("I'm\nthinking\n</think>\n\nHello"));
+ assert_equals(std::string("Hello"), b.msg().content);
+ assert_equals(std::string("special_function"), a.finish().tool_calls.at(0).name);
+ }
+
+ // a continued message starts from the prefill, unless it is echoed
+ {
+ common_chat_templates_inputs cont;
+ cont.messages = { message_user, message_assist_prefill_content };
+ cont.add_generation_prompt = false;
+ cont.continue_final_message = COMMON_CHAT_CONTINUATION_CONTENT;
+ cont.reasoning_format = COMMON_REASONING_FORMAT_AUTO;
+
+ common_chat_session session(tmpls.get(), nullptr, cont);
+ assert_equals(std::string("<|im_start|>assistant\n<think>\nI'm thinking\n</think>\n\nHello, "),
+ session.generation_prompt());
+ assert_msg_equals(simple_assist_msg("Hello, ", "I'm thinking\n"), session.msg());
+ session.feed(common_chat_input("world!"));
+ assert_equals(std::string("Hello, world!"), session.msg().content);
+
+ common_chat_session_params echo;
+ echo.echo = true;
+ common_chat_session echoed(tmpls.get(), nullptr, cont, echo);
+ assert_equals(true, echoed.msg().empty());
+ }
+}
+
static void test_reasoning_budget_tokens_per_request() {
LOG_DBG("%s\n", __func__);
// Use Qwen3 template which has <think>...</think> reasoning markers.
@@ -7811,7 +7871,8 @@ static void test_reasoning_budget_tokens_per_request() {
{"reasoning_budget_tokens", 0},
};
std::vector<raw_buffer> out_files;
- auto llama_params = oaicompat_chat_params_parse(body, opt, out_files);
+ common_chat_session out_session;
+ auto llama_params = oaicompat_chat_params_parse(nullptr, body, opt, out_files, out_session);
// The per-request value must win over the server default (-1).
if (!llama_params.contains("reasoning_budget_tokens")) {
@@ -7844,7 +7905,8 @@ static void test_reasoning_budget_message_per_request() {
{"reasoning_budget_message", per_request_message},
};
std::vector<raw_buffer> out_files;
- auto llama_params = oaicompat_chat_params_parse(body, opt, out_files);
+ common_chat_session out_session;
+ auto llama_params = oaicompat_chat_params_parse(nullptr, body, opt, out_files, out_session);
// The per-request value must win over the server default.
if (!llama_params.contains("reasoning_budget_message")) {
@@ -8035,6 +8097,7 @@ int main(int argc, char ** argv) {
test_deepseek_v4_tool_result_ordering();
test_template_generation_prompt();
test_reasoning_effort_caps();
+ test_chat_session();
test_reasoning_budget_tokens_per_request();
test_reasoning_budget_message_per_request();
test_template_output_peg_parsers(detailed_debug);
diff --git a/tests/test-peg-parser.cpp b/tests/test-peg-parser.cpp
index 7d22d7761..863bfe8a0 100644
--- a/tests/test-peg-parser.cpp
+++ b/tests/test-peg-parser.cpp
@@ -19,7 +19,6 @@ int main(int argc, char *argv[]) {
t.test("unicode", test_unicode);
t.test("json", test_json_parser);
t.test("gbnf", test_gbnf_generation);
- t.test("serialization", test_json_serialization);
t.test("python-dict", test_python_dict_parser);
return t.summary();
diff --git a/tools/server/README.md b/tools/server/README.md
index 41dca6152..8fe260a32 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -1353,10 +1353,6 @@ The `response_format` parameter supports both plain JSON output (e.g. `{"type":
`reasoning_control`: Arms realtime reasoning control for this completion so it can be ended early via `/v1/chat/completions/control`. Defaults to `false`.
-`generation_prompt`: The generation prompt that was prefilled in by the template. Prepended to model output before parsing.
-
-`parse_tool_calls`: Whether to parse the generated tool call.
-
`parallel_tool_calls` : Whether to enable parallel/multiple tool calls (only supported on some models, verification is based on jinja template).
For multimodal input (typed content, `messages[i].content[j]`):
diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp
index d364ba898..5f77d43d0 100644
--- a/tools/server/server-common.cpp
+++ b/tools/server/server-common.cpp
@@ -1266,9 +1266,11 @@ server_tokens tokenize_oai_content_array(const llama_vocab * vocab, mtmd_context
// used by /chat/completions endpoint
json oaicompat_chat_params_parse(
+ const llama_vocab * vocab,
json & body, /* openai api json semantics */
const server_chat_params & opt,
- std::vector<raw_buffer> & out_files)
+ std::vector<raw_buffer> & out_files,
+ common_chat_session & out_session)
{
json llama_params;
@@ -1393,7 +1395,6 @@ json oaicompat_chat_params_parse(
if (body.contains("grammar")) {
throw std::invalid_argument("Cannot use custom grammar constraints with tools.");
}
- llama_params["parse_tool_calls"] = true;
}
// merge the template args provided from command line with the args provided in the user request
@@ -1427,31 +1428,11 @@ json oaicompat_chat_params_parse(
inputs.force_pure_content = opt.force_pure_content;
// Apply chat template to the list of messages
- auto chat_params = common_chat_templates_apply(opt.tmpls.get(), inputs);
+ common_chat_session_params session_params;
+ session_params.echo = json_value(body, "echo", false);
+ out_session = common_chat_session(opt.tmpls.get(), vocab, inputs, session_params);
- llama_params["chat_format"] = static_cast<int>(chat_params.format);
- llama_params["prompt"] = chat_params.prompt;
- if (!chat_params.grammar.empty()) {
- llama_params["grammar"] = chat_params.grammar;
- llama_params["grammar_type"] = std::string("tool_calls");
- }
- llama_params["grammar_lazy"] = chat_params.grammar_lazy;
- auto grammar_triggers = json::array();
- for (const auto & trigger : chat_params.grammar_triggers) {
- server_grammar_trigger ct(trigger);
- grammar_triggers.push_back(ct.to_json());
- }
- llama_params["grammar_triggers"] = grammar_triggers;
- llama_params["preserved_tokens"] = chat_params.preserved_tokens;
- llama_params["generation_prompt"] = chat_params.generation_prompt;
- for (const auto & stop : chat_params.additional_stops) {
- llama_params["stop"].push_back(stop);
- }
- if (!chat_params.parser.empty()) {
- llama_params["chat_parser"] = chat_params.parser;
- }
-
- llama_params["message_delimiters"] = chat_params.message_delimiters.to_json();
+ llama_params["prompt"] = out_session.prompt();
// Reasoning budget: pass parameters through to sampling layer
{
@@ -1461,10 +1442,10 @@ json oaicompat_chat_params_parse(
reasoning_budget = opt.reasoning_budget;
}
- if (!chat_params.thinking_end_tags.empty()) {
+ if (!out_session.thinking_end_tags().empty()) {
llama_params["reasoning_budget_tokens"] = reasoning_budget;
- llama_params["reasoning_budget_start_tag"] = chat_params.thinking_start_tag;
- llama_params["reasoning_budget_end_tags"] = chat_params.thinking_end_tags;
+ llama_params["reasoning_budget_start_tag"] = out_session.thinking_start_tag();
+ llama_params["reasoning_budget_end_tags"] = out_session.thinking_end_tags();
llama_params["reasoning_budget_message"] = json_value(body, "reasoning_budget_message", opt.reasoning_budget_message);
llama_params["reasoning_control"] = json_value(body, "reasoning_control", false);
}
@@ -1491,6 +1472,15 @@ json oaicompat_chat_params_parse(
}
}
+ // the session owns these, the server applies them with server_task::apply_chat_session()
+ for (const char * key : { "grammar_lazy", "grammar_triggers", "preserved_tokens" }) {
+ llama_params.erase(key);
+ }
+ if (!out_session.grammar().empty()) {
+ llama_params.erase("grammar");
+ llama_params.erase("json_schema");
+ }
+
return llama_params;
}
diff --git a/tools/server/server-common.h b/tools/server/server-common.h
index 6165d871c..5bd2ed57a 100644
--- a/tools/server/server-common.h
+++ b/tools/server/server-common.h
@@ -355,9 +355,11 @@ json oaicompat_completion_params_parse(const json & body);
// used by /chat/completions endpoint
json oaicompat_chat_params_parse(
+ const llama_vocab * vocab,
json & body, /* openai api json semantics */
const server_chat_params & opt,
- std::vector<raw_buffer> & out_files);
+ std::vector<raw_buffer> & out_files,
+ common_chat_session & out_session);
// used by /embeddings endpoint, content has the same format as a chat message content array
server_tokens tokenize_oai_content_array(
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 1e3021d4a..9a9c0c90d 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -4767,7 +4767,8 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
server_task_type type,
const json & data,
const std::vector<raw_buffer> & files,
- task_response_type res_type) {
+ task_response_type res_type,
+ const common_chat_session & chat_session) {
GGML_ASSERT(type == SERVER_TASK_TYPE_COMPLETION || type == SERVER_TASK_TYPE_INFILL);
auto res = create_response();
@@ -4809,11 +4810,6 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
// tasks.reserve(inputs.size()); // TODO: this is inaccurate due to child tasks
- // message delimiters for checkpointing
- json delims = json_value(data, "message_delimiters", json::array());
- auto delimiters = common_chat_msg_delimiters_parse(delims);
- delimiters.tokenize(ctx_server.vocab);
-
for (size_t i = 0; i < inputs.size(); i++) {
server_task task = server_task(type);
@@ -4826,7 +4822,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
meta->logit_bias_eog,
data);
- task.params.message_spans = task.tokens.find_message_spans(delimiters);
+ task.apply_chat_session(chat_session);
task.id_slot = json_value(data, "id_slot", -1);
sse_ping_interval = task.params.sse_ping_interval;
@@ -4847,7 +4843,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
tasks.push_back(std::move(task));
}
- rd.post_tasks(std::move(tasks));
+ rd.post_tasks(std::move(tasks), chat_session);
} catch (const std::exception & e) {
res->error(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST));
return res;
@@ -5444,16 +5440,20 @@ void server_routes::init_routes() {
auto res = create_response();
std::vector<raw_buffer> files;
json body = json::parse(req.body);
+ common_chat_session session;
json body_parsed = oaicompat_chat_params_parse(
+ ctx_server.vocab,
body,
meta->chat_params,
- files);
+ files,
+ session);
return handle_completions_impl(
req,
SERVER_TASK_TYPE_COMPLETION,
body_parsed,
files,
- TASK_RESPONSE_TYPE_OAI_CHAT);
+ TASK_RESPONSE_TYPE_OAI_CHAT,
+ session);
};
this->post_chat_completions_tok = [this](const server_http_req & req) {
@@ -5503,16 +5503,20 @@ void server_routes::init_routes() {
json body = server_chat_convert_responses_to_chatcmpl(json::parse(req.body));
SRV_DBG("%s\n", "Request converted: OpenAI Responses -> OpenAI Chat Completions");
SRV_DBG("converted request: %s\n", body.dump().c_str());
+ common_chat_session session;
json body_parsed = oaicompat_chat_params_parse(
+ ctx_server.vocab,
body,
meta->chat_params,
- files);
+ files,
+ session);
return handle_completions_impl(
req,
SERVER_TASK_TYPE_COMPLETION,
body_parsed,
files,
- TASK_RESPONSE_TYPE_OAI_RESP);
+ TASK_RESPONSE_TYPE_OAI_RESP,
+ session);
};
this->post_responses_tok_oai = [this](const server_http_req & req) {
@@ -5535,16 +5539,20 @@ void server_routes::init_routes() {
files);
SRV_DBG("%s\n", "Request converted: OpenAI Transcriptions -> OpenAI Chat Completions");
SRV_DBG("converted request: %s\n", body.dump().c_str());
+ common_chat_session session;
json body_parsed = oaicompat_chat_params_parse(
+ ctx_server.vocab,
body,
meta->chat_params,
- files);
+ files,
+ session);
return handle_completions_impl(
req,
SERVER_TASK_TYPE_COMPLETION,
body_parsed,
files,
- TASK_RESPONSE_TYPE_OAI_ASR);
+ TASK_RESPONSE_TYPE_OAI_ASR,
+ session);
};
this->post_anthropic_messages = [this](const server_http_req & req) {
@@ -5553,16 +5561,20 @@ void server_routes::init_routes() {
json body = server_chat_convert_anthropic_to_oai(json::parse(req.body));
SRV_DBG("%s\n", "Request converted: Anthropic -> OpenAI Chat Completions");
SRV_DBG("converted request: %s\n", body.dump().c_str());
+ common_chat_session session;
json body_parsed = oaicompat_chat_params_parse(
+ ctx_server.vocab,
body,
meta->chat_params,
- files);
+ files,
+ session);
return handle_completions_impl(
req,
SERVER_TASK_TYPE_COMPLETION,
body_parsed,
files,
- TASK_RESPONSE_TYPE_ANTHROPIC);
+ TASK_RESPONSE_TYPE_ANTHROPIC,
+ session);
};
this->post_anthropic_count_tokens = [this](const server_http_req & req) {
@@ -5573,11 +5585,14 @@ void server_routes::init_routes() {
this->post_apply_template = [this](const server_http_req & req) {
auto res = create_response();
std::vector<raw_buffer> files; // dummy, unused
+ common_chat_session session; // dummy, unused
json body = json::parse(req.body);
json data = oaicompat_chat_params_parse(
+ ctx_server.vocab,
body,
meta->chat_params,
- files);
+ files,
+ session);
res->ok({{ "prompt", std::move(data.at("prompt")) }});
return res;
};
@@ -6129,10 +6144,13 @@ std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const s
return res;
}
+ common_chat_session session; // dummy, unused
json body_parsed = oaicompat_chat_params_parse(
+ ctx_server.vocab,
body,
meta->chat_params,
- files);
+ files,
+ session);
json prompt = body_parsed.at("prompt");
// SRV_DBG("prompt = %s\n", prompt.dump().c_str());
diff --git a/tools/server/server-context.h b/tools/server/server-context.h
index 7fe003531..ed23f7d4e 100644
--- a/tools/server/server-context.h
+++ b/tools/server/server-context.h
@@ -166,7 +166,8 @@ private:
server_task_type type,
const json & data,
const std::vector<raw_buffer> & files,
- task_response_type res_type);
+ task_response_type res_type,
+ const common_chat_session & chat_session = {});
std::unique_ptr<server_res_generator> handle_slots_save(const server_http_req & req, int id_slot);
std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cpp
index 78169e9a5..3352c3cb5 100644
--- a/tools/server/server-queue.cpp
+++ b/tools/server/server-queue.cpp
@@ -517,23 +517,23 @@ void server_response_reader::post_task(server_task && task, bool front) {
GGML_ASSERT(!task.is_parent() && "not supported, use post_tasks() instead");
task.index = 0;
id_tasks.insert(task.id);
- states.push_back(task.create_state());
+ states.push_back(task_result_state());
queue_results.add_waiting_task_id(task.id);
queue_tasks.post(std::move(task), front);
}
-void server_response_reader::post_tasks(std::vector<server_task> && tasks, bool front) {
+void server_response_reader::post_tasks(std::vector<server_task> && tasks, const common_chat_session & session, bool front) {
GGML_ASSERT(id_tasks.empty() && "post_tasks() can only be called once per reader");
id_tasks = server_task::get_list_id(tasks);
states.reserve(tasks.size());
size_t index = 0;
for (auto & task : tasks) {
task.index = index++;
- states.push_back(task.create_state());
+ states.push_back(task_result_state(session));
// for child tasks
for (auto & child_task : task.child_tasks) {
child_task.index = index++;
- states.push_back(child_task.create_state());
+ states.push_back(task_result_state(session));
}
}
GGML_ASSERT(states.size() == id_tasks.size());
diff --git a/tools/server/server-queue.h b/tools/server/server-queue.h
index e17733a74..692255e4f 100644
--- a/tools/server/server-queue.h
+++ b/tools/server/server-queue.h
@@ -225,7 +225,7 @@ struct server_response_reader {
// if front = true, the task will be posted to the front of the queue (high priority)
void post_task(server_task && task, bool front = false);
- void post_tasks(std::vector<server_task> && tasks, bool front = false);
+ void post_tasks(std::vector<server_task> && tasks, const common_chat_session & session = {}, bool front = false);
bool has_next() const;
// return nullptr if should_stop() is true before receiving a result
diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp
index c7d0bc4de..54e126aa0 100644
--- a/tools/server/server-schema.cpp
+++ b/tools/server/server-schema.cpp
@@ -291,54 +291,23 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
// Chat parser params
//
- // TODO: change this to string field instead
- add((new field_json("chat_format"))
- ->set_desc("Chat format used internally by the server")
- ->set_handler([&](field_eval_context & ctx, const json & data) {
- ctx.params.chat_parser_params.format = static_cast<common_chat_format>(data.at("chat_format").get<int>());
- SRV_TRC("chat format: %s\n", common_chat_format_name(ctx.params.chat_parser_params.format));
- }));
-
add((new field_str("reasoning_format"))
->set_desc("Reasoning format for chain-of-thought models")
->set_handler([&](field_eval_context & ctx, const json & data) {
- auto reasoning_format = common_reasoning_format_from_name(data.at("reasoning_format").get<std::string>());
- ctx.params.chat_parser_params.reasoning_format = reasoning_format;
- ctx.params.chat_parser_params.reasoning_in_content = ctx.params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);
- }));
-
- add((new field_str("generation_prompt"))
- ->set_desc("Generation prompt appended to the chat template output")
- ->set_handler([&](field_eval_context & ctx, const json & data) {
- std::string s = data.at("generation_prompt").get<std::string>();
- ctx.params.sampling.generation_prompt = s;
-
- if (ctx.vocab == nullptr) {
- ctx.params.chat_parser_params.generation_prompt = common_chat_input(s);
- return;
- }
-
- ctx.params.chat_parser_params.generation_prompt = common_chat_input_tokenize(ctx.vocab, s);
- }));
-
- add((new field_bool("parse_tool_calls", params.chat_parser_params.parse_tool_calls))
- ->set_desc("Whether to parse tool calls from the generated output"));
-
- add((new field_str("chat_parser"))
- ->set_desc("Chat parser configuration string")
- ->set_handler([&](field_eval_context & ctx, const json & data) {
- ctx.params.chat_parser_params.parser.load(data.at("chat_parser").get<std::string>());
+ ctx.params.reasoning_format = common_reasoning_format_from_name(data.at("reasoning_format").get<std::string>());
}));
add((new field_json("continue_final_message"))
->set_desc("Whether to continue the final message of the chat template")
- ->set_handler([&](field_eval_context & ctx, const json & data) {
- auto continuation = common_chat_continuation_parse(data.at("continue_final_message"));
- ctx.params.chat_parser_params.is_continuation = continuation != COMMON_CHAT_CONTINUATION_NONE;
+ ->set_handler([&](field_eval_context &, const json & data) {
+ common_chat_continuation_parse(data.at("continue_final_message"));
}));
- add((new field_bool("echo", params.chat_parser_params.echo))
- ->set_desc("Whether to echo the input tokens in the output"));
+ add((new field_json("echo"))
+ ->set_desc("Whether to include the continued assistant message in the output")
+ ->set_handler([&](field_eval_context &, const json & data) {
+ data.at("echo").get<bool>();
+ }));
//
// Token-level fields (require vocab)
@@ -347,44 +316,17 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
add((new field_json("preserved_tokens"))
->set_desc("List of token strings that must not be split during tokenization")
->set_handler([&](field_eval_context & ctx, const json & data) {
- GGML_ASSERT(ctx.vocab != nullptr);
- for (const auto & t : data.at("preserved_tokens")) {
- auto ids = common_tokenize(ctx.vocab, t.get<std::string>(), false, true);
- if (ids.size() == 1) {
- ctx.params.sampling.preserved_tokens.insert(ids[0]);
- }
- }
+ common_sampling_add_preserved_tokens(ctx.params.sampling, ctx.vocab, data.at("preserved_tokens").get<std::vector<std::string>>());
}));
add((new field_json("grammar_triggers"))
->set_desc("List of strings or patterns that trigger grammar-constrained generation")
->set_handler([&](field_eval_context & ctx, const json & data) {
- GGML_ASSERT(ctx.vocab != nullptr);
+ std::vector<common_grammar_trigger> triggers;
for (const auto & t : data.at("grammar_triggers")) {
- server_grammar_trigger ct(t);
- if (ct.value.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {
- const auto & word = ct.value.value;
- auto ids = common_tokenize(ctx.vocab, word, false, true);
- if (ids.size() == 1) {
- auto token = ids[0];
- if (std::find(ctx.params.sampling.preserved_tokens.begin(), ctx.params.sampling.preserved_tokens.end(), (llama_token) token) == ctx.params.sampling.preserved_tokens.end()) {
- throw std::runtime_error("Grammar trigger word should be marked as preserved token: " + word);
- }
- common_grammar_trigger trigger;
- trigger.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN;
- trigger.value = word;
- trigger.token = token;
- ctx.params.sampling.grammar_triggers.push_back(std::move(trigger));
- } else {
- ctx.params.sampling.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word});
- }
- } else {
- ctx.params.sampling.grammar_triggers.emplace_back(std::move(ct.value));
- }
- }
- if (ctx.params.sampling.grammar_lazy && ctx.params.sampling.grammar_triggers.empty()) {
- throw std::runtime_error("Error: no triggers set for lazy grammar!");
+ triggers.push_back(server_grammar_trigger(t).value);
}
+ common_sampling_add_grammar_triggers(ctx.params.sampling, ctx.vocab, std::move(triggers));
}));
add((new field_bool("reasoning_control", params.sampling.reasoning_control))
@@ -542,7 +484,7 @@ task_params eval_llama_cmpl_schema(
// enabling this will output extra debug information in the HTTP responses from the server
params.verbose = params_base.verbosity > 9;
- params.chat_parser_params.reasoning_format = params_base.reasoning_format;
+ params.reasoning_format = params_base.reasoning_format;
// create context and schema
field_eval_context ctx(params);
@@ -556,13 +498,6 @@ task_params eval_llama_cmpl_schema(
f->eval(ctx, data);
}
- // post-processing
- {
- // if "reasoning_format" is not provided, its handler will not be called, we will need to handle it here
- auto reasoning_format = params.chat_parser_params.reasoning_format;
- params.chat_parser_params.reasoning_in_content = params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);
- }
-
// debugging
{
auto budget = params.sampling.reasoning_budget_tokens;
diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
index 21c1fe4e1..3f30da2cd 100644
--- a/tools/server/server-task.cpp
+++ b/tools/server/server-task.cpp
@@ -73,10 +73,10 @@ json task_params::to_json(bool only_metrics) const {
{"stream", stream},
{"n_probs", sampling.n_probs},
{"min_keep", sampling.min_keep},
- {"chat_format", common_chat_format_name(chat_parser_params.format)},
- {"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},
- {"reasoning_in_content", chat_parser_params.reasoning_in_content},
- {"generation_prompt", chat_parser_params.generation_prompt.text},
+ {"chat_format", common_chat_format_name(chat_format)},
+ {"reasoning_format", common_reasoning_format_name(reasoning_format)},
+ {"reasoning_in_content", stream && reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY},
+ {"generation_prompt", sampling.generation_prompt},
{"samplers", samplers},
{"speculative.types", common_speculative_type_name_str(speculative.types)},
{"timings_per_token", timings_per_token},
@@ -132,10 +132,10 @@ json task_params::to_json(bool only_metrics) const {
{"grammar_lazy", sampling.grammar_lazy},
{"grammar_triggers", grammar_triggers},
{"preserved_tokens", sampling.preserved_tokens},
- {"chat_format", common_chat_format_name(chat_parser_params.format)},
- {"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},
- {"reasoning_in_content", chat_parser_params.reasoning_in_content},
- {"generation_prompt", chat_parser_params.generation_prompt.text},
+ {"chat_format", common_chat_format_name(chat_format)},
+ {"reasoning_format", common_reasoning_format_name(reasoning_format)},
+ {"reasoning_in_content", stream && reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY},
+ {"generation_prompt", sampling.generation_prompt},
{"samplers", samplers},
{"speculative.types", common_speculative_type_name_str(speculative.types)},
{"timings_per_token", timings_per_token},
@@ -148,15 +148,12 @@ json task_params::to_json(bool only_metrics) const {
//
// task_result_state
//
-task_result_state::task_result_state(const common_chat_parser_params & chat_parser_params)
- : chat_parser_params(chat_parser_params)
+task_result_state::task_result_state(common_chat_session session)
+ : chat_session(std::move(session))
+ , chat_msg(chat_session.msg())
, oai_resp_id("resp_" + random_string())
, oai_resp_reasoning_id("rs_" + random_string())
, oai_resp_message_id("msg_" + random_string()) {
- if (chat_parser_params.is_continuation && !chat_parser_params.echo) {
- // initialize chat_msg to avoid emitting a delta containing the assistant prefill
- chat_msg = common_chat_parse(generated_input, true, chat_parser_params);
- }
}
common_chat_msg task_result_state::update_chat_msg(
@@ -164,13 +161,8 @@ common_chat_msg task_result_state::update_chat_msg(
bool is_partial,
std::vector<common_chat_msg_diff> & diffs,
bool filter_tool_calls) {
- generated_input.append(added);
auto msg_prv_copy = chat_msg;
- //SRV_DBG("Parsing chat message: %s\n", generated_input.text.c_str());
- auto new_msg = common_chat_parse(
- generated_input,
- is_partial,
- chat_parser_params);
+ auto new_msg = is_partial ? chat_session.feed(added) : chat_session.finish(added);
if (!new_msg.empty()) {
new_msg.set_tool_call_ids(generated_tool_call_ids, gen_tool_call_id);
chat_msg = new_msg;
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index 0cbc5f353..b8d6a3b67 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -89,8 +89,9 @@ struct task_params {
std::string control_action;
std::string control_cmpl_id;
- // per-request parameters for chat parsing
- common_chat_parser_params chat_parser_params;
+ // reported in generation_settings, parsing itself is owned by the chat session
+ common_chat_format chat_format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
+ common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE;
// message spans for checkpointing
common_chat_msg_spans message_spans;
@@ -106,9 +107,8 @@ struct task_params {
struct task_result_state {
// tracking diffs for partial tool calls
std::vector<common_chat_msg_diff> diffs;
- common_chat_parser_params chat_parser_params;
+ common_chat_session chat_session; // owns all parsing for this generation
common_chat_msg chat_msg;
- common_chat_input generated_input; // append new chunks of generated text here
std::vector<std::string> generated_tool_call_ids;
std::unordered_set<size_t> sent_tool_call_names;
@@ -124,7 +124,7 @@ struct task_result_state {
const std::string oai_resp_message_id;
std::string oai_resp_fc_id; // function call ID for current args delta
- task_result_state(const common_chat_parser_params & chat_parser_params);
+ task_result_state(common_chat_session session = {});
// parse partial tool calls and update the internal state
common_chat_msg update_chat_msg(
@@ -258,6 +258,17 @@ struct server_task {
return ids;
}
+ void apply_chat_session(const common_chat_session & session) {
+ if (!session.has_template()) {
+ return;
+ }
+
+ session.apply_sampling(params.sampling);
+ params.chat_format = session.format();
+ params.antiprompt.insert(params.antiprompt.end(), session.additional_stops().begin(), session.additional_stops().end());
+ params.message_spans = tokens.find_message_spans(session.message_delimiters());
+ }
+
void add_child(int id_parent, int id_child) {
server_task copy;
@@ -277,12 +288,6 @@ struct server_task {
child_tasks.push_back(std::move(copy));
}
- // the task will be moved into queue, then onto slots
- // however, the state must be kept by caller (e.g., HTTP thread)
- task_result_state create_state() const {
- return task_result_state(params.chat_parser_params);
- }
-
bool is_parent() const {
return child_tasks.size() > 0;
}