Commit dd099a6bf for llama.cpp
commit dd099a6bf30c50627f6ae18915fbb687539a704d
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Sun Nov 15 09:41:44 2026 +0100
chat: fix inkling 2 begin/end of image tokens placement (#32294)
* chat: fix inkling 2 begin/end of image tokens placement
* add test
diff --git a/common/chat.cpp b/common/chat.cpp
index 997c135d9..e060362d2 100644
--- a/common/chat.cpp
+++ b/common/chat.cpp
@@ -1609,6 +1609,9 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
auto_params.thinking_end_tags = {std::move(end_tag)};
}
}
+ if (is_inkling2_template(src)) {
+ common_chat_inkling2_split_media(auto_params.prompt);
+ }
const auto & arena = auto_params.parser;
LOG_DBG("%s: generated parser:\n%s\n\nparser generation prompt: %s\n", __func__, arena.dump(arena.root()).c_str(), auto_params.generation_prompt.c_str());
return auto_params;
diff --git a/common/parsers/inkling2.cpp b/common/parsers/inkling2.cpp
new file mode 100644
index 000000000..4e2b01c63
--- /dev/null
+++ b/common/parsers/inkling2.cpp
@@ -0,0 +1,46 @@
+#include "parsers.h"
+
+bool is_inkling2_template(const std::string & src) {
+ return src.find("<|message_user|>") != std::string::npos &&
+ src.find("<|content_image|>") != std::string::npos;
+}
+
+// the template makes one message block per content part, but a media marker comes inside the text of a message
+// <header><|content_text|>{media}text<|end_message|> --> <header>{media}<|end_message|><header><|content_text|>text<|end_message|>
+void common_chat_inkling2_split_media(common_chat_prompt & prompt) {
+ const std::string TEXT = "<|content_text|>";
+ const std::string END = "<|end_message|>";
+ const std::string MEDIA_BEG = "<__media_";
+ const std::string MEDIA_END = "__>";
+
+ std::vector<common_chat_prompt_part> out;
+ for (size_t i = 0; i < prompt.parts.size(); i++) {
+ const auto & part = prompt.parts[i];
+ const bool in_text_block = part.is_input && part.text.find(MEDIA_BEG) != std::string::npos &&
+ !out.empty() && !out.back().is_input && string_ends_with(out.back().text, TEXT) &&
+ i + 1 < prompt.parts.size() && !prompt.parts[i + 1].is_input && string_starts_with(prompt.parts[i + 1].text, END);
+ const size_t header_pos = in_text_block ? out.back().text.rfind("<|message_") : std::string::npos;
+ if (header_pos == std::string::npos) {
+ out.push_back(part);
+ continue;
+ }
+ const std::string header = out.back().text.substr(header_pos, out.back().text.size() - TEXT.size() - header_pos);
+
+ std::string rest = part.text;
+ for (bool first = true; !rest.empty(); first = false) {
+ const size_t beg = rest.find(MEDIA_BEG);
+ const size_t end = beg == std::string::npos ? beg : rest.find(MEDIA_END, beg);
+ const bool is_media = beg == 0 && end != std::string::npos;
+ const size_t len = is_media ? end + MEDIA_END.size() : (end == std::string::npos ? rest.size() : beg);
+ if (!first) {
+ out.push_back({ END + header + (is_media ? "" : TEXT), false });
+ } else if (is_media) {
+ // mtmd opens the media block
+ out.back().text.resize(out.back().text.size() - TEXT.size());
+ }
+ out.push_back({ rest.substr(0, len), true });
+ rest.erase(0, len);
+ }
+ }
+ prompt.parts = std::move(out);
+}
diff --git a/common/parsers/parsers.h b/common/parsers/parsers.h
index b1b6d166a..35dc04ac3 100644
--- a/common/parsers/parsers.h
+++ b/common/parsers/parsers.h
@@ -41,6 +41,11 @@ std::string common_chat_template_generation_prompt_impl(
bool is_lfm2_template(const std::string & src);
+bool is_inkling2_template(const std::string & src);
+
+// give each media marker of a text block its own message block
+void common_chat_inkling2_split_media(common_chat_prompt & prompt);
+
common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
diff --git a/common/parsers/sources.cmake b/common/parsers/sources.cmake
index 2174f4519..97dd490e8 100644
--- a/common/parsers/sources.cmake
+++ b/common/parsers/sources.cmake
@@ -10,6 +10,7 @@ set(LLAMA_CHAT_PARSERS_SOURCES
${CMAKE_CURRENT_LIST_DIR}/gigachat-v3.cpp
${CMAKE_CURRENT_LIST_DIR}/gpt-oss.cpp
${CMAKE_CURRENT_LIST_DIR}/harmony-xml.cpp
+ ${CMAKE_CURRENT_LIST_DIR}/inkling2.cpp
${CMAKE_CURRENT_LIST_DIR}/k2-horizon.cpp
${CMAKE_CURRENT_LIST_DIR}/kimi-k2.cpp
${CMAKE_CURRENT_LIST_DIR}/kimi-k3.cpp
diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp
index 53a1b6d1a..b34de6562 100644
--- a/tests/test-chat.cpp
+++ b/tests/test-chat.cpp
@@ -166,6 +166,18 @@ static const char * DEEPSEEK_V4_1_MIN_TEMPLATE = R"({%- for m in messages -%}
{%- if reasoning_effort is defined and reasoning_effort is not none -%}<｜effort={{ reasoning_effort }}｜>{%- endif -%}
{%- if add_generation_prompt -%}<｜Assistant｜>{%- if enable_thinking -%}<think>{%- else -%}</think>{%- endif -%}{%- endif -%})";
+// Inkling 2 stand-in: the real template ships in the GGUF; this keeps only the message blocks,
+// one per content part, and the tokens used for detection
+static const char * INKLING2_MIN_TEMPLATE = R"({%- set role_token = {"user": "<|message_user|>", "assistant": "<|message_model|>", "system": "<|message_system|>"} -%}
+{%- for m in messages -%}
+{%- if m.content is string -%}{{ role_token[m.role] }}<|content_text|>{{ m.content }}<|end_message|>
+{%- else -%}{%- for p in m.content -%}{{ role_token[m.role] }}
+{%- if p.type == 'image' -%}<|content_image|>{%- else -%}<|content_text|>{{ p.text }}{%- endif -%}
+<|end_message|>{%- endfor -%}
+{%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}<|message_model|>{%- endif -%})";
+
static std::unique_ptr<llama_grammar> build_grammar(const std::string & grammar_str) {
return std::unique_ptr<llama_grammar>(
llama_grammar_init_impl(nullptr, grammar_str.c_str(), "root", false, nullptr, 0, nullptr, 0));
@@ -7737,6 +7749,37 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
}
}
+// Inkling 2: each media marker moves out of the text block into its own message block
+static void test_inkling2_media_blocks() {
+ LOG_DBG("%s\n", __func__);
+
+ // must be the marker used by the server, the split depends on its format
+ const std::string M = get_media_marker();
+ const std::string U = "<|message_user|>";
+ const std::string T = "<|content_text|>";
+ const std::string E = "<|end_message|>";
+
+ common_chat_templates_ptr tmpls(common_chat_templates_init(nullptr, INKLING2_MIN_TEMPLATE));
+ auto render = [&](const std::vector<std::string> & parts) {
+ common_chat_msg msg;
+ msg.role = "user";
+ for (const auto & text : parts) {
+ msg.content_parts.push_back({ /* .type = */ text == M ? "media_marker" : "text", /* .text = */ text, /* .name = */ "" });
+ }
+ common_chat_templates_inputs inputs;
+ inputs.messages = { msg };
+ inputs.add_generation_prompt = false;
+ return common_chat_templates_apply(tmpls.get(), inputs).prompt.str();
+ };
+
+ assert_equals(U + M + E + U + T + "a" + E, render({ M, "a" }));
+ assert_equals(U + T + "a" + E + U + M + E, render({ "a", M }));
+ assert_equals(U + M + E, render({ M }));
+ assert_equals(U + M + E + U + M + E + U + T + "a" + E, render({ M, M, "a" }));
+ assert_equals(U + T + "a" + E + U + M + E + U + T + "b" + E, render({ "a", M, "b" }));
+ assert_equals(U + T + "a" + E, render({ "a" }));
+}
+
static void test_template_generation_prompt() {
common_chat_msg system_msg;
system_msg.role = "system";
@@ -8733,6 +8776,7 @@ int main(int argc, char ** argv) {
test_deepseek_v4_thinking_retention();
test_deepseek_v4_tool_result_ordering();
test_deepseek_v4_1_message_fields();
+ test_inkling2_media_blocks();
test_template_generation_prompt();
test_reasoning_effort_caps();
test_prompt_parts();
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 929f34620..63a8e3954 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -907,9 +907,8 @@ struct mtmd_context {
} break;
case PROJECTOR_TYPE_INKLING2V:
{
- // image goes in its own message block (always user role)
- img_beg = "<|end_message|><|message_user|><|content_image|>";
- img_end = "<|end_message|><|message_user|><|content_text|>";
+ // the chat template code gives the image its own message block
+ img_beg = "<|content_image|>";
image_preproc = std::make_unique<mtmd_image_preprocessor_inkling2v>(models.v.get());
} break;
case PROJECTOR_TYPE_MUSE_GLIMMER: