Commit c6fb3b5be for llama.cpp
commit c6fb3b5be2209f6c12339833198190f6f6829581
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Mon Oct 26 12:45:35 2026 +0100
mtmd: add mtmd_helper_get_capabilities (#31183)
* mtmd: add mtmd_helper_get_capabilities
* cap audio_stream
* add back old api for compat
* audio_gen_spk_ref, text_gen
diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp
index 1d7d65ef8..faabd4840 100644
--- a/tools/mtmd/mtmd-cli.cpp
+++ b/tools/mtmd/mtmd-cli.cpp
@@ -75,6 +75,7 @@ static void inject_test_response_marker() {
struct mtmd_cli_context {
mtmd::context_ptr ctx_vision;
+ mtmd_helper_cap mtmd_cap = {};
common_init_result_ptr llama_init;
llama_model * model;
@@ -120,7 +121,8 @@ struct mtmd_cli_context {
init_vision_context(params);
- if (!mtmd_helper_model_can_chat(lctx, ctx_vision.get())) {
+ mtmd_cap = mtmd_helper_get_capabilities(model, ctx_vision.get());
+ if (!mtmd_cap.chat) {
LOG_ERR("Model does not support chat mode\n");
LOG_ERR("Hint: for TTS models, please use llama-tts\n");
exit(1);
@@ -565,7 +567,7 @@ int main(int argc, char ** argv) {
if (g_is_interrupted) return 130;
- if (mtmd_support_audio_stream(ctx.ctx_vision.get())) {
+ if (ctx.mtmd_cap.audio_stream) {
if (params.image.size() != 1) {
LOG_ERR("This model transcribes audio in realtime, pass exactly one --audio file\n");
return 1;
@@ -624,7 +626,7 @@ int main(int argc, char ** argv) {
if (mtmd_support_audio(ctx.ctx_vision.get())) {
LOG("\n /audio <path> load an audio");
}
- if (mtmd_helper_support_video(ctx.ctx_vision.get())) {
+ if (ctx.mtmd_cap.video) {
LOG("\n /video <path> load a video");
}
LOG("\n /clear clear the chat history");
diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp
index eba034f79..d06af8f02 100644
--- a/tools/mtmd/mtmd-helper.cpp
+++ b/tools/mtmd/mtmd-helper.cpp
@@ -501,15 +501,6 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(const mtmd_context
return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size(), placeholder, opt);
}
-bool mtmd_helper_support_video(const mtmd_context * ctx) {
-#ifdef MTMD_VIDEO
- return mtmd_support_vision(ctx);
-#else
- GGML_UNUSED(ctx);
- return false;
-#endif
-}
-
//
// Video input helpers
//
@@ -1033,17 +1024,40 @@ int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx,
#endif
}
-bool mtmd_helper_model_can_chat(const llama_context * lctx, const mtmd_context * mctx) {
+mtmd_helper_cap mtmd_helper_get_capabilities(const llama_model * model, const mtmd_context * mctx) {
+ mtmd_helper_cap cap = {};
+ cap.chat = true;
+ cap.text_gen = true;
if (!mctx) {
- return true;
+ return cap;
}
- auto * model = llama_get_model(lctx);
- auto * tmpl = llama_model_chat_template(model, nullptr);
- auto info = mtmd_gen_audio_get_info(mctx);
+ cap.vision = mtmd_support_vision(mctx);
+ cap.audio = mtmd_support_audio(mctx);
+ cap.audio_stream = mtmd_support_audio_stream(mctx);
+#ifdef MTMD_VIDEO
+ cap.video = cap.vision;
+#endif
+ const auto gen_audio_type = mtmd_gen_audio_get_info(mctx).type;
+ cap.audio_gen = gen_audio_type != MTMD_GEN_AUDIO_TYPE_NONE;
+ cap.audio_gen_spk_ref = gen_audio_type == MTMD_GEN_AUDIO_TYPE_QWEN3TTS
+ || gen_audio_type == MTMD_GEN_AUDIO_TYPE_POCKETTTS;
// tts-only model cannot be used for chat (no chat template)
- bool is_tts_only = info.type != MTMD_GEN_AUDIO_TYPE_NONE && tmpl == nullptr;
+ cap.chat = !cap.audio_gen || llama_model_chat_template(model, nullptr) != nullptr;
- return !is_tts_only;
+ return cap;
+}
+
+bool mtmd_helper_model_can_chat(const llama_context * lctx, const mtmd_context * mctx) {
+ return mtmd_helper_get_capabilities(llama_get_model(lctx), mctx).chat;
+}
+
+bool mtmd_helper_support_video(const mtmd_context * ctx) {
+#ifdef MTMD_VIDEO
+ return mtmd_support_vision(ctx);
+#else
+ GGML_UNUSED(ctx);
+ return false;
+#endif
}
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index acbfefcba..e46b5d36a 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -45,9 +45,6 @@ MTMD_API struct mtmd_helper_init_opt mtmd_helper_init_opt_default(void);
// Note: this also call mtmd_log_set() internally
MTMD_API void mtmd_helper_log_set(ggml_log_callback log_callback, void * user_data);
-// Returns true if this build includes video support (MTMD_VIDEO was ON at compile time).
-MTMD_API bool mtmd_helper_support_video(const mtmd_context * ctx);
-
struct mtmd_helper_bitmap_wrapper {
mtmd_bitmap * bitmap;
mtmd_helper_video * video_ctx;
@@ -186,8 +183,31 @@ MTMD_API int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx,
mtmd_bitmap ** out_bitmap,
char ** out_text);
-// return true if model can be used for chat
-MTMD_API bool mtmd_helper_model_can_chat(const struct llama_context * lctx, const struct mtmd_context * mctx);
+struct mtmd_helper_cap {
+ bool chat; // false for tts-only model (no chat template)
+
+ // input capabilities
+ bool vision;
+ bool audio;
+ bool video; // also depending on MTMD_VIDEO build option
+
+ // realtime capabilities
+ bool audio_stream; // realtime transcription, see mtmd_audio_stream_init()
+
+ // output capabilities
+ bool text_gen; // always true for now
+ bool audio_gen;
+ bool audio_gen_spk_ref; // accepts a speaker reference audio for voice cloning
+};
+typedef struct mtmd_helper_cap mtmd_helper_cap;
+
+// mctx can be NULL (text-only model)
+MTMD_API struct mtmd_helper_cap mtmd_helper_get_capabilities(const struct llama_model * model, const struct mtmd_context * mctx);
+
+DEPRECATED(MTMD_API bool mtmd_helper_model_can_chat(const struct llama_context * lctx, const struct mtmd_context * mctx),
+ "use mtmd_helper_get_capabilities() instead");
+DEPRECATED(MTMD_API bool mtmd_helper_support_video(const struct mtmd_context * ctx),
+ "use mtmd_helper_get_capabilities() instead");
//
// Audio generation helpers
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index a5cba1d71..9043fa2c0 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -1012,16 +1012,17 @@ public:
llama_model * model_tgt = nullptr;
mtmd_context * mctx = nullptr;
+ mtmd_helper_cap mtmd_cap = {};
// note: video_params.ffmpeg_bin_dir points into params_base, which outlives this struct
mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
const llama_vocab * vocab = nullptr;
bool has_cap_tts() const {
- return mctx != nullptr && mtmd_gen_audio_get_info(mctx).type != MTMD_GEN_AUDIO_TYPE_NONE;
+ return mtmd_cap.audio_gen;
}
bool has_cap_chat() const {
- return mctx == nullptr || mtmd_helper_model_can_chat(ctx_tgt, mctx);
+ return mtmd_cap.chat;
}
server_decision_context decision;
@@ -1397,6 +1398,8 @@ private:
}
}
+ mtmd_cap = mtmd_helper_get_capabilities(model_tgt, mctx);
+
if (!llama_memory_can_shift(llama_get_memory(ctx_tgt))) {
if (params_base.ctx_shift) {
params_base.ctx_shift = false;
@@ -1705,9 +1708,9 @@ private:
/* reasoning_format */ params_base.reasoning_format,
/* chat_template_kwargs */ params_base.default_template_kwargs,
/* tmpls */ std::move(chat_templates),
- /* allow_image */ mctx ? mtmd_support_vision(mctx) : false,
- /* allow_audio */ mctx ? mtmd_support_audio (mctx) : false,
- /* allow_video */ mctx ? mtmd_helper_support_video(mctx) : false,
+ /* allow_image */ mtmd_cap.vision,
+ /* allow_audio */ mtmd_cap.audio,
+ /* allow_video */ mtmd_cap.video,
/* enable_thinking */ enable_thinking,
/* reasoning_budget */ params_base.sampling.reasoning_budget_tokens,
/* reasoning_budget_msg */ params_base.sampling.reasoning_budget_message,
@@ -5127,7 +5130,7 @@ server_context_meta server_context::get_meta() const {
/* has_inp_video */ impl->chat_params.allow_video,
/* has_cap_chat */ impl->has_cap_chat(),
/* has_cap_tts */ impl->has_cap_tts(),
- /* has_audio_stream */ impl->mctx != nullptr && mtmd_support_audio_stream(impl->mctx),
+ /* has_audio_stream */ impl->mtmd_cap.audio_stream,
/* json_ui_settings */ impl->json_ui_settings,
/* slot_n_ctx */ impl->n_ctx_slot(),
/* pooling_type */ llama_pooling_type(impl->ctx_tgt),
diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
index e1e006f10..575140132 100644
--- a/tools/server/server-models.cpp
+++ b/tools/server/server-models.cpp
@@ -533,7 +533,7 @@ void server_model_meta::update_caps(const common_params & base) {
std::vector<std::string> output_modalities = {"text"};
if (!params.model.path.empty()) {
const common_gguf_info info = common_get_gguf_info(params.model.path);
- // same rule as mtmd_helper_model_can_chat()
+ // same rule as mtmd_helper_get_capabilities()
const bool can_chat = !gen_audio || info.has_chat_tmpl;
output_modalities = server_model_output_modalities(info.decision_type, can_chat, gen_audio);
n_ctx_train = info.n_ctx_train;