Commit 149f5641b for llama.cpp
commit 149f5641b931aaf33697d70f9abc90c3e5300247
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Tue Nov 3 23:57:26 2026 +0100
mtmd: accurate support for qwen3-vl video prompt format (#31619)
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp
index 536dfaa80..b837f45da 100644
--- a/tests/test-mtmd-impl.cpp
+++ b/tests/test-mtmd-impl.cpp
@@ -1,11 +1,13 @@
#include "testing.h"
#include "mtmd-audio.h"
+#include "mtmd-helper.h"
#include "mtmd-image.h"
#include "mtmd-internal.h"
#include <cmath>
#include <cstdint>
+#include <cstdlib>
#include <iostream>
#include <stdexcept>
#include <string>
@@ -92,6 +94,33 @@ MAKE_TEST(test_image_preprocessor_dyn_size) {
}
}
+MAKE_TEST(test_image_preprocessor_qwen3vl_video) {
+ clip_hparams hparams;
+ hparams.patch_size = 16;
+ hparams.n_merge = 2;
+ const mtmd_image_preprocessor_qwen3vl preproc(hparams);
+
+ // { n_frames, frame size, expected size from Qwen3VLVideoProcessor smart_resize }
+ const std::vector<std::tuple<int, clip_image_size, clip_image_size>> cases = {
+ { 20, { 720, 358 }, { 704, 352 } }, // 720 / 32 = 22.5 rounds half to even
+ { 768, { 1920, 1080 }, { 224, 128 } }, // over the longest_edge budget
+ { 4, { 16, 40 }, { 32, 64 } }, // edge below the factor is upscaled first
+ { 2, { 32, 32 }, { 64, 64 } }, // under the shortest_edge budget
+ { 5, { 2760, 1552 }, { 2752, 1536 } }, // 5 / 2 = 2.5 rounds half to even
+ { 4, { 250, 6000 }, { 256, 6016 } },
+ };
+
+ for (const auto & [n_frames, input, expected] : cases) {
+ clip_image_size actual;
+ const std::string name = std::to_string(n_frames) + " frames of " + std::to_string(input.width) + "x" + std::to_string(input.height);
+ if (!t.assert_true(name + " has a frame size", preproc.get_video_frame_size(n_frames, input.width, input.height, actual))) {
+ continue;
+ }
+ t.assert_equal(name + " width", expected.width, actual.width);
+ t.assert_equal(name + " height", expected.height, actual.height);
+ }
+}
+
static std::string image_size_to_str(const clip_image_size & size) {
return std::to_string(size.width) + "x" + std::to_string(size.height);
}
@@ -301,6 +330,130 @@ MAKE_TEST(test_temporal_merge_grouping) {
}
}
+//
+// mtmd_helper video
+//
+
+MAKE_TEST(test_video_config) {
+ clip_hparams hparams;
+ hparams.patch_size = 16;
+ hparams.n_merge = 2;
+ const mtmd_image_preprocessor_qwen3vl preproc(hparams);
+
+ // { template, n_frames and fps of the source, expected fps, n_frames and frame size }
+ // the 1920x1080 sizes are from Qwen3VLVideoProcessor smart_resize
+ const std::vector<std::tuple<mtmd_video_template, int32_t, float, float, int32_t, clip_image_size>> cases = {
+ { MTMD_VIDEO_TEMPLATE_QWEN3VL, 36000, 30.0f, 2.0f, 768, { 224, 128 } },
+ { MTMD_VIDEO_TEMPLATE_LFM3, 36000, 30.0f, 2.0f, 768, { 224, 128 } },
+ { MTMD_VIDEO_TEMPLATE_LFM3_OMNI, 36000, 30.0f, 2.0f, 384, { 320, 192 } },
+ { MTMD_VIDEO_TEMPLATE_QWEN3VL, -1, 30.0f, 2.0f, 0, { 0, 0 } }, // frame count unknown
+ { MTMD_VIDEO_TEMPLATE_DEFAULT, 36000, 30.0f, 0.0f, 0, { 0, 0 } },
+ };
+
+ for (const auto & [tmpl, n_frames, fps, expected_fps, expected_n_frames, expected_size] : cases) {
+ const mtmd_video_config cfg = mtmd_video_config_init(tmpl, &preproc, 1920, 1080, n_frames, fps);
+ const std::string name = "template " + std::to_string(tmpl) + " with " + std::to_string(n_frames) + " frames";
+ t.assert_equal(name + " template", (int) tmpl, (int) cfg.tmpl);
+ t.assert_equal(name + " fps", expected_fps, cfg.fps);
+ t.assert_equal(name + " n_frames", expected_n_frames, cfg.n_frames);
+ t.assert_equal(name + " frame size", image_size_to_str(expected_size), image_size_to_str({ cfg.frame_width, cfg.frame_height }));
+ }
+}
+
+// n_frames source frames, each one starts with its index
+struct test_video_source : mtmd_helper_video_seq {
+ int32_t n_frames;
+ int32_t n_read = 0;
+
+ test_video_source(int32_t n_frames) : n_frames(n_frames) {}
+
+ bool read_raw_frame() override {
+ if (n_read >= n_frames) {
+ return false;
+ }
+ frame_buf[0] = static_cast<uint8_t>(n_read & 255);
+ frame_buf[1] = static_cast<uint8_t>(n_read >> 8);
+ n_read++;
+ return true;
+ }
+};
+
+// text as is, a frame as [source index], an audio bitmap as {n_samples}
+static std::string video_sequence_to_str(testing & t, mtmd_helper_video_seq & video) {
+ std::string out;
+ mtmd_bitmap * bitmap;
+ char * text;
+ int32_t ret;
+ while ((ret = video.read_next(&bitmap, &text)) == 0) {
+ if (text) {
+ out += text;
+ free(text);
+ } else if (mtmd_bitmap_is_audio(bitmap)) {
+ out += "{" + std::to_string(mtmd_bitmap_get_n_bytes(bitmap) / sizeof(float)) + "}";
+ mtmd_bitmap_free(bitmap);
+ } else {
+ const unsigned char * data = mtmd_bitmap_get_data(bitmap);
+ out += "[" + std::to_string(data[0] | data[1] << 8) + "]";
+ mtmd_bitmap_free(bitmap);
+ }
+ }
+ t.assert_equal("end of stream", -1, ret);
+ return out;
+}
+
+MAKE_TEST(test_video_sequence_qwen3vl) {
+ clip_hparams hparams;
+ hparams.patch_size = 16;
+ hparams.n_merge = 2;
+ const mtmd_image_preprocessor_qwen3vl preproc(hparams);
+
+ // { source video, expected frame size and sequence from Qwen3VLProcessor (transformers 4.57.1) }
+ const std::vector<std::tuple<mtmd_helper_video_src, clip_image_size, std::string>> cases = {
+ { { 720, 358, 24.0f, 240 }, { 704, 352 },
+ "<0.3 seconds>[0][13]<1.3 seconds>[25][38]<2.4 seconds>[50][63]<3.4 seconds>[75][88]<4.5 seconds>[101][113]"
+ "<5.5 seconds>[126][138]<6.6 seconds>[151][164]<7.6 seconds>[176][189]<8.6 seconds>[201][214]<9.7 seconds>[226][239]" },
+ // odd frame count: the last group repeats its last frame
+ { { 640, 480, 10.0f, 35 }, { 640, 480 },
+ "<0.3 seconds>[0][6]<1.4 seconds>[11][17]<2.5 seconds>[23][28]<3.4 seconds>[34]" },
+ // fewer frames than the minimum of 4
+ { { 320, 240, 30.0f, 3 }, { 320, 256 },
+ "<0.0 seconds>[0][1]<0.1 seconds>[2]" },
+ { { 1280, 720, 30000.0f / 1001.0f, 300 }, { 1280, 704 },
+ "<0.3 seconds>[0][16]<1.3 seconds>[31][47]<2.4 seconds>[63][79]<3.4 seconds>[94][110]<4.5 seconds>[126][142]"
+ "<5.5 seconds>[157][173]<6.6 seconds>[189][205]<7.6 seconds>[220][236]<8.7 seconds>[252][268]<9.7 seconds>[283][299]" },
+ };
+
+ for (const auto & [src, expected_size, expected] : cases) {
+ const std::string name = std::to_string(src.n_frames) + " frames of " + std::to_string(src.width) + "x" + std::to_string(src.height);
+ const mtmd_video_config cfg = mtmd_video_config_init(MTMD_VIDEO_TEMPLATE_QWEN3VL, &preproc, src.width, src.height, src.n_frames, src.fps);
+ test_video_source video(src.n_frames);
+ video.configure(cfg, src, 0.0f);
+ video.src_alive = true;
+ t.assert_equal(name + " frame size", image_size_to_str(expected_size), image_size_to_str({ (int) video.info.width, (int) video.info.height }));
+ t.assert_equal(name + " sequence", expected, video_sequence_to_str(t, video));
+ }
+}
+
+// the soundtrack of each group comes after its frames
+MAKE_TEST(test_video_sequence_lfm3_omni) {
+ clip_hparams hparams;
+ hparams.patch_size = 16;
+ hparams.n_merge = 2;
+ const mtmd_image_preprocessor_lfm3vl preproc(hparams);
+
+ const mtmd_helper_video_src src = { 64, 64, 2.0f, 9 };
+ const mtmd_video_config cfg = mtmd_video_config_init(MTMD_VIDEO_TEMPLATE_LFM3_OMNI, &preproc, src.width, src.height, src.n_frames, src.fps);
+ test_video_source video(src.n_frames);
+ video.configure(cfg, src, 0.0f);
+ video.src_alive = true;
+ video.audio_rate = 100;
+ video.soundtrack.assign(450, 0.0f);
+
+ t.assert_equal("sequence",
+ std::string("<0.2 seconds>[0][1]{100}<1.2 seconds>[2][3]{100}<2.2 seconds>[4][5]{100}<3.2 seconds>[6][7]{100}<4.0 seconds>[8]{50}"),
+ video_sequence_to_str(t, video));
+}
+
//
// mtmd_audio
//
@@ -357,6 +510,13 @@ int main(int argc, char ** argv) {
testing t(std::cout);
t.verbose = true;
+ // the video helper logs every frame at debug level
+ mtmd_helper_log_set([](ggml_log_level level, const char * text, void *) {
+ if (level != GGML_LOG_LEVEL_DEBUG) {
+ fputs(text, stderr);
+ }
+ }, nullptr);
+
// usage: test-mtmd-impl [filter_regex]
for (int i = 1; i < argc; i++) {
t.set_filter(argv[i]);
diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp
index 29812219c..d86537e55 100644
--- a/tools/mtmd/mtmd-helper.cpp
+++ b/tools/mtmd/mtmd-helper.cpp
@@ -508,8 +508,6 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(const mtmd_context
// Video input helpers
//
-#ifdef MTMD_VIDEO
-
// MSVC deprecates strdup(), use a std::string version directly
static char * mtmd_strdup(const std::string & s) {
size_t n = s.size() + 1;
@@ -520,15 +518,218 @@ static char * mtmd_strdup(const std::string & s) {
return p;
}
-struct mtmd_helper_video {
+mtmd_helper_video_seq::~mtmd_helper_video_seq() {
+ mtmd_bitmap_free(pending_frame);
+ mtmd_bitmap_free(pending_audio);
+}
+
+int32_t mtmd_helper_video_seq::ts_per_n_frames() const {
+ return tmpl == MTMD_VIDEO_TEMPLATE_LFM3 || tmpl == MTMD_VIDEO_TEMPLATE_LFM3_OMNI || tmpl == MTMD_VIDEO_TEMPLATE_QWEN3VL ? 2 : 0;
+}
+
+void mtmd_helper_video_seq::configure(const mtmd_video_config & cfg, const mtmd_helper_video_src & src, float fps_target_arg) {
+ tmpl = cfg.tmpl;
+ uint32_t width = src.width;
+ uint32_t height = src.height;
+ float duration = src.duration;
+ if (duration < 0.0f && src.n_frames > 0) {
+ duration = (float)src.n_frames / src.fps;
+ }
+
+ // the model's recommended sampling rate wins over the generic default
+ fps_target = cfg.fps > 0.0f ? cfg.fps
+ : fps_target_arg > 0.0f ? fps_target_arg : src.fps;
+
+ if (cfg.n_frames > 0 && duration > 0.0f) {
+ const int32_t n = cfg.n_frames;
+ target_frames = n;
+ fps_target = (float) n / duration;
+
+ // same frames as the reference processors: round(linspace(0, n_frames - 1, n))
+ const double step = n > 1 ? (double) (src.n_frames - 1) / (n - 1) : 0.0;
+ frame_indices.resize(n);
+ for (int32_t i = 0; i < n; i++) {
+ frame_indices[i] = (int32_t) std::nearbyint(i * step);
+ }
+ src_fps = src.fps;
+ n_src_frames = src.n_frames;
+
+ if (cfg.frame_width > 0 && cfg.frame_height > 0 &&
+ (cfg.frame_width != (int32_t) width || cfg.frame_height != (int32_t) height)) {
+ out_width = cfg.frame_width;
+ out_height = cfg.frame_height;
+ }
+ }
+
+ if (out_width > 0 && out_height > 0) {
+ width = (uint32_t) out_width;
+ height = (uint32_t) out_height;
+ }
+
+ info.width = width;
+ info.height = height;
+ info.fps = fps_target;
+ LOG_DBG("%s: %ux%u fps=%.2f duration=%.2fs n_frames=%d target_frames=%d\n",
+ __func__, width, height, fps_target, duration, src.n_frames, target_frames);
+ info.n_frames = target_frames > 0 ? target_frames
+ : duration > 0.0f ? (int32_t)(duration * fps_target + 0.5f) : -1;
+ frame_buf.resize((size_t)width * height * 3);
+}
+
+// the soundtrack of the group of frames that ends with frame idx, nullptr if the soundtrack has ended
+mtmd_bitmap * mtmd_helper_video_seq::soundtrack_slice(int32_t idx) const {
+ const int32_t ts_per_n_frames = this->ts_per_n_frames();
+ const int32_t group = idx / ts_per_n_frames;
+ const int32_t n_groups = ((int32_t) frame_indices.size() + ts_per_n_frames - 1) / ts_per_n_frames;
+ auto group_start = [&](int32_t g) {
+ const double t = frame_indices[g * ts_per_n_frames] / src_fps;
+ return std::min(soundtrack.size(), (size_t) std::nearbyint(t * audio_rate));
+ };
+ const size_t begin = group == 0 ? 0 : group_start(group);
+ const size_t end = group == n_groups - 1 ? soundtrack.size() : group_start(group + 1);
+ if (begin >= end) {
+ return nullptr;
+ }
+ mtmd_bitmap * slice = mtmd_bitmap_init_from_audio(end - begin, soundtrack.data() + begin);
+ if (!id.empty()) {
+ mtmd_bitmap_set_id(slice, (id + "+audio" + std::to_string(group)).c_str());
+ }
+ return slice;
+}
+
+mtmd_bitmap * mtmd_helper_video_seq::read_next_frame() {
+ if (!src_alive) return nullptr;
+
+ if (!frame_indices.empty()) {
+ if (next_index >= frame_indices.size()) {
+ return nullptr;
+ }
+ // skip to the wanted source frame; a repeated index reuses the frame in frame_buf
+ const int32_t want = frame_indices[next_index];
+ while (src_frame <= want) {
+ if (!read_raw_frame()) {
+ src_alive = false;
+ return nullptr;
+ }
+ src_frame++;
+ }
+ next_index++;
+ } else if (!read_raw_frame()) {
+ src_alive = false;
+ return nullptr;
+ }
+
+ LOG_DBG("%s: frame %d read OK\n", __func__, current_frame);
+ mtmd_bitmap * frame = mtmd_bitmap_init(info.width, info.height, frame_buf.data());
+ mtmd_bitmap_set_mergeable(frame, true);
+ mtmd_bitmap_set_is_video(frame, true);
+ if (!id.empty()) {
+ // each frame gets a unique id in the form of {hash}+{frame}, so that it can be identified in cache
+ std::string frame_id = id + "+" + std::to_string(current_frame);
+ mtmd_bitmap_set_id(frame, frame_id.c_str());
+ }
+ current_frame++;
+ return frame;
+}
+
+int32_t mtmd_helper_video_seq::read_next(mtmd_bitmap ** out_bitmap, char ** out_text) {
+ *out_bitmap = nullptr;
+ *out_text = nullptr;
+
+ if (!pending_text.empty()) {
+ *out_text = mtmd_strdup(pending_text);
+ pending_text.clear();
+ return *out_text ? 0 : -2;
+ }
+
+ if (pending_frame) {
+ *out_bitmap = pending_frame;
+ pending_frame = nullptr;
+ return 0;
+ }
+
+ if (pending_audio) {
+ *out_bitmap = pending_audio;
+ pending_audio = nullptr;
+ return 0;
+ }
+
+ LOG_DBG("%s: proc_alive=%d start_emitted=%d current_frame=%d\n",
+ __func__, (int)src_alive, (int)start_emitted, current_frame);
+
+ if (!src_alive) {
+ return (current_frame == 0) ? -2 : -1;
+ }
+
+ if (!start_emitted) {
+ start_emitted = true;
+ // model templates have no prompt prefix, the video begin/end tokens are added by mtmd_tokenize
+ if (tmpl == MTMD_VIDEO_TEMPLATE_DEFAULT && !prompt_start.empty()) {
+ *out_text = mtmd_strdup(prompt_start);
+ return *out_text ? 0 : -2;
+ }
+ }
+
+ mtmd_bitmap * frame = read_next_frame();
+ if (!frame) {
+ return -1;
+ }
+
+ const int32_t ts_per_n_frames = this->ts_per_n_frames();
+ if (ts_per_n_frames > 0) {
+ // emit the group timestamp before the group's first frame; the value is the
+ // midpoint time of the group, matching HF processors that stamp each temporal group
+ const int32_t idx = current_frame - 1; // read_next_frame() already incremented
+ // the soundtrack of a group follows its last frame
+ const bool group_ends = idx % ts_per_n_frames == ts_per_n_frames - 1 || idx + 1 == (int32_t) frame_indices.size();
+ if (!soundtrack.empty() && !frame_indices.empty() && group_ends) {
+ pending_audio = soundtrack_slice(idx);
+ }
+ if (idx % ts_per_n_frames == 0) {
+ char ts_buf[64];
+ double ts_s = ((double)idx + 0.5*(double)(ts_per_n_frames - 1)) / info.fps;
+ if (!frame_indices.empty()) {
+ // stamp from the source frames; a short last group repeats its last frame
+ const size_t first = idx;
+ const size_t last = std::min(first + ts_per_n_frames - 1, frame_indices.size() - 1);
+ ts_s = (frame_indices[first] / src_fps + frame_indices[last] / src_fps) / 2.0;
+ }
+ snprintf(ts_buf, sizeof(ts_buf), "<%.1f seconds>", ts_s);
+ pending_frame = frame;
+ *out_text = mtmd_strdup(ts_buf);
+ return *out_text ? 0 : -2;
+ }
+ *out_bitmap = frame;
+ return 0;
+ }
+
+ *out_bitmap = frame;
+
+ if (timestamp_interval_ms > 0) {
+ // current_frame was already incremented by read_next_frame(); undo for elapsed calc
+ float elapsed_ms = (float)(current_frame - 1) / info.fps * 1000.0f;
+ if (elapsed_ms >= next_timestamp_ms) {
+ char ts_buf[32];
+ float elapsed_s = elapsed_ms / 1000.0f;
+ int minutes = (int)(elapsed_s / 60);
+ float seconds = elapsed_s - minutes * 60.0f;
+ snprintf(ts_buf, sizeof(ts_buf), "[%dm%.2fs]", minutes, seconds);
+ pending_text = ts_buf;
+ next_timestamp_ms += (float)timestamp_interval_ms;
+ }
+ }
+
+ return 0;
+}
+
+#ifdef MTMD_VIDEO
+
+struct mtmd_helper_video : mtmd_helper_video_seq {
const mtmd_context * mctx;
std::string path;
std::vector<uint8_t> input_buf; // non-empty when initialized from buffer
std::string ffmpeg_bin;
std::string ffprobe_bin;
- float fps_target = 0.0f;
- mtmd_helper_video_info info = {};
- std::string id; // hash of the input video
// RAII wrapper for managing subprocess
struct subprocess_handle {
@@ -596,46 +797,15 @@ struct mtmd_helper_video {
};
subprocess_handle sp;
- int32_t current_frame = 0;
-
- std::string prompt_start = "Video:";
- int32_t timestamp_interval_ms = 5000; // emit a timestamp text every N ms (0 = disabled)
- float next_timestamp_ms = 0.0f; // next elapsed-ms threshold at which to emit
// apply mtmd_get_video_config(); false for still images decoded via ffmpeg
- bool use_video_config = false;
- mtmd_video_template tmpl = MTMD_VIDEO_TEMPLATE_DEFAULT;
-
- int32_t target_frames = 0; // exact frame count to emit (0 = no limit)
- int32_t out_width = 0; // frame size from mtmd_get_video_config (0 = source size)
- int32_t out_height = 0;
-
- // source frames to emit, as picked by the reference processors (empty = use the fps filter)
- std::vector<int32_t> frame_indices;
- size_t next_index = 0;
- int32_t src_frame = 0; // source frames read from ffmpeg so far
- double src_fps = 0.0;
-
- std::vector<uint8_t> frame_buf;
- std::string pending_text; // text queued to be returned before the next frame
- mtmd_bitmap * pending_frame = nullptr; // frame queued to be returned after its group timestamp
-
- // soundtrack: after each group of frames comes the audio from its first frame to the next group's
- double video_start_s = 0.0;
- int32_t n_src_frames = 0;
- std::vector<float> soundtrack;
- mtmd_bitmap * pending_audio = nullptr;
- bool start_emitted = false;
+ bool use_video_config = false;
+ double video_start_s = 0.0; // start time of the video stream, to align the soundtrack
bool is_buf_input() const {
return !input_buf.empty();
}
- // number of frames per timestamp group (0 = timestamp every timestamp_interval_ms)
- int32_t ts_per_n_frames() const {
- return tmpl == MTMD_VIDEO_TEMPLATE_LFM3 || tmpl == MTMD_VIDEO_TEMPLATE_LFM3_OMNI ? 2 : 0;
- }
-
bool probe(float fps_target_arg) {
// cache: makes stdin seekable; without it ffprobe reports width/height 0
// for mp4 files whose moov atom sits at the end
@@ -671,11 +841,7 @@ struct mtmd_helper_video {
probe_sp.start_feeder(input_buf);
}
- uint32_t width = 0;
- uint32_t height = 0;
- float orig_fps = 0.0f;
- float duration = -1.0f;
- int32_t n_frames_orig = -1;
+ mtmd_helper_video_src src;
char line[256];
FILE * fp = probe_sp.stdout_pipe();
@@ -689,15 +855,15 @@ struct mtmd_helper_video {
if (nl) *nl = '\0';
if (strcmp(key, "width") == 0) {
- width = (uint32_t)atoi(val);
+ src.width = (uint32_t)atoi(val);
} else if (strcmp(key, "height") == 0) {
- height = (uint32_t)atoi(val);
+ src.height = (uint32_t)atoi(val);
} else if (strcmp(key, "r_frame_rate") == 0) {
- orig_fps = parse_rational(val);
+ src.fps = parse_rational(val);
} else if (strcmp(key, "nb_frames") == 0 && strcmp(val, "N/A") != 0) {
- n_frames_orig = atoi(val);
+ src.n_frames = atoi(val);
} else if (strcmp(key, "duration") == 0 && strcmp(val, "N/A") != 0) {
- duration = (float)atof(val);
+ src.duration = (float)atof(val);
} else if (strcmp(key, "start_time") == 0 && strcmp(val, "N/A") != 0) {
video_start_s = atof(val);
}
@@ -705,58 +871,15 @@ struct mtmd_helper_video {
probe_sp.stop();
- if (width == 0 || height == 0 || orig_fps <= 0.0f) {
+ if (src.width == 0 || src.height == 0 || src.fps <= 0.0f) {
return false;
}
- if (duration < 0.0f && n_frames_orig > 0) {
- duration = (float)n_frames_orig / orig_fps;
- }
-
mtmd_video_config cfg{MTMD_VIDEO_TEMPLATE_DEFAULT, 0.0f, 0, 0, 0};
if (use_video_config) {
- cfg = mtmd_get_video_config(mctx, width, height, n_frames_orig, orig_fps);
- tmpl = cfg.tmpl;
- }
-
- // the model's recommended sampling rate wins over the generic default
- fps_target = cfg.fps > 0.0f ? cfg.fps
- : fps_target_arg > 0.0f ? fps_target_arg : orig_fps;
-
- if (cfg.n_frames > 0 && duration > 0.0f) {
- const int32_t n = cfg.n_frames;
- target_frames = n;
- fps_target = (float) n / duration;
-
- // same frames as the reference processors: round(linspace(0, n_frames - 1, n))
- const double step = n > 1 ? (double) (n_frames_orig - 1) / (n - 1) : 0.0;
- frame_indices.resize(n);
- for (int32_t i = 0; i < n; i++) {
- frame_indices[i] = (int32_t) std::nearbyint(i * step);
- }
- src_fps = orig_fps;
- n_src_frames = n_frames_orig;
-
- if (cfg.frame_width > 0 && cfg.frame_height > 0 &&
- (cfg.frame_width != (int32_t) width || cfg.frame_height != (int32_t) height)) {
- out_width = cfg.frame_width;
- out_height = cfg.frame_height;
- }
+ cfg = mtmd_get_video_config(mctx, src.width, src.height, src.n_frames, src.fps);
}
-
- if (out_width > 0 && out_height > 0) {
- width = (uint32_t) out_width;
- height = (uint32_t) out_height;
- }
-
- info.width = width;
- info.height = height;
- info.fps = fps_target;
- LOG_DBG("%s: %ux%u fps=%.2f duration=%.2fs n_frames=%d target_frames=%d\n",
- __func__, width, height, fps_target, duration, n_frames_orig, target_frames);
- info.n_frames = target_frames > 0 ? target_frames
- : duration > 0.0f ? (int32_t)(duration * fps_target + 0.5f) : -1;
- frame_buf.resize((size_t)width * height * 3);
+ configure(cfg, src, fps_target_arg);
return true;
}
@@ -800,31 +923,10 @@ struct mtmd_helper_video {
LOG_WRN("%s: the video has no soundtrack\n", __func__);
return;
}
+ audio_rate = rate;
soundtrack.assign(pcm.begin() + start, pcm.begin() + std::min(pcm.size(), start + len));
}
- // the soundtrack of the group of frames that ends with frame idx, nullptr if the soundtrack has ended
- mtmd_bitmap * soundtrack_slice(int32_t idx) const {
- const int32_t ts_per_n_frames = this->ts_per_n_frames();
- const int32_t group = idx / ts_per_n_frames;
- const int32_t n_groups = ((int32_t) frame_indices.size() + ts_per_n_frames - 1) / ts_per_n_frames;
- const int rate = mtmd_get_audio_sample_rate(mctx);
- auto group_start = [&](int32_t g) {
- const double t = frame_indices[g * ts_per_n_frames] / src_fps;
- return std::min(soundtrack.size(), (size_t) std::nearbyint(t * rate));
- };
- const size_t begin = group == 0 ? 0 : group_start(group);
- const size_t end = group == n_groups - 1 ? soundtrack.size() : group_start(group + 1);
- if (begin >= end) {
- return nullptr;
- }
- mtmd_bitmap * slice = mtmd_bitmap_init_from_audio(end - begin, soundtrack.data() + begin);
- if (!id.empty()) {
- mtmd_bitmap_set_id(slice, (id + "+audio" + std::to_string(group)).c_str());
- }
- return slice;
- }
-
bool start_ffmpeg(float seek_seconds) {
char seek_buf[64];
char fps_buf[64];
@@ -917,6 +1019,7 @@ struct mtmd_helper_video {
sp.start_feeder(input_buf);
}
+ src_alive = sp.alive;
return sp.alive;
}
@@ -925,7 +1028,7 @@ struct mtmd_helper_video {
}
// read one raw frame from ffmpeg into frame_buf
- bool read_raw_frame() {
+ bool read_raw_frame() override {
FILE * fp = sp.stdout_pipe();
const size_t frame_size = (size_t)info.width * info.height * 3;
LOG_DBG("%s: reading frame %d, expecting %zu bytes (%ux%u)\n",
@@ -938,7 +1041,6 @@ struct mtmd_helper_video {
// clean EOF only if no bytes read yet; partial frame is an error
LOG_DBG("%s: fread returned 0 after %zu/%zu bytes (ferror=%d)\n",
__func__, total_read, frame_size, ferror(fp));
- sp.alive = false;
return false;
}
total_read += n;
@@ -946,129 +1048,6 @@ struct mtmd_helper_video {
return true;
}
- mtmd_bitmap * read_next_frame() {
- if (!sp.alive) return nullptr;
-
- if (!frame_indices.empty()) {
- if (next_index >= frame_indices.size()) {
- return nullptr;
- }
- // skip to the wanted source frame; a repeated index reuses the frame in frame_buf
- const int32_t want = frame_indices[next_index];
- while (src_frame <= want) {
- if (!read_raw_frame()) {
- return nullptr;
- }
- src_frame++;
- }
- next_index++;
- } else if (!read_raw_frame()) {
- return nullptr;
- }
-
- LOG_DBG("%s: frame %d read OK\n", __func__, current_frame);
- mtmd_bitmap * frame = mtmd_bitmap_init(info.width, info.height, frame_buf.data());
- mtmd_bitmap_set_mergeable(frame, true);
- mtmd_bitmap_set_is_video(frame, true);
- if (!id.empty()) {
- // each frame gets a unique id in the form of {hash}+{frame}, so that it can be identified in cache
- std::string frame_id = id + "+" + std::to_string(current_frame);
- mtmd_bitmap_set_id(frame, frame_id.c_str());
- }
- current_frame++;
- return frame;
- }
-
- int32_t read_next(mtmd_bitmap ** out_bitmap, char ** out_text) {
- *out_bitmap = nullptr;
- *out_text = nullptr;
-
- if (!pending_text.empty()) {
- *out_text = mtmd_strdup(pending_text);
- pending_text.clear();
- return *out_text ? 0 : -2;
- }
-
- if (pending_frame) {
- *out_bitmap = pending_frame;
- pending_frame = nullptr;
- return 0;
- }
-
- if (pending_audio) {
- *out_bitmap = pending_audio;
- pending_audio = nullptr;
- return 0;
- }
-
- LOG_DBG("%s: proc_alive=%d start_emitted=%d current_frame=%d\n",
- __func__, (int)sp.alive, (int)start_emitted, current_frame);
-
- if (!sp.alive) {
- return (current_frame == 0) ? -2 : -1;
- }
-
- if (!start_emitted) {
- start_emitted = true;
- // model templates have no prompt prefix, the video begin/end tokens are added by mtmd_tokenize
- if (tmpl == MTMD_VIDEO_TEMPLATE_DEFAULT && !prompt_start.empty()) {
- *out_text = mtmd_strdup(prompt_start);
- return *out_text ? 0 : -2;
- }
- }
-
- mtmd_bitmap * frame = read_next_frame();
- if (!frame) {
- return -1;
- }
-
- const int32_t ts_per_n_frames = this->ts_per_n_frames();
- if (ts_per_n_frames > 0) {
- // emit the group timestamp before the group's first frame; the value is the
- // midpoint time of the group, matching HF processors that stamp each temporal group
- const int32_t idx = current_frame - 1; // read_next_frame() already incremented
- // the soundtrack of a group follows its last frame
- const bool group_ends = idx % ts_per_n_frames == ts_per_n_frames - 1 || idx + 1 == (int32_t) frame_indices.size();
- if (!soundtrack.empty() && !frame_indices.empty() && group_ends) {
- pending_audio = soundtrack_slice(idx);
- }
- if (idx % ts_per_n_frames == 0) {
- char ts_buf[64];
- double ts_s = ((double)idx + 0.5*(double)(ts_per_n_frames - 1)) / info.fps;
- if (!frame_indices.empty()) {
- // stamp from the source frames; a short last group repeats its last frame
- const size_t first = idx;
- const size_t last = std::min(first + ts_per_n_frames - 1, frame_indices.size() - 1);
- ts_s = (frame_indices[first] / src_fps + frame_indices[last] / src_fps) / 2.0;
- }
- snprintf(ts_buf, sizeof(ts_buf), "<%.1f seconds>", ts_s);
- pending_frame = frame;
- *out_text = mtmd_strdup(ts_buf);
- return *out_text ? 0 : -2;
- }
- *out_bitmap = frame;
- return 0;
- }
-
- *out_bitmap = frame;
-
- if (timestamp_interval_ms > 0) {
- // current_frame was already incremented by read_next_frame(); undo for elapsed calc
- float elapsed_ms = (float)(current_frame - 1) / info.fps * 1000.0f;
- if (elapsed_ms >= next_timestamp_ms) {
- char ts_buf[32];
- float elapsed_s = elapsed_ms / 1000.0f;
- int minutes = (int)(elapsed_s / 60);
- float seconds = elapsed_s - minutes * 60.0f;
- snprintf(ts_buf, sizeof(ts_buf), "[%dm%.2fs]", minutes, seconds);
- pending_text = ts_buf;
- next_timestamp_ms += (float)timestamp_interval_ms;
- }
- }
-
- return 0;
- }
-
static float parse_rational(const char * s) {
int num = 0, den = 1;
if (sscanf(s, "%d/%d", &num, &den) == 2 && den > 0) {
@@ -1233,8 +1212,6 @@ void mtmd_helper_video_free(mtmd_helper_video * ctx) {
#ifdef MTMD_VIDEO
if (!ctx) return;
ctx->stop_ffmpeg();
- mtmd_bitmap_free(ctx->pending_frame);
- mtmd_bitmap_free(ctx->pending_audio);
delete ctx;
#else
GGML_UNUSED(ctx);
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index e562aae18..a9ffdf71e 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -318,6 +318,7 @@ MTMD_API int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx);
#ifdef __cplusplus
#include <set>
#include <memory>
+#include <string>
#include <vector>
namespace mtmd_helper {
@@ -408,6 +409,68 @@ inline std::vector<llama_logit_bias> audio_stream_logit_bias(const mtmd_context
}
} // namespace mtmd_helper
+
+//
+// test API (C++ only)
+// internals of mtmd_helper_video, do not use in applications
+//
+
+struct mtmd_helper_video_src {
+ uint32_t width = 0;
+ uint32_t height = 0;
+ float fps = 0.0f;
+ int32_t n_frames = -1;
+ float duration = -1.0f;
+};
+
+struct mtmd_helper_video_seq {
+ float fps_target = 0.0f;
+ mtmd_helper_video_info info = {};
+ std::string id;
+
+ bool src_alive = false;
+ int32_t current_frame = 0;
+
+ std::string prompt_start = "Video:";
+ int32_t timestamp_interval_ms = 5000;
+ float next_timestamp_ms = 0.0f;
+
+ mtmd_video_template tmpl = MTMD_VIDEO_TEMPLATE_DEFAULT;
+
+ int32_t target_frames = 0;
+ int32_t out_width = 0;
+ int32_t out_height = 0;
+
+ std::vector<int32_t> frame_indices;
+ size_t next_index = 0;
+ int32_t src_frame = 0;
+ double src_fps = 0.0;
+
+ std::vector<uint8_t> frame_buf;
+ std::string pending_text;
+ mtmd_bitmap * pending_frame = nullptr;
+
+ int32_t n_src_frames = 0;
+ int audio_rate = 0;
+ std::vector<float> soundtrack;
+ mtmd_bitmap * pending_audio = nullptr;
+ bool start_emitted = false;
+
+ mtmd_helper_video_seq() = default;
+ mtmd_helper_video_seq(const mtmd_helper_video_seq &) = delete;
+ mtmd_helper_video_seq & operator=(const mtmd_helper_video_seq &) = delete;
+ virtual ~mtmd_helper_video_seq();
+
+ virtual bool read_raw_frame() = 0;
+
+ void configure(const mtmd_video_config & cfg, const mtmd_helper_video_src & src, float fps_target_arg);
+
+ int32_t ts_per_n_frames() const;
+
+ mtmd_bitmap * soundtrack_slice(int32_t idx) const;
+ mtmd_bitmap * read_next_frame();
+ int32_t read_next(mtmd_bitmap ** out_bitmap, char ** out_text);
+};
#endif
#endif
diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp
index 2750bc3fc..8c0f433e3 100644
--- a/tools/mtmd/mtmd-image.cpp
+++ b/tools/mtmd/mtmd-image.cpp
@@ -819,12 +819,12 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_i
}
//
-// mtmd_image_preprocessor_lfm3vl
+// mtmd_image_preprocessor_qwen3vl
//
// rounds before the pixel checks (an edge may round to 0) and computes in double, as transformers does
-clip_image_size mtmd_image_preprocessor_lfm3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels,
- int n_frames, int temporal_factor) {
+clip_image_size mtmd_image_preprocessor_qwen3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels,
+ int n_frames, int temporal_factor) {
double h = height;
double w = width;
// the video variant first upscales edges below factor
@@ -850,18 +850,13 @@ clip_image_size mtmd_image_preprocessor_lfm3vl::smart_resize(int width, int heig
return {std::max(factor, w_bar), std::max(factor, h_bar)};
}
-clip_image_size mtmd_image_preprocessor_lfm3vl::target_size(const clip_image_size & original_size) const {
- return smart_resize(original_size.width, original_size.height, hparams.patch_size * hparams.n_merge,
- hparams.image_min_pixels, hparams.image_max_pixels);
-}
-
-bool mtmd_image_preprocessor_lfm3vl::get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const {
- // Lfm3VlVideoProcessor size: shortest_edge 4096, longest_edge 25165824, temporal_patch_size 2
+bool mtmd_image_preprocessor_qwen3vl::get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const {
+ // Qwen3VLVideoProcessor (and Lfm3VlVideoProcessor) size: shortest_edge 4096, longest_edge 25165824, temporal_patch_size 2
out = smart_resize(width, height, hparams.patch_size * hparams.n_merge, 4096, 25165824, n_frames, 2);
return true;
}
-mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::preprocess_video_frame(const clip_image_u8 & img) const {
+mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::preprocess_video_frame(const clip_image_u8 & img) const {
const int f = hparams.patch_size * hparams.n_merge;
const auto size = img.get_size();
return resize_to(img, {
@@ -870,7 +865,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::preprocess_video_frame(co
});
}
-mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::resize_to(const clip_image_u8 & img, const clip_image_size & size) const {
+mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::resize_to(const clip_image_u8 & img, const clip_image_size & size) const {
clip_image_u8 resized_image;
img_tool::resize(img, resized_image, size,
hparams.image_resize_algo,
@@ -881,6 +876,15 @@ mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::resize_to(const clip_imag
return output;
}
+//
+// mtmd_image_preprocessor_lfm3vl
+//
+
+clip_image_size mtmd_image_preprocessor_lfm3vl::target_size(const clip_image_size & original_size) const {
+ return smart_resize(original_size.width, original_size.height, hparams.patch_size * hparams.n_merge,
+ hparams.image_min_pixels, hparams.image_max_pixels);
+}
+
//
// mtmd_image_preprocessor_glm5v
//
diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h
index 07feae4b2..6eb49ca34 100644
--- a/tools/mtmd/mtmd-image.h
+++ b/tools/mtmd/mtmd-image.h
@@ -137,15 +137,15 @@ struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor {
// this is used by models with native support for dynamic image size, for example: Qwen-VL, Pixtral, Kimi-VL, etc
struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor {
mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
+ mtmd_image_preprocessor_dyn_size(const clip_hparams & hparams) : mtmd_image_preprocessor(hparams) {}
static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size);
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
virtual clip_image_size target_size(const clip_image_size & original_size) const { return get_target_size(hparams, original_size); }
};
-// LFM3-VL, same as dyn_size but sized exactly as the transformers Qwen2-VL smart_resize
-struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_dyn_size {
- mtmd_image_preprocessor_lfm3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_dyn_size(ctx) {}
- clip_image_size target_size(const clip_image_size & original_size) const override;
+// Qwen3-VL, same as dyn_size for images, video frames are sized as Qwen3VLVideoProcessor
+struct mtmd_image_preprocessor_qwen3vl : mtmd_image_preprocessor_dyn_size {
+ using mtmd_image_preprocessor_dyn_size::mtmd_image_preprocessor_dyn_size;
// the video helper sizes the frames (get_video_frame_size), they are only snapped to the grid
mtmd_image_preproc_out preprocess_video_frame(const clip_image_u8 & img) const override;
bool get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const override;
@@ -156,6 +156,12 @@ struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_dyn_size {
int n_frames = 1, int temporal_factor = 1);
};
+// LFM3-VL, same as qwen3vl but images are sized exactly as the transformers Qwen2-VL smart_resize
+struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_qwen3vl {
+ using mtmd_image_preprocessor_qwen3vl::mtmd_image_preprocessor_qwen3vl;
+ clip_image_size target_size(const clip_image_size & original_size) const override;
+};
+
// GLM 5.3 flash, similar to dyn_size, but each edge is aligned up to patch_size*n_merge, and max budget is met by a search over the height with the width scaled proportionally
struct mtmd_image_preprocessor_glm5v : mtmd_image_preprocessor {
mtmd_image_preprocessor_glm5v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
diff --git a/tools/mtmd/mtmd-internal.h b/tools/mtmd/mtmd-internal.h
index e7c62773e..eb961d31e 100644
--- a/tools/mtmd/mtmd-internal.h
+++ b/tools/mtmd/mtmd-internal.h
@@ -19,3 +19,10 @@ struct mtmd_internal_part {
// [QWEN_VIDEO] merged parts are erased from `parts`, so one group always maps to one part
std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::vector<mtmd_internal_part> & parts, int n_merge);
+
+struct mtmd_image_preprocessor;
+
+// mtmd_get_video_config() for a given template, without a context
+mtmd_video_config mtmd_video_config_init(mtmd_video_template tmpl, const mtmd_image_preprocessor * preproc,
+ int32_t src_width, int32_t src_height,
+ int32_t src_n_frames, float src_fps);
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 1c89eabfa..128e0ebd6 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -805,9 +805,15 @@ struct mtmd_context {
image_preproc = std::make_unique<mtmd_image_preprocessor_minicpmv>(ctx_v);
}
} break;
+ case PROJECTOR_TYPE_QWEN3VL:
+ {
+ // <|vision_start|> ... (image embeddings) ... <|vision_end|>
+ img_beg = "<|vision_start|>";
+ img_end = "<|vision_end|>";
+ image_preproc = std::make_unique<mtmd_image_preprocessor_qwen3vl>(ctx_v);
+ } break;
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
- case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_LING3VL:
case PROJECTOR_TYPE_MIMOVL:
{
@@ -3259,36 +3265,51 @@ int mtmd_get_audio_sample_rate(const mtmd_context * ctx) {
return clip_get_hparams(ctx->ctx_a)->audio_sample_rate;
}
-mtmd_video_config mtmd_get_video_config(const mtmd_context * ctx,
- int32_t src_width, int32_t src_height,
- int32_t src_n_frames, float src_fps) {
- mtmd_video_config cfg{MTMD_VIDEO_TEMPLATE_DEFAULT, 0.0f, 0, 0, 0};
- if (!ctx->ctx_v || clip_get_projector_type(ctx->ctx_v) != PROJECTOR_TYPE_LFM3VL) {
+mtmd_video_config mtmd_video_config_init(mtmd_video_template tmpl, const mtmd_image_preprocessor * preproc,
+ int32_t src_width, int32_t src_height,
+ int32_t src_n_frames, float src_fps) {
+ mtmd_video_config cfg{tmpl, 0.0f, 0, 0, 0};
+ if (tmpl == MTMD_VIDEO_TEMPLATE_DEFAULT) {
return cfg;
}
- // matches Lfm3VlVideoProcessor: 2 fps sampling with a [4, 768] frame clamp
+ // matches Qwen3VLVideoProcessor and Lfm3VlVideoProcessor: 2 fps sampling with a [4, 768] frame clamp
// LFM3-Omni samples at most 384 frames and hears the soundtrack
- const bool omni = ctx->video_has_soundtrack();
- cfg.tmpl = omni ? MTMD_VIDEO_TEMPLATE_LFM3_OMNI : MTMD_VIDEO_TEMPLATE_LFM3;
- cfg.fps = 2.0f;
+ cfg.fps = 2.0f;
if (src_n_frames <= 0 || src_fps <= 0.0f) {
return cfg;
}
int32_t n = (int32_t)((double) src_n_frames / src_fps * cfg.fps);
- n = std::clamp(n, 4, omni ? 384 : 768);
+ n = std::clamp(n, 4, tmpl == MTMD_VIDEO_TEMPLATE_LFM3_OMNI ? 384 : 768);
n = std::min(n, src_n_frames);
cfg.n_frames = n;
clip_image_size size;
- if (ctx->image_preproc && ctx->image_preproc->get_video_frame_size(n, src_width, src_height, size)) {
+ if (preproc && preproc->get_video_frame_size(n, src_width, src_height, size)) {
cfg.frame_width = size.width;
cfg.frame_height = size.height;
}
return cfg;
}
+mtmd_video_config mtmd_get_video_config(const mtmd_context * ctx,
+ int32_t src_width, int32_t src_height,
+ int32_t src_n_frames, float src_fps) {
+ mtmd_video_template tmpl = MTMD_VIDEO_TEMPLATE_DEFAULT;
+ switch (ctx->ctx_v ? clip_get_projector_type(ctx->ctx_v) : PROJECTOR_TYPE_UNKNOWN) {
+ case PROJECTOR_TYPE_LFM3VL:
+ tmpl = ctx->video_has_soundtrack() ? MTMD_VIDEO_TEMPLATE_LFM3_OMNI : MTMD_VIDEO_TEMPLATE_LFM3;
+ break;
+ case PROJECTOR_TYPE_QWEN3VL:
+ tmpl = MTMD_VIDEO_TEMPLATE_QWEN3VL;
+ break;
+ default:
+ break;
+ }
+ return mtmd_video_config_init(tmpl, ctx->image_preproc.get(), src_width, src_height, src_n_frames, src_fps);
+}
+
const char * mtmd_get_marker(const mtmd_context * ctx) {
return ctx->media_marker.c_str();
}
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index d490e9668..a593e7889 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -63,6 +63,7 @@ enum mtmd_video_template {
MTMD_VIDEO_TEMPLATE_DEFAULT, // "Video:" prefix, then a "[XmY.YYs]" timestamp every N ms
MTMD_VIDEO_TEMPLATE_LFM3, // "<X.X seconds>" before every group of 2 frames
MTMD_VIDEO_TEMPLATE_LFM3_OMNI, // same as LFM3, then the soundtrack of each group, as an audio bitmap after its frames
+ MTMD_VIDEO_TEMPLATE_QWEN3VL, // "<X.X seconds>" before every group of 2 frames, no markers around the whole video
};
// opaque types