Commit 0a1ef309c for llama.cpp
commit 0a1ef309cdbb9f2024f58a38fb4c42352c5db569
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Wed Oct 21 12:59:54 2026 +0200
mtmd: add state object for mtmd_gen (#30902)
diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md
index 70d5c5da0..5dad86100 100644
--- a/tools/mtmd/README-dev.md
+++ b/tools/mtmd/README-dev.md
@@ -55,7 +55,7 @@ For example, Qwen3-TTS:
Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system is designed to be flexible and reusable by new models.
`mtmd_gen_audio` is split into 2 main API:
-- Core API `mtmd.h`: handles main inference. Important: the API surface must be stateless; caller must handle state management and audio frame accumulation.
+- Core API `mtmd.h`: handles main inference. Important: `mtmd_gen_state` holds only the per-sequence data that the mmproj itself writes and reads back (RNG, decoder KV / conv state, output buffers). Data that goes to or comes from the backbone (hidden states, codes, PCM) must stay explicit in `mtmd_gen_inp` / `mtmd_gen_out`; the caller drives the backbone and handles audio frame accumulation.
- Helper API `mtmd-helper.h`: provides a model-agnostic stateful API. Usage example can be found in the `tools/tts` directory.
### Checklist for porting new audio generation models to mtmd
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index 8c4ffc9bb..71d349cc5 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -16,6 +16,7 @@
#include <vector>
#include <memory>
#include <fstream>
+#include <random>
#ifdef _WIN32
#ifndef NOMINMAX
@@ -463,6 +464,15 @@ struct clip_enc_state {
std::vector<ggml_tensor *> v;
};
+// per-sequence state of audio generation, see clip_gen_state_init()
+struct clip_gen_state {
+ std::mt19937 rng;
+
+ ggml_context_ptr ctx_data;
+ ggml_backend_buffer_ptr buf;
+ std::vector<ggml_tensor *> slots; // same order as list_gen_state_slots(), F32 [ne0, ne1]
+};
+
// forward declaration
// TODO: improve this later
struct clip_ctx;
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index e4750e52e..359ec5ac3 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -174,10 +174,6 @@ struct clip_ctx {
bool support_batch = false;
- // for audio gen, reseeded only when the caller asks for another seed
- std::mt19937 rng{std::random_device{}()};
- uint32_t rng_seed = UINT32_MAX;
-
clip_ctx(clip_context_params & ctx_params) {
flash_attn_type = ctx_params.flash_attn_type;
no_alloc = ctx_params.no_alloc;
@@ -4717,6 +4713,59 @@ static std::vector<c2w_state_slot> list_gen_state_slots(const clip_hparams & hpa
}
}
+// state_out can have another shape than its slot, only the byte size must match
+static void copy_gen_state_out(ggml_tensor * src, ggml_tensor * dst) {
+ GGML_ASSERT(ggml_nbytes(src) == ggml_nbytes(dst));
+ if (src->type == dst->type && ggml_are_same_shape(src, dst) && ggml_is_contiguous(src)) {
+ ggml_backend_tensor_copy(src, dst);
+ return;
+ }
+ std::vector<uint8_t> tmp(ggml_nbytes(src));
+ ggml_backend_tensor_get(src, tmp.data(), 0, tmp.size());
+ ggml_backend_tensor_set(dst, tmp.data(), 0, tmp.size());
+}
+
+clip_gen_state * clip_gen_state_init(clip_ctx * ctx, uint32_t seed) {
+ auto st = std::make_unique<clip_gen_state>();
+ st->rng.seed(seed == UINT32_MAX ? std::random_device{}() : seed);
+
+ const auto slots = list_gen_state_slots(ctx->model.hparams, ctx->model);
+ if (slots.empty()) {
+ return st.release();
+ }
+
+ ggml_init_params params = {
+ /*.mem_size =*/ ggml_tensor_overhead() * slots.size(),
+ /*.mem_buffer =*/ nullptr,
+ /*.no_alloc =*/ true,
+ };
+ st->ctx_data.reset(ggml_init(params));
+ for (const auto & slot : slots) {
+ ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1);
+ ggml_format_name(t, "gen_state_%s", slot.name.c_str());
+ st->slots.push_back(t);
+ }
+ st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), ctx->backend));
+ if (!st->buf) {
+ LOG_ERR("%s: failed to allocate the gen state\n", __func__);
+ return nullptr;
+ }
+ // a zero state is the cold start
+ ggml_backend_buffer_clear(st->buf.get(), 0);
+
+ return st.release();
+}
+
+void clip_gen_state_free(clip_gen_state * state) {
+ delete state;
+}
+
+void clip_gen_state_reset(clip_gen_state * state) {
+ if (state->buf) {
+ ggml_backend_buffer_clear(state->buf.get(), 0);
+ }
+}
+
clip_enc_state * clip_enc_state_init(clip_ctx * ctx, int32_t n_frames_max) {
const auto & hparams = ctx->model.hparams;
if (ctx->proj_type() != PROJECTOR_TYPE_VOXTRAL_RT || hparams.attn_window_size <= 0) {
@@ -4782,9 +4831,9 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
clip_model_loader::warmup(*ctx, *params->imgs);
}
- if (params->seed != ctx->rng_seed) {
- ctx->rng_seed = params->seed;
- ctx->rng.seed(params->seed == UINT32_MAX ? std::random_device{}() : params->seed);
+ if (params->gen_process != CLIP_GEN_PROCESS_GEN_UNKNOWN && params->gen_state == nullptr) {
+ LOG_ERR("%s: audio gen requires a gen state\n", __func__);
+ return false;
}
// build the inference graph
@@ -4848,19 +4897,13 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur));
};
- // upload the decoder state from the previous call, or zero-fill on a cold start
+ // state of the previous call, zero on a cold start
auto set_gen_state_in = [&]() {
- size_t offset = 0;
+ const auto & slots = params->gen_state->slots;
+ size_t i = 0;
for (const auto & slot : list_gen_state_slots(hparams, model)) {
- ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str());
- const size_t nb = ggml_nbytes(t);
- if (params->state_in && params->state_in->size() >= offset + nb) {
- ggml_backend_tensor_set(t, params->state_in->data() + offset, 0, nb);
- } else {
- std::vector<uint8_t> zeros(nb, 0);
- ggml_backend_tensor_set(t, zeros.data(), 0, nb);
- }
- offset += nb;
+ GGML_ASSERT(i < slots.size());
+ ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str()));
}
};
@@ -5584,7 +5627,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
std::normal_distribution<float> dist(0.0f, std::sqrt(temp));
std::vector<float> noise(ggml_nelements(t));
for (auto & v : noise) {
- v = dist(ctx->rng);
+ v = dist(params->gen_state->rng);
}
set_input_f32("inp_noise", noise);
}
@@ -5732,7 +5775,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
std::uniform_real_distribution<float> dist(0.0f, 1.0f);
const int64_t n_acoustic = model.gen_code_head_w->ne[2];
for (int64_t g = 0; g < n_acoustic; g++) {
- std::vector<float> r = { dist(ctx->rng) };
+ std::vector<float> r = { dist(params->gen_state->rng) };
set_input_f32(("inp_rand_" + std::to_string(g)).c_str(), r);
}
}
@@ -6262,22 +6305,16 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
st->n_past += n_new;
}
- if (params->state_out != nullptr) {
- auto & state_out = *params->state_out;
- size_t total = 0;
- for (const auto & slot : list_gen_state_slots(hparams, model)) {
- total += (size_t) (slot.ne0 * slot.ne1) * sizeof(float);
- }
- state_out.resize(total);
- size_t offset = 0;
+ if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
+ const auto & slots = params->gen_state->slots;
+ size_t i = 0;
for (const auto & slot : list_gen_state_slots(hparams, model)) {
ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str());
if (t == nullptr) {
- GGML_ABORT("state_out requested but graph has no \"state_out_%s\" tensor", slot.name.c_str());
+ GGML_ABORT("graph has no \"state_out_%s\" tensor", slot.name.c_str());
}
- const size_t nb = ggml_nbytes(t);
- ggml_backend_tensor_get(t, state_out.data() + offset, 0, nb);
- offset += nb;
+ GGML_ASSERT(i < slots.size());
+ copy_gen_state_out(t, slots[i++]);
}
}
diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h
index cdbaeae0d..cd3f434da 100644
--- a/tools/mtmd/clip.h
+++ b/tools/mtmd/clip.h
@@ -14,6 +14,7 @@
struct clip_ctx;
struct clip_enc_state;
+struct clip_gen_state;
struct clip_image_size {
int width;
@@ -110,7 +111,6 @@ struct clip_encode_params {
float top_p = 1.0f;
std::vector<int32_t> * out_codes = nullptr; // this frame's 16 sampled codes
std::vector<float> * out_feats = nullptr; // continuous counterpart of out_codes
- uint32_t seed = UINT32_MAX; // UINT32_MAX for random
float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders
bool * out_is_eos = nullptr;
@@ -118,8 +118,9 @@ struct clip_encode_params {
const std::vector<int32_t> * codes = nullptr; // this frame's 16 RVQ codes
const std::vector<float> * feats = nullptr; // continuous counterpart of codes
std::vector<float> * out_audio = nullptr; // decoded PCM samples, F32
- const std::vector<uint8_t> * state_in = nullptr; // state from previous call, null or wrong size means cold start
- std::vector<uint8_t> * state_out = nullptr; // state for the next call
+
+ // audio gen: state of one sequence, clip_encode() updates it in place
+ struct clip_gen_state * gen_state = nullptr;
// streaming audio input, updated in place by clip_encode()
struct clip_enc_state * enc_state = nullptr;
@@ -132,6 +133,12 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params);
struct clip_enc_state * clip_enc_state_init(struct clip_ctx * ctx, int32_t n_frames_max);
void clip_enc_state_free(struct clip_enc_state * state);
+// per-sequence state of audio generation: RNG and the slots kept between GEN_WAV calls
+// seed UINT32_MAX means random; reset() clears the slots and keeps the RNG stream
+struct clip_gen_state * clip_gen_state_init(struct clip_ctx * ctx, uint32_t seed);
+void clip_gen_state_free(struct clip_gen_state * state);
+void clip_gen_state_reset(struct clip_gen_state * state);
+
bool clip_is_llava(const struct clip_ctx * ctx);
// note for contributor: this clip_is_(model) pattern is deprecated
// do NOT add new functions like this
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 47493d906..d8299ba44 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -105,12 +105,25 @@ public:
virtual int32_t flush() = 0;
protected:
+ // one state per generation, so a fixed seed always gives the same output
+ bool init_gen_state(uint32_t seed) {
+ mtmd_gen_state_params params = mtmd_gen_state_params_default();
+ params.seed = seed;
+ gen_state.reset(mtmd_gen_state_init(mctx, params));
+ if (!gen_state) {
+ LOG_ERR("mtmd_helper_gen_audio: failed to create the gen state\n");
+ return false;
+ }
+ return true;
+ }
+
llama_context * lctx;
mtmd_context * mctx;
const llama_model * model;
const llama_vocab * vocab;
int n_embd;
mtmd_gen_audio_info info;
+ mtmd::gen_state_ptr gen_state;
};
// Qwen3-TTS: backbone samples codec_0, code_predictor gives the other 15 codebooks,
@@ -123,7 +136,7 @@ public:
seq_id = 0;
pos = 0;
codes_buf.clear();
- c2w_state.clear();
+ gen_state.reset();
audio_pcm.clear();
overlay.clear();
h_state_buf.clear();
@@ -220,9 +233,11 @@ public:
const mtmd_gen_inp def = mtmd_gen_inp_default(mctx);
top_k = inp->top_k > 0 ? inp->top_k : def.top_k;
top_p = inp->top_p > 0 ? inp->top_p : def.top_p;
- seed = inp->seed;
out_type = inp->out_type;
stream = inp->stream;
+ if (!init_gen_state(inp->seed)) {
+ return 1;
+ }
// the prompt above holds the whole text stream up to tts_eos, so every generated
// frame adds tts_pad on top of the codes embedding
@@ -279,9 +294,8 @@ public:
inp.embd = const_cast<float *>(h_state_in);
inp.top_k = top_k;
inp.top_p = top_p;
- inp.seed = seed;
mtmd_gen_out out{};
- if (mtmd_gen_audio_process(mctx, &inp, &out) != 0) {
+ if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: gen_code process failed\n");
return 1;
}
@@ -449,16 +463,12 @@ private:
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.codes = codes_buf.data();
inp.n_codes = codes_buf.size();
- inp.seed = seed; // same seed as gen_code, else clip reseeds mid-generation
- inp.state_data = c2w_state.empty() ? nullptr : (const char *) c2w_state.data();
- inp.state_size = c2w_state.size();
mtmd_gen_out out{};
- if (mtmd_gen_audio_process(mctx, &inp, &out) != 0) {
+ if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: gen_wav process failed\n");
return false;
}
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
- c2w_state.assign(out.state_data, out.state_data + out.state_size);
codes_buf.clear();
return true;
}
@@ -491,9 +501,7 @@ private:
int prompt_pos = 0;
int32_t top_k = 50;
float top_p = 1.0f;
- uint32_t seed = UINT32_MAX;
std::vector<int32_t> codes_buf;
- std::vector<uint8_t> c2w_state;
std::vector<float> audio_pcm;
std::vector<float> overlay;
std::vector<float> h_state_buf;
@@ -539,7 +547,7 @@ public:
seq_id = 0;
pos = 0;
feats_buf.clear();
- dec_state.clear();
+ gen_state.reset();
audio_pcm.clear();
h_state_buf.clear();
out_buf.clear();
@@ -623,9 +631,11 @@ public:
prompt_batch->set_position_normal(0, seq_id);
prompt_pos = 0;
- seed = inp->seed;
out_type = inp->out_type;
stream = inp->stream;
+ if (!init_gen_state(inp->seed)) {
+ return 1;
+ }
return 0;
}
@@ -663,13 +673,11 @@ public:
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.embd = const_cast<float *>(h_state_in);
- // clip only reseeds when the seed changes, so pass the same one on every step
- inp.seed = seed;
if (pack.temp > 0.0f) {
inp.temp = pack.temp;
}
mtmd_gen_out out{};
- if (mtmd_gen_audio_process(mctx, &inp, &out) != 0) {
+ if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: flow decode failed\n");
return 1;
}
@@ -895,7 +903,7 @@ private:
return 1;
}
// the decoder restarts too, the next chunk's audio is not continuous with this one
- dec_state.clear();
+ mtmd_gen_state_reset(gen_state.get());
if (chunk_idx + 1 >= chunks.size()) {
*out_stop = true;
@@ -1023,16 +1031,12 @@ private:
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.feats = feats_buf.data();
inp.n_feats = feats_buf.size();
- inp.seed = seed;
- inp.state_data = dec_state.empty() ? nullptr : (const char *) dec_state.data();
- inp.state_size = dec_state.size();
mtmd_gen_out out{};
- if (mtmd_gen_audio_process(mctx, &inp, &out) != 0) {
+ if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: mimi decode failed\n");
return false;
}
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
- dec_state.assign(out.state_data, out.state_data + out.state_size);
feats_buf.clear();
return true;
}
@@ -1049,7 +1053,6 @@ private:
std::unique_ptr<decode_embd_batch> prompt_batch;
int n_prompt = 0;
int prompt_pos = 0;
- uint32_t seed = UINT32_MAX;
// end-of-speech is latched, then a few more frames are generated as tail padding
int step_idx = 0;
int eos_step = -1;
@@ -1064,7 +1067,6 @@ private:
// latents are decoded a window at a time, the decoder state bridges the windows
size_t window_frames = 8;
std::vector<float> feats_buf;
- std::vector<uint8_t> dec_state;
std::vector<float> audio_pcm;
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index ffd35b2bb..9e2c5f3c9 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -523,11 +523,6 @@ struct mtmd_context {
// generation context
struct clip_ctx * ctx_gen_a; // audio
- std::vector<int32_t> gen_out_codes; // this frame's 16 sampled codes (GEN_CODE)
- std::vector<float> gen_out_feats; // this frame's continuous features, if any (GEN_CODE)
- std::vector<float> gen_out_embd; // next-step hidden state fed back to backbone (GEN_CODE)
- std::vector<float> gen_out_audio; // decoded PCM samples for the current frame (GEN_WAV)
- std::vector<uint8_t> gen_out_state; // state to feed into the next GEN_WAV call
bool print_timings;
int n_threads;
@@ -2080,7 +2075,6 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
mtmd_gen_inp inp{};
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
- inp.seed = UINT32_MAX;
if (!ctx->ctx_gen_a) {
return inp;
}
@@ -2104,12 +2098,58 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
return inp;
}
-static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
+struct mtmd_gen_state {
+ clip_ctx * ctx_clip = nullptr;
+ clip_gen_state * clip_state = nullptr;
+
+ std::vector<int32_t> out_codes; // this frame's 16 sampled codes (GEN_CODE)
+ std::vector<float> out_feats; // this frame's continuous features, if any (GEN_CODE)
+ std::vector<float> out_embd; // next-step hidden state fed back to backbone (GEN_CODE)
+ std::vector<float> out_audio; // decoded PCM samples for the current frame (GEN_WAV)
+
+ ~mtmd_gen_state() {
+ clip_gen_state_free(clip_state);
+ }
+};
+
+mtmd_gen_state_params mtmd_gen_state_params_default(void) {
+ mtmd_gen_state_params params{};
+ params.seed = UINT32_MAX;
+ return params;
+}
+
+mtmd_gen_state * mtmd_gen_state_init(mtmd_context * ctx, mtmd_gen_state_params params) {
+ if (!ctx->ctx_gen_a) {
+ LOG_ERR("%s: model does not support audio generation\n", __func__);
+ return nullptr;
+ }
+ auto st = std::make_unique<mtmd_gen_state>();
+ st->ctx_clip = ctx->ctx_gen_a;
+ st->clip_state = clip_gen_state_init(ctx->ctx_gen_a, params.seed);
+ if (!st->clip_state) {
+ return nullptr;
+ }
+ return st.release();
+}
+
+void mtmd_gen_state_free(mtmd_gen_state * state) {
+ delete state;
+}
+
+void mtmd_gen_state_reset(mtmd_gen_state * state) {
+ clip_gen_state_reset(state->clip_state);
+}
+
+static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
clip_ctx * ctx_clip = ctx->ctx_gen_a;
if (!ctx_clip) {
LOG_ERR("%s: model does not support audio generation\n", __func__);
return 1;
}
+ if (!state || state->ctx_clip != ctx_clip) {
+ LOG_ERR("%s: invalid gen state\n", __func__);
+ return 1;
+ }
*out = {};
@@ -2139,24 +2179,24 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_in
params.code0 = inp->code0;
params.top_k = inp->top_k;
params.top_p = inp->top_p;
- params.seed = inp->seed;
params.temp = inp->temp;
params.out_is_eos = &is_eos;
+ params.gen_state = state->clip_state;
if (!clip_encode(ctx_clip, ¶ms)) {
LOG_ERR("%s: clip_encode failed (gen_code)\n", __func__);
return 1;
}
- ctx->gen_out_embd = std::move(out_embd);
- ctx->gen_out_codes = std::move(out_codes);
- ctx->gen_out_feats = std::move(out_feats);
+ state->out_embd = std::move(out_embd);
+ state->out_codes = std::move(out_codes);
+ state->out_feats = std::move(out_feats);
- out->embd = ctx->gen_out_embd.data();
- out->codes = ctx->gen_out_codes.data();
- out->n_codes = ctx->gen_out_codes.size();
- out->feats = ctx->gen_out_feats.data();
- out->n_feats = ctx->gen_out_feats.size();
+ out->embd = state->out_embd.data();
+ out->codes = state->out_codes.data();
+ out->n_codes = state->out_codes.size();
+ out->feats = state->out_feats.data();
+ out->n_feats = state->out_feats.size();
out->is_eos = is_eos;
return 0;
}
@@ -2175,10 +2215,6 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_in
} else {
in_feats.assign(inp->feats, inp->feats + inp->n_feats);
}
- std::vector<uint8_t> in_state;
- if (inp->state_data) {
- in_state.assign(inp->state_data, inp->state_data + inp->state_size);
- }
// gen_wav has no hidden-state input, the batch entry is an unused placeholder
// TODO @ngxson : some models in the future may require hidden-state input, need to update this code later
@@ -2194,30 +2230,25 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_in
params.imgs = &batch;
params.n_threads = ctx->n_threads;
params.gen_process = CLIP_GEN_PROCESS_GEN_WAV;
- // gen_wav draws no randomness, but keep the seed so it does not reseed mid-generation
- params.seed = inp->seed;
params.codes = has_codes ? &in_codes : nullptr;
params.feats = has_feats ? &in_feats : nullptr;
- params.out_audio = &ctx->gen_out_audio;
- params.state_in = inp->state_data ? &in_state : nullptr;
- params.state_out = &ctx->gen_out_state;
+ params.out_audio = &state->out_audio;
+ params.gen_state = state->clip_state;
if (!clip_encode(ctx_clip, ¶ms)) {
LOG_ERR("%s: clip_encode failed (code2wav)\n", __func__);
return 1;
}
- out->audio = ctx->gen_out_audio.data();
- out->n_samples = ctx->gen_out_audio.size();
- out->state_data = (const char *) ctx->gen_out_state.data();
- out->state_size = ctx->gen_out_state.size();
+ out->audio = state->out_audio.data();
+ out->n_samples = state->out_audio.size();
return 0;
}
-int32_t mtmd_gen_audio_process(mtmd_context * ctx, const struct mtmd_gen_inp * inp, struct mtmd_gen_out * out) {
+int32_t mtmd_gen_audio_process(mtmd_context * ctx, mtmd_gen_state * state, const struct mtmd_gen_inp * inp, struct mtmd_gen_out * out) {
try {
- return mtmd_gen_audio_process_impl(ctx, inp, out);
+ return mtmd_gen_audio_process_impl(ctx, state, inp, out);
} catch (const std::exception & e) {
LOG_ERR("%s: error: %s\n", __func__, e.what());
return 1;
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index c740f1e9f..f248ed739 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -65,6 +65,7 @@ struct mtmd_image_tokens;
struct mtmd_input_chunk;
struct mtmd_input_chunks;
struct mtmd_batch;
+struct mtmd_gen_state;
struct mtmd_input_text {
const char * text;
@@ -79,6 +80,7 @@ struct mtmd_input_part {
const struct mtmd_bitmap * bitmap;
};
+
//
// C API
//
@@ -91,6 +93,7 @@ typedef struct mtmd_input_chunks mtmd_input_chunks;
typedef struct mtmd_input_text mtmd_input_text;
typedef struct mtmd_input_part mtmd_input_part;
typedef struct mtmd_batch mtmd_batch;
+typedef struct mtmd_gen_state mtmd_gen_state;
typedef bool (*mtmd_progress_callback)(float progress, void * user_data);
@@ -443,8 +446,7 @@ struct mtmd_gen_inp {
float * embd; // the hidden state from backbone, must have n_text_embd elements
int32_t top_k;
float top_p;
- uint32_t seed; // UINT32_MAX for random
- float temp; // sampling temperature, or noise scale for flow-matching decoders
+ float temp; // sampling temperature, or noise scale for flow-matching decoders
// for MTMD_GEN_PROCESS_TYPE_GEN_WAV
// pass either codes (discrete) or feats (continuous), depending on the pipeline
@@ -452,12 +454,10 @@ struct mtmd_gen_inp {
size_t n_codes;
const float * feats;
size_t n_feats;
- const char * state_data;
- size_t state_size;
};
struct mtmd_gen_out {
- // note: output memory is allocated by the context, valid until next process() call
+ // note: output memory is owned by the state, valid until the next process() call with this state
// for MTMD_GEN_PROCESS_TYPE_GEN_CODE
const int32_t * codes;
@@ -471,15 +471,28 @@ struct mtmd_gen_out {
// for MTMD_GEN_PROCESS_TYPE_GEN_WAV
const float * audio;
size_t n_samples;
- const char * state_data;
- size_t state_size;
};
+struct mtmd_gen_state_params {
+ uint32_t seed; // UINT32_MAX for random
+};
+
+MTMD_API struct mtmd_gen_state_params mtmd_gen_state_params_default(void);
+
+// returns nullptr if the model does not support audio generation
+MTMD_API mtmd_gen_state * mtmd_gen_state_init(mtmd_context * ctx, struct mtmd_gen_state_params params);
+MTMD_API void mtmd_gen_state_free(mtmd_gen_state * state);
+
+// clear the decoder state (cold start), keep the RNG stream
+MTMD_API void mtmd_gen_state_reset(mtmd_gen_state * state);
+
// defaults tuned for the loaded pipeline, callers override only what they care about
MTMD_API struct mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx);
-// note: this API is stateless, caller must handle state management and audio frame accumulation
+// run one step and update the state in place
+// the caller drives the backbone and collects the audio frames
MTMD_API int32_t mtmd_gen_audio_process(mtmd_context * ctx,
+ mtmd_gen_state * state,
const struct mtmd_gen_inp * inp,
struct mtmd_gen_out * out);
@@ -544,6 +557,11 @@ struct mtmd_audio_stream_deleter {
};
using audio_stream_ptr = std::unique_ptr<mtmd_audio_stream, mtmd_audio_stream_deleter>;
+struct mtmd_gen_state_deleter {
+ void operator()(mtmd_gen_state * val) { mtmd_gen_state_free(val); }
+};
+using gen_state_ptr = std::unique_ptr<mtmd_gen_state, mtmd_gen_state_deleter>;
+
struct bitmap {
bitmap_ptr ptr;
bitmap() : ptr(nullptr) {}