Commit 5c0b49a23 for llama.cpp
commit 5c0b49a23b46e20b2c03f48d2241ee1d6781f0d8
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Wed Oct 21 01:19:42 2026 +0200
server : add POST /tts endpoint for TTS models (#26603)
* mtmd: add audio out stream api
* add /tts endpoint
* add docs
* wire up mtmd_helper_model_can_chat
* upload speaker_ref via form-data
* expose seed to api
* fix header for pcm
* wip
* fix penalty_last_n
* fix server_output_limits
* fix pocket-tts stream
* wip
* add server_schema for tts
* add server routing
* server: coerce seed and stream from multipart form fields
Form fields arrive as strings, so -F seed=42 and -F stream=true were
rejected by the schema. seed joins the integer fields, parsed as a long
long so a full uint32 seed fits, and stream is read as a boolean when
it comes as a string.
---------
Co-authored-by: Pascal <admin@serveurperso.com>
diff --git a/common/common.cpp b/common/common.cpp
index b0ddfcb91..6ef3b785e 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1235,6 +1235,9 @@ common_gguf_info common_get_gguf_info(const std::string & fname) {
info.n_ctx_train = gguf_get_val_u32(gguf_ctx.get(), ctx_id);
}
+ // same key as llama_model_chat_template(model, nullptr)
+ info.has_chat_tmpl = gguf_find_key(gguf_ctx.get(), "tokenizer.chat_template") >= 0;
+
return info;
}
diff --git a/common/common.h b/common/common.h
index b089abd7e..70d6a671f 100644
--- a/common/common.h
+++ b/common/common.h
@@ -985,6 +985,7 @@ common_decision_type common_get_decision_type(const struct llama_model * model);
struct common_gguf_info {
common_decision_type decision_type = COMMON_DECISION_TYPE_UNKNOWN; // UNKNOWN if the file is missing, unreadable, or invalid
uint32_t n_ctx_train = 0; // 0 if unknown
+ bool has_chat_tmpl = false; // has the default tokenizer.chat_template
};
common_gguf_info common_get_gguf_info(const std::string & fname);
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index df61f82f0..e4750e52e 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -4278,8 +4278,9 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
struct clip_cap clip_get_cap(const char * fname) {
clip_cap res;
clip_model_loader loader(fname, /* skip_tensors= */ true);
- res.has_vision = loader.has_vision;
- res.has_audio = loader.has_audio;
+ res.has_vision = loader.has_vision;
+ res.has_audio = loader.has_audio;
+ res.has_gen_audio = loader.has_gen_audio;
return res;
}
diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h
index 19db47608..cdbaeae0d 100644
--- a/tools/mtmd/clip.h
+++ b/tools/mtmd/clip.h
@@ -148,5 +148,6 @@ std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx *
struct clip_cap {
bool has_vision;
bool has_audio;
+ bool has_gen_audio;
};
struct clip_cap clip_get_cap(const char * fname);
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 5fb7ea9a0..47493d906 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -50,29 +50,38 @@ static llama_token find_special_token(const llama_vocab * vocab, const std::stri
return LLAMA_TOKEN_NULL;
}
+static void put_bytes(std::vector<char> & buf, const void * p, size_t n) {
+ const char * c = (const char *) p;
+ buf.insert(buf.end(), c, c + n);
+}
+
+// data_sz == UINT32_MAX writes the "unknown length" sentinel (streaming), same as ffmpeg does on a pipe
+static void write_wav16_header(std::vector<char> & buf, uint32_t data_sz, int32_t rate) {
+ const uint32_t riff_sz = data_sz == UINT32_MAX ? UINT32_MAX : 36 + data_sz;
+ const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2;
+ const uint16_t fmt = 1, ch = 1, align = 2, bits = 16;
+ const uint32_t rate32 = (uint32_t) rate;
+ put_bytes(buf, "RIFF", 4); put_bytes(buf, &riff_sz, 4); put_bytes(buf, "WAVE", 4);
+ put_bytes(buf, "fmt ", 4); put_bytes(buf, &fmt_sz, 4);
+ put_bytes(buf, &fmt, 2); put_bytes(buf, &ch, 2); put_bytes(buf, &rate32, 4);
+ put_bytes(buf, &byte_rate, 4); put_bytes(buf, &align, 2); put_bytes(buf, &bits, 2);
+ put_bytes(buf, "data", 4); put_bytes(buf, &data_sz, 4);
+}
+
+static void append_wav16_pcm(std::vector<char> & buf, const float * pcm, size_t n) {
+ for (size_t i = 0; i < n; i++) {
+ int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f);
+ put_bytes(buf, &s, 2);
+ }
+}
+
static bool write_wav16(std::vector<char> & buf, const std::vector<float> & pcm, int32_t rate) {
// RIFF chunk sizes are 32-bit; refuse to emit a file with a truncated header
if (pcm.size() > ((size_t) UINT32_MAX - 36) / 2) {
return false;
}
- const uint32_t data_sz = (uint32_t) (pcm.size() * 2);
- const uint32_t riff_sz = 36 + data_sz;
- const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2;
- const uint16_t fmt = 1, ch = 1, align = 2, bits = 16;
- const uint32_t rate32 = (uint32_t) rate;
- auto put = [&](const void * p, size_t n) {
- const char * c = (const char *) p;
- buf.insert(buf.end(), c, c + n);
- };
- put("RIFF", 4); put(&riff_sz, 4); put("WAVE", 4);
- put("fmt ", 4); put(&fmt_sz, 4);
- put(&fmt, 2); put(&ch, 2); put(&rate32, 4);
- put(&byte_rate, 4); put(&align, 2); put(&bits, 2);
- put("data", 4); put(&data_sz, 4);
- for (float v : pcm) {
- int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, v)) * 32767.0f);
- put(&s, 2);
- }
+ write_wav16_header(buf, (uint32_t) (pcm.size() * 2), rate);
+ append_wav16_pcm(buf, pcm.data(), pcm.size());
return true;
}
@@ -92,6 +101,8 @@ public:
// set out_stop on end-of-speech, h_state_out must be null if no frame is generated
virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) = 0;
virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0;
+ // forces any buffered codes through code2wav now, regardless of window_frames
+ virtual int32_t flush() = 0;
protected:
llama_context * lctx;
@@ -121,6 +132,9 @@ public:
prompt_batch.reset();
n_prompt = 0;
prompt_pos = 0;
+ stream = false;
+ pcm_sent = 0;
+ wav_header_sent = false;
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
@@ -208,6 +222,7 @@ public:
top_p = inp->top_p > 0 ? inp->top_p : def.top_p;
seed = inp->seed;
out_type = inp->out_type;
+ stream = inp->stream;
// the prompt above holds the whole text stream up to tts_eos, so every generated
// frame adds tts_pad on top of the codes embedding
@@ -301,31 +316,60 @@ public:
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
- if (!flush_gen_wav()) {
- return 1;
+ *out_sample_rate = info.sample_rate;
+
+ if (!stream) {
+ // one-shot call: force out whatever's left, regardless of window_frames
+ if (!flush_gen_wav()) {
+ return 1;
+ }
+ if (out_n_samples) {
+ *out_n_samples = (int64_t) audio_pcm.size();
+ }
+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+ *out_data = (const char *) audio_pcm.data();
+ *out_data_len = audio_pcm.size() * sizeof(float);
+ return 0;
+ }
+ out_buf.clear();
+ if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
+ LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
+ return 1;
+ }
+ *out_data = out_buf.data();
+ *out_data_len = out_buf.size();
+ return 0;
}
- *out_sample_rate = info.sample_rate;
+ // streaming: only return audio produced since the previous call
+ const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
- *out_n_samples = (int64_t) audio_pcm.size();
+ *out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
- *out_data = (const char *) audio_pcm.data();
- *out_data_len = audio_pcm.size() * sizeof(float);
+ *out_data = (const char *) (audio_pcm.data() + pcm_sent);
+ *out_data_len = n_new * sizeof(float);
+ pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
- if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
- LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
- return 1;
+ if (!wav_header_sent) {
+ write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
+ wav_header_sent = true;
}
+ append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
+ pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
+ int32_t flush() override {
+ return flush_gen_wav() ? 0 : 1;
+ }
+
private:
bool ensure_cache() {
if (specials_ok) {
@@ -369,7 +413,7 @@ private:
LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n");
return false;
}
- const std::string marker = mtmd_default_marker();
+ const std::string marker = mtmd_get_marker(mctx);
mtmd_input_text text{ marker.c_str(), marker.size(), false, true };
mtmd_input_chunks * chunks = mtmd_input_chunks_init();
const mtmd_bitmap * bptr = bitmap;
@@ -455,6 +499,9 @@ private:
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
+ bool stream = false;
+ size_t pcm_sent = 0; // samples already returned by get_output()
+ bool wav_header_sent = false;
};
// settings that only live in the reference's per-pack yaml, not in the checkpoint
@@ -506,6 +553,9 @@ public:
chunk_idx = 0;
n_voice_pos = 0;
chunk_budget = 0;
+ stream = false;
+ pcm_sent = 0;
+ wav_header_sent = false;
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
@@ -575,6 +625,7 @@ public:
seed = inp->seed;
out_type = inp->out_type;
+ stream = inp->stream;
return 0;
}
@@ -662,31 +713,60 @@ public:
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
- if (!flush_gen_wav()) {
- return 1;
+ *out_sample_rate = info.sample_rate;
+
+ if (!stream) {
+ // one-shot call: force out whatever's left, regardless of window_frames
+ if (!flush_gen_wav()) {
+ return 1;
+ }
+ if (out_n_samples) {
+ *out_n_samples = (int64_t) audio_pcm.size();
+ }
+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+ *out_data = (const char *) audio_pcm.data();
+ *out_data_len = audio_pcm.size() * sizeof(float);
+ return 0;
+ }
+ out_buf.clear();
+ if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
+ LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
+ return 1;
+ }
+ *out_data = out_buf.data();
+ *out_data_len = out_buf.size();
+ return 0;
}
- *out_sample_rate = info.sample_rate;
+ // streaming: only return audio produced since the previous call
+ const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
- *out_n_samples = (int64_t) audio_pcm.size();
+ *out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
- *out_data = (const char *) audio_pcm.data();
- *out_data_len = audio_pcm.size() * sizeof(float);
+ *out_data = (const char *) (audio_pcm.data() + pcm_sent);
+ *out_data_len = n_new * sizeof(float);
+ pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
- if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
- LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
- return 1;
+ if (!wav_header_sent) {
+ write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
+ wav_header_sent = true;
}
+ append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
+ pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
+ int32_t flush() override {
+ return flush_gen_wav() ? 0 : 1;
+ }
+
private:
bool ensure_cache() {
if (specials_ok) {
@@ -907,7 +987,7 @@ private:
LOG_ERR("mtmd_helper_gen_audio: mmproj has no voice encoder\n");
return false;
}
- const std::string marker = mtmd_default_marker();
+ const std::string marker = mtmd_get_marker(mctx);
mtmd_input_text text{ marker.c_str(), marker.size(), false, true };
mtmd_input_chunks * chunks = mtmd_input_chunks_init();
const mtmd_bitmap * bptr = bitmap;
@@ -989,6 +1069,9 @@ private:
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
+ bool stream = false;
+ size_t pcm_sent = 0; // samples already returned by get_output()
+ bool wav_header_sent = false;
};
static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
@@ -1017,11 +1100,20 @@ void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx) {
}
void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) {
- if (ctx->pipeline) {
+ if (ctx && ctx->pipeline) {
ctx->pipeline->reset();
}
}
+struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) {
+ mtmd_helper_gen_audio_inp inp{};
+ inp.top_k = 50;
+ inp.top_p = 1.0f;
+ inp.seed = UINT32_MAX; // random
+ inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
+ return inp;
+}
+
int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_helper_gen_audio_inp * inp) {
if (!ctx->pipeline) {
LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n");
@@ -1058,3 +1150,10 @@ int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t *
}
return ctx->pipeline->get_output(out_sample_rate, out_data, out_data_len, out_n_samples);
}
+
+int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx) {
+ if (!ctx->pipeline) {
+ return 1;
+ }
+ return ctx->pipeline->flush();
+}
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index 7436230f0..acbfefcba 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -204,6 +204,7 @@ enum mtmd_helper_gen_audio_outtype {
MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV, // WAV PCM 16-bit LE, mono
};
struct mtmd_helper_gen_audio_inp {
+ bool stream; // if true, output() must be called after each step_gen()
llama_seq_id seq_id;
const char * prompt;
@@ -219,6 +220,8 @@ struct mtmd_helper_gen_audio_inp {
enum mtmd_helper_gen_audio_outtype out_type;
};
+MTMD_API struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void);
+
MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(
struct llama_context * lctx,
struct mtmd_context * mctx);
@@ -250,6 +253,8 @@ MTMD_API int32_t mtmd_helper_gen_audio_step_gen(
// out_data valid until next get_output() or reset() call
// out_n_samples (optional, can be NULL) receives the number of generated PCM samples
+// if inp->stream is true: returns only audio produced since the previous call, and
+// *out_data_len == 0 whenever a full window_frames batch hasn't accumulated yet
MTMD_API int32_t mtmd_helper_gen_audio_get_output(
mtmd_helper_gen_audio * ctx,
int32_t * out_sample_rate,
@@ -257,6 +262,10 @@ MTMD_API int32_t mtmd_helper_gen_audio_get_output(
size_t * out_data_len,
int64_t * out_n_samples);
+// forces any buffered codes through code2wav now, regardless of window_frames;
+// call once when generation has ended, before the last get_output() in stream mode
+MTMD_API int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx);
+
#ifdef __cplusplus
} // extern "C"
#endif
@@ -283,8 +292,41 @@ struct mtmd_helper_gen_audio_deleter {
};
using gen_audio_ptr = std::unique_ptr<mtmd_helper_gen_audio, mtmd_helper_gen_audio_deleter>;
struct gen_audio {
+
+ // sub-struct, RAII wrapper for mtmd_helper_gen_audio_inp
+ struct inp {
+ mtmd_helper_gen_audio_inp data = mtmd_helper_gen_audio_inp_default();
+ std::string prompt_str;
+ std::string lang_str;
+ mtmd::bitmap_ptr speaker_ref_ptr;
+
+ inp() = default;
+ inp(inp &&) = default;
+ inp & operator=(inp &&) = default;
+ inp(const inp &) = delete;
+ inp & operator=(const inp &) = delete;
+
+ void set_prompt (std::string p) { prompt_str = std::move(p); }
+ void set_lang (std::string l) { lang_str = std::move(l); }
+ void set_speaker_ref(mtmd::bitmap_ptr bmp) { speaker_ref_ptr = std::move(bmp); }
+
+ // pointers are only valid as long as *this is alive
+ const mtmd_helper_gen_audio_inp * get() {
+ data.prompt = prompt_str.c_str();
+ data.prompt_len = prompt_str.size();
+ data.lang = lang_str.empty() ? nullptr : lang_str.c_str();
+ data.speaker_ref = speaker_ref_ptr.get();
+ return &data;
+ }
+ };
+
gen_audio_ptr ctx;
- gen_audio(struct llama_context * lctx, struct mtmd_context * mctx) : ctx(mtmd_helper_gen_audio_init(lctx, mctx)) {}
+ void init(struct llama_context * lctx, struct mtmd_context * mctx) {
+ ctx.reset(mtmd_helper_gen_audio_init(lctx, mctx));
+ }
+ bool valid() const {
+ return ctx.get() != nullptr;
+ }
void reset() {
mtmd_helper_gen_audio_reset(ctx.get());
}
@@ -300,6 +342,9 @@ struct gen_audio {
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples = nullptr) {
return mtmd_helper_gen_audio_get_output(ctx.get(), out_sample_rate, out_data, out_data_len, out_n_samples);
}
+ int32_t flush() {
+ return mtmd_helper_gen_audio_flush(ctx.get());
+ }
};
} // namespace mtmd_helper
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index b3ac1979e..ffd35b2bb 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -2846,10 +2846,11 @@ struct mtmd_caps mtmd_get_cap_from_file(const char * fname) {
mtmd_caps cap;
cap.inp_audio = tmp.has_audio;
cap.inp_vision = tmp.has_vision;
+ cap.gen_audio = tmp.has_gen_audio;
return cap;
} catch (const std::exception & e) {
LOG_ERR("%s: failed to get capabilities from file '%s': %s\n", __func__, fname, e.what());
- return mtmd_caps{ false, false };
+ return mtmd_caps{ false, false, false };
}
}
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index df24c5656..c740f1e9f 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -360,6 +360,7 @@ MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data);
struct mtmd_caps {
bool inp_vision;
bool inp_audio;
+ bool gen_audio;
};
MTMD_API struct mtmd_caps mtmd_get_cap_from_file(const char * mmproj_fname);
diff --git a/tools/server/README.md b/tools/server/README.md
index 2977fcd4d..2886bca15 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -802,6 +802,50 @@ curl http://127.0.0.1:8012/v1/rerank \
}' | jq
```
+### POST `/tts`: Generate speech audio from text
+
+Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info, see [tts/README.md](../tts/README.md)
+
+*Options:*
+
+`input`: The text to speak (alias: `prompt`).
+
+`lang`: Language code for the utterance (model-dependent, e.g. `en`, `zh`). Optional.
+
+`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence.
+
+`top_k`, `top_p`: Sampling params for the acoustic code predictor. Optional, model-dependent defaults apply.
+
+`repeat_penalty`: Repetition penalty applied to the backbone sampler over the whole generation (default `1.05`). Without this, the backbone can loop and re-generate the same utterance.
+
+`seed`: RNG seed for both the backbone sampler and the codec/vocoder (`-1` = random, the default). Set to a fixed value for reproducible output.
+
+`n_predict`: Max number of audio frames to generate. Defaults to `512`; generation normally stops earlier once the model emits an end-of-speech token.
+
+`response_format`: `wav` (default) or `pcm` (raw `float32` little-endian mono samples, no header; the response `Content-Type` carries the sample rate, e.g. `audio/pcm;rate=24000;encoding=float;bits=32`).
+
+`stream`: If `true`, the response is streamed as audio becomes available instead of waiting for the full generation to finish. WAV streaming writes an RFC-noncompliant header with an unknown (`0xFFFFFFFF`) size field, since the final length isn't known up front; most players and decoders (ffmpeg, VLC, ...) handle this by reading until EOF.
+
+Note: it's highly recommended to always provide a speaker reference voice; otherwise, the model's performance may be degraded.
+
+*Examples:*
+
+```shell
+curl -X POST http://127.0.0.1:9931/tts \
+ -H "Content-Type: application/json" \
+ -d '{"input": "Hello, this is a test."}' \
+ -o output.wav
+```
+
+With a speaker reference uploaded as a file (`multipart/form-data`), instead of base64-encoding it into the JSON body:
+
+```shell
+curl -X POST http://127.0.0.1:9931/tts \
+ -F "input=Hello, this is a test." \
+ -F "speaker_ref=@/path/to/speaker-reference.wav;type=audio/wav" \
+ -o output.wav
+```
+
### POST `/infill`: For code infilling.
Takes a prefix and a suffix and returns the predicted completion as stream.
@@ -1262,11 +1306,12 @@ Each object in `data` has an `architecture` object. It has two string arrays:
- `input_modalities` lists what the model can read. It always has `text`, plus each media type that the model supports.
- `output_modalities` lists what the model can produce.
-One output value is special:
+Some output values are special:
| Value | Meaning |
|---|---|
| `decisions` | The model is a native decision model. Serve it with [`/v1/systemone`](#post-v1systemone-typesafe-compatible-system-one-api). |
+| `audio` | The model can generate speech. Serve it with [`/tts`](#post-tts-generate-speech-audio-from-text). A TTS-only model has `["audio"]` without `text`. |
A language model that classifies with prompts does not get `decisions`. Only native decision models do.
@@ -2054,7 +2099,7 @@ Note:
- If a model is not running, it will be added or updated according to the source
2. When the model is loaded, the info from `/v1/models` is forwarded to router's `/v1/models`. This includes metadata about the model and the runtime instance.
-Each object in `data` has the same `architecture` object as [`GET /v1/models`](#get-v1models-openai-compatible-model-info-api) of a direct server. The server computes both arrays offline. It does not load the model, download files, or run inference. `output_modalities` comes from the GGUF metadata. `input_modalities` comes from the projector file. A native decision model shows `decisions` before its first load, after unload, and while it sleeps:
+Each object in `data` has the same `architecture` object as [`GET /v1/models`](#get-v1models-openai-compatible-model-info-api) of a direct server. The server computes both arrays offline. It does not load the model, download files, or run inference. `output_modalities` comes from the GGUF metadata, plus `audio` when the projector file can generate speech. `input_modalities` comes from the projector file. A native decision model shows `decisions` before its first load, after unload, and while it sleeps:
```json
{
diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp
index 075687602..baaca58eb 100644
--- a/tools/server/server-common.cpp
+++ b/tools/server/server-common.cpp
@@ -202,7 +202,8 @@ const char * get_media_marker() {
// model output modalities
//
-std::vector<std::string> server_model_output_modalities(common_decision_type decision_type) {
+std::vector<std::string> server_model_output_modalities(common_decision_type decision_type, bool has_cap_chat, bool has_cap_tts) {
+ std::vector<std::string> out;
switch (decision_type) {
case COMMON_DECISION_TYPE_OPENJEV:
case COMMON_DECISION_TYPE_LEV:
@@ -216,13 +217,27 @@ std::vector<std::string> server_model_output_modalities(common_decision_type dec
case COMMON_DECISION_TYPE_LFM2_D1:
case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
case COMMON_DECISION_TYPE_H2O_LIGHTNING:
- return {"decisions"};
+ out = {"decisions"};
+ break;
case COMMON_DECISION_TYPE_GRANITE_SWITCH:
- return {"text", "decisions"};
+ out = {"text", "decisions"};
+ break;
default:
- // fallback when there is no decision type or the metadata is bad
- return {"text"};
+ // a tts-only model cannot generate text
+ if (has_cap_chat || !has_cap_tts) {
+ // fallback when there is no decision type or the metadata is bad
+ out = {"text"};
+ }
+ break;
}
+ if (has_cap_tts) {
+ out.push_back("audio");
+ }
+ return out;
+}
+
+std::vector<std::string> server_model_output_modalities(const llama_model * model, bool has_cap_chat, bool has_cap_tts) {
+ return server_model_output_modalities(common_get_decision_type(model), has_cap_chat, has_cap_tts);
}
json server_model_architecture_json(
diff --git a/tools/server/server-common.h b/tools/server/server-common.h
index 277752122..538e008e9 100644
--- a/tools/server/server-common.h
+++ b/tools/server/server-common.h
@@ -114,7 +114,10 @@ const char * get_media_marker();
//
// output modalities for architecture.output_modalities in GET /models
-std::vector<std::string> server_model_output_modalities(common_decision_type decision_type);
+// has_cap_chat: false for a tts-only model, has_cap_tts: the model can generate audio
+std::vector<std::string> server_model_output_modalities(common_decision_type decision_type, bool has_cap_chat, bool has_cap_tts);
+// same, with the decision type read from a loaded model
+std::vector<std::string> server_model_output_modalities(const llama_model * model, bool has_cap_chat, bool has_cap_tts);
// architecture object of GET /models; shared by the direct server and the router
json server_model_architecture_json(
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 6d8f8e7be..b8e132d7b 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -20,6 +20,7 @@
#include "speculative.h"
#include "mtmd.h"
#include "mtmd-helper.h"
+#include "base64.hpp"
#include <algorithm>
#include <cstddef>
@@ -43,6 +44,14 @@
constexpr int HTTP_POLLING_SECONDS = 1;
static common_speculative_output_limits server_output_limits(const common_params & params) {
+ if (!params.mmproj.path.empty()) {
+ const auto mcaps = mtmd_get_cap_from_file(params.mmproj.path.c_str());
+ if (mcaps.gen_audio) {
+ // some TTS models decode with embeddings output for the whole batch
+ return { params.n_batch, params.n_batch };
+ }
+ }
+
if (params.embedding ||
(params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE)) {
return { params.n_batch, 1 };
@@ -257,6 +266,27 @@ struct server_slot {
mtmd_context * mctx = nullptr;
mtmd::batch_ptr mbatch = nullptr;
+ struct tts_ctx {
+ mtmd_helper::gen_audio ctx;
+ std::vector<float> h_state_prompt; // the first h_state after step_prompt done
+ const float * h_state;
+ llama_token sampled;
+ int32_t n_decoded;
+ bool stop;
+ bool is_supported() const {
+ return ctx.valid();
+ }
+ void reset() {
+ ctx.reset();
+ h_state_prompt.clear();
+ h_state = nullptr;
+ sampled = LLAMA_TOKEN_NULL;
+ n_decoded = 0;
+ stop = false;
+ }
+ };
+ tts_ctx tts;
+
// speculative decoding
common_speculative * spec;
@@ -421,6 +451,9 @@ struct server_slot {
// clear multimodal state
mbatch.reset();
+
+ tts.reset();
+
audio_stream.reset();
audio_wait = false;
inp_embd.clear();
@@ -622,6 +655,11 @@ struct server_slot {
prompt_clear();
}
+ // mtmd_helper always re-eval the whole prompt
+ if (task->type == SERVER_TASK_TYPE_TTS) {
+ prompt_clear();
+ }
+
callback_on_reset(*this);
reset();
@@ -975,6 +1013,14 @@ public:
mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
const llama_vocab * vocab = nullptr;
+ bool has_cap_tts() const {
+ return mctx != nullptr && mtmd_gen_audio_get_info(mctx).type != MTMD_GEN_AUDIO_TYPE_NONE;
+ }
+
+ bool has_cap_chat() const {
+ return mctx == nullptr || mtmd_helper_model_can_chat(ctx_tgt, mctx);
+ }
+
server_decision_context decision;
server_queue queue_tasks;
@@ -1463,6 +1509,10 @@ private:
slot.mctx = mctx;
slot.prompt.tokens.has_mtmd = mctx != nullptr;
+ if (has_cap_tts()) {
+ slot.tts.ctx.init(ctx_tgt, mctx);
+ }
+
SLT_TRC(slot, "new slot, n_ctx = %d\n", slot.n_ctx);
slot.callback_on_release = [this](int id_slot) {
@@ -1944,6 +1994,20 @@ private:
SLT_DBG(slot, "launching slot : %s\n", safe_json_to_str(slot.to_json()).c_str());
+ if (task.type == SERVER_TASK_TYPE_TTS) {
+ GGML_ASSERT(has_cap_tts()); // should already checked in route handler
+ if (!slot.tts.is_supported()) {
+ slot.tts.ctx.init(ctx_tgt, slot.mctx);
+ }
+ // mtmd_helper always re-eval the whole prompt
+ slot.prompt_clear();
+ task.tts_inp.data.seq_id = slot.id;
+ if (slot.tts.ctx.set_input(task.tts_inp.get()) != 0) {
+ send_error(task, "failed to process TTS prompt", ERROR_TYPE_SERVER);
+ return false;
+ }
+ }
+
// initialize samplers
if (task.need_sampling()) {
try {
@@ -1961,6 +2025,9 @@ private:
// TODO: getting pre sampling logits is not yet supported with backend sampling
use_backend_sampling &= !need_pre_sample_logits;
+ // TODO: check verify if this actually works with TTS
+ use_backend_sampling &= task.type != SERVER_TASK_TYPE_TTS;
+
// TODO: tmp until backend sampling is fully implemented
if (use_backend_sampling) {
llama_set_sampler(ctx_tgt, slot.id, common_sampler_get(slot.smpl.get()));
@@ -1986,9 +2053,13 @@ private:
slot.task = std::make_unique<const server_task>(std::move(task));
- slot.state = slot.task->is_child()
- ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt
- : SLOT_STATE_STARTED;
+ if (slot.task->type == SERVER_TASK_TYPE_TTS) {
+ slot.state = SLOT_STATE_PROCESSING_PROMPT;
+ } else {
+ slot.state = slot.task->is_child()
+ ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt
+ : SLOT_STATE_STARTED;
+ }
// no prompt, the rows come from the audio stream, which audio controls reach from now on
if (slot.task->type == SERVER_TASK_TYPE_TRANSCRIBE_RT) {
@@ -2268,6 +2339,18 @@ private:
queue_results.send(std::move(res));
}
+ void send_tts_result(server_slot & slot, int32_t sample_rate, const char * data, size_t data_len, bool final) {
+ auto res = std::make_unique<server_task_result_tts>();
+
+ res->id = slot.task->id;
+ res->index = slot.task->index;
+ res->sample_rate = sample_rate;
+ res->audio.assign(data, data_len);
+ res->final = final;
+
+ queue_results.send(std::move(res));
+ }
+
void send_final_response(server_slot & slot) {
auto res = std::make_unique<server_task_result_cmpl_final>();
@@ -2872,6 +2955,7 @@ private:
case SERVER_TASK_TYPE_TRANSCRIBE_RT:
case SERVER_TASK_TYPE_EMBEDDING:
case SERVER_TASK_TYPE_RERANK:
+ case SERVER_TASK_TYPE_TTS:
case SERVER_TASK_TYPE_DECISION:
{
// special case: if input is provided via CLI, tokenize it first
@@ -3397,6 +3481,14 @@ private:
return;
}
+ // note: TTS slots bypass the shared batch entirely
+ try {
+ process_tts_slots();
+ } catch (const std::exception & e) {
+ SRV_ERR("process_tts_slots() failed: %s\n", e.what());
+ abort_all_slots("process_tts_slots() failed: " + std::string(e.what()));
+ }
+
GGML_ASSERT(batch.slot_batched || batch.size() == 0);
if (batch.slot_batched) {
@@ -3459,10 +3551,89 @@ private:
}
}
+ void process_tts_slots() {
+ iterate(slots, [&](server_slot & slot) {
+ if (!slot.is_processing() || slot.task->type != SERVER_TASK_TYPE_TTS) {
+ return;
+ }
+
+ llama_set_embeddings(ctx_tgt, true);
+
+ if (slot.state == SLOT_STATE_PROCESSING_PROMPT) {
+ const int32_t ret = slot.tts.ctx.step_prompt(llama_n_batch(ctx_tgt));
+ if (ret < 0) {
+ send_error(slot, "TTS prompt processing failed", ERROR_TYPE_SERVER);
+ slot.release();
+ } else if (ret == 0) {
+ // done prompt, do sample and save h_state_prompt
+ slot.tts.sampled = common_sampler_sample(slot.smpl.get(), ctx_tgt, -1);
+ common_sampler_accept(slot.smpl.get(), slot.tts.sampled, true);
+ const float * h_embd = llama_get_embeddings_ith(ctx_tgt, -1);
+ slot.tts.h_state_prompt.assign(h_embd, h_embd + llama_model_n_embd(model_tgt));
+ slot.tts.h_state = slot.tts.h_state_prompt.data();
+ slot.state = SLOT_STATE_GENERATING;
+ }
+ return;
+ }
+
+ const int32_t n_predict = slot.n_predict_max > 0 ? slot.n_predict_max : 512;
+ if (slot.tts.stop || slot.tts.n_decoded >= n_predict) {
+ int32_t sample_rate = 0;
+ const char * data = nullptr;
+ size_t data_len = 0;
+ // generation truly ends here: force out any sub-window remainder still buffered
+ if (slot.tts.ctx.flush() != 0 || slot.tts.ctx.get_output(&sample_rate, &data, &data_len) != 0) {
+ send_error(slot, "failed to finalize TTS output", ERROR_TYPE_SERVER);
+ } else {
+ send_tts_result(slot, sample_rate, data, data_len, true);
+ }
+ slot.release();
+ return;
+ }
+
+ const float * h_state_next = nullptr;
+ bool stop = false;
+ if (slot.tts.ctx.step_gen(slot.tts.sampled, slot.tts.h_state, &h_state_next, &stop) != 0) {
+ send_error(slot, "TTS generation failed", ERROR_TYPE_SERVER);
+ slot.release();
+ return;
+ }
+ if (h_state_next == nullptr) {
+ // end-of-speech without a new frame (e.g. pocket-tts eos head)
+ slot.tts.stop = true;
+ return;
+ }
+ slot.tts.h_state = h_state_next;
+ slot.tts.n_decoded++;
+ slot.tts.stop = stop;
+
+ if (!stop) {
+ slot.tts.sampled = common_sampler_sample(slot.smpl.get(), ctx_tgt, -1);
+ common_sampler_accept(slot.smpl.get(), slot.tts.sampled, true);
+ }
+
+ if (slot.task->params.stream) {
+ int32_t sample_rate = 0;
+ const char * data = nullptr;
+ size_t data_len = 0;
+ if (slot.tts.ctx.get_output(&sample_rate, &data, &data_len) != 0) {
+ send_error(slot, "TTS streaming output failed", ERROR_TYPE_SERVER);
+ slot.release();
+ } else if (data_len > 0) {
+ send_tts_result(slot, sample_rate, data, data_len, false);
+ }
+ }
+ });
+ }
+
void pre_decode() {
// apply context-shift if needed
// TODO: simplify and improve
iterate(slots, [&](server_slot & slot) {
+ if (slot.task && slot.task->type == SERVER_TASK_TYPE_TTS) {
+ // TTS slots drive their own decode loop in process_tts_slots(), never enter the shared batch
+ return;
+ }
if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) {
if (!params_base.ctx_shift) {
// this check is redundant (for good)
@@ -3535,7 +3706,7 @@ private:
// determine which slots are generating and drafting
iterate(slots, [&](server_slot & slot) {
- if (slot.state != SLOT_STATE_GENERATING) {
+ if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS) {
return;
}
@@ -3681,7 +3852,7 @@ private:
return; // batch is full, skip remaining slots
}
- if (!slot.is_processing()) {
+ if (!slot.is_processing() || slot.task->type == SERVER_TASK_TYPE_TTS) {
return;
}
@@ -4926,11 +5097,13 @@ server_context_meta server_context::get_meta() const {
/* model_aliases */ impl->model_aliases,
/* model_tags */ impl->model_tags,
/* model_path */ impl->params_base.model.path,
- /* model_output_modalities */ server_model_output_modalities(common_get_decision_type(impl->model_tgt)),
+ /* model_output_modalities */ server_model_output_modalities(impl->model_tgt, impl->has_cap_chat(), impl->has_cap_tts()),
/* has_mtmd */ impl->mctx != nullptr,
/* has_inp_image */ impl->chat_params.allow_image,
/* has_inp_audio */ impl->chat_params.allow_audio,
/* has_inp_video */ impl->chat_params.allow_video,
+ /* has_cap_chat */ impl->has_cap_chat(),
+ /* has_cap_tts */ impl->has_cap_tts(),
/* has_audio_stream */ impl->mctx != nullptr && mtmd_support_audio_stream(impl->mctx),
/* json_ui_settings */ impl->json_ui_settings,
/* slot_n_ctx */ impl->n_ctx_slot(),
@@ -5006,6 +5179,11 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
res->set_req(&req); // will also set spipe if needed
+ if (!ctx_server.has_cap_chat()) {
+ res->error(format_error_response("this server does not support chat/completions", ERROR_TYPE_NOT_SUPPORTED));
+ return res;
+ }
+
int32_t sse_ping_interval = params.sse_ping_interval;
try {
@@ -6175,6 +6353,171 @@ void server_routes::init_routes() {
return res;
};
+ this->post_tts = [this](const server_http_req & req) {
+ auto res = create_response();
+ res->set_req(&req); // will also set spipe if needed
+
+ if (!ctx_server.has_cap_tts()) {
+ res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED));
+ return res;
+ }
+
+ const auto info = mtmd_gen_audio_get_info(ctx_server.mctx);
+
+ json body = json::parse(req.body);
+
+ // multipart form fields arrive as strings, correct the types of the numeric and boolean params
+ for (const char * k : { "top_k", "n_predict", "max_tokens", "seed" }) {
+ if (body.contains(k) && body[k].is_string()) {
+ body[k] = std::stoll(body[k].get<std::string>());
+ }
+ }
+ for (const char * k : { "top_p", "repeat_penalty" }) {
+ if (body.contains(k) && body[k].is_string()) {
+ body[k] = std::stof(body[k].get<std::string>());
+ }
+ }
+ if (body.contains("stream") && body["stream"].is_string()) {
+ body["stream"] = body["stream"] == "true";
+ }
+
+ std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string()));
+ if (prompt.empty()) {
+ res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+
+ const std::string response_format = json_value(body, "response_format", std::string("wav"));
+
+ server_task task(SERVER_TASK_TYPE_TTS);
+ server_schema::eval_tts_schema(params, task, body);
+ const bool stream = task.params.stream;
+ task.tts_inp.set_prompt(prompt);
+ task.tts_inp.set_lang(json_value(body, "lang", std::string()));
+ task.tts_inp.data.stream = stream;
+ task.tts_inp.data.out_type = response_format == "pcm"
+ ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM
+ : MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
+ // note: -1 is clamped to 0 by the sampler, it will disable the penalty
+ task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512;
+ task.params.sampling.seed = task.tts_inp.data.seed; // same seed for the codec/vocoder RNG
+ if (task.tts_inp.data.top_k > 0) {
+ task.params.sampling.top_k = task.tts_inp.data.top_k;
+ }
+ if (task.tts_inp.data.top_p > 0) {
+ task.params.sampling.top_p = task.tts_inp.data.top_p;
+ }
+
+ // speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64")
+ const unsigned char * speaker_ref_data = nullptr;
+ size_t speaker_ref_len = 0;
+ std::string speaker_ref_b64_decoded;
+
+ auto speaker_ref_file = req.files.find("speaker_ref");
+ if (speaker_ref_file != req.files.end()) {
+ speaker_ref_data = speaker_ref_file->second.data.data();
+ speaker_ref_len = speaker_ref_file->second.data.size();
+ } else {
+ std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string());
+ if (!speaker_ref_b64.empty()) {
+ try {
+ speaker_ref_b64_decoded = base64::decode(speaker_ref_b64);
+ } catch (const std::exception &) {
+ res->error(format_error_response("\"speaker_ref_b64\" is not valid base64", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+ speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data();
+ speaker_ref_len = speaker_ref_b64_decoded.size();
+ }
+ }
+
+ if (speaker_ref_len > 0) {
+ auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt);
+ if (!wrapper.bitmap) {
+ res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+ task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap));
+ } else {
+ SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n");
+ }
+
+ auto & rd = res->rd;
+ task.id = rd.get_new_id();
+ rd.post_task(std::move(task));
+
+ // raw float32 LE mono samples; audio/L16 would imply 16-bit big-endian (RFC 2586)
+ const std::string content_type = response_format == "pcm"
+ ? "audio/pcm;rate=" + std::to_string(info.sample_rate) + ";encoding=float;bits=32"
+ : "audio/wav";
+
+ if (!stream) {
+ auto result = rd.next(req.should_stop);
+ if (!result) {
+ GGML_ASSERT(req.should_stop());
+ return res; // connection is closed
+ }
+ if (result->is_error()) {
+ res->error(result->to_json());
+ return res;
+ }
+ auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
+ GGML_ASSERT(tts_res != nullptr);
+ res->status = 200;
+ res->content_type = content_type;
+ res->data = std::move(tts_res->audio);
+ return res;
+ } else {
+ auto first_result = rd.next(req.should_stop);
+ if (!first_result) {
+ GGML_ASSERT(req.should_stop());
+ return res; // connection is closed
+ }
+ if (first_result->is_error()) {
+ res->error(first_result->to_json());
+ return res;
+ }
+ auto * first_tts_res = dynamic_cast<server_task_result_tts *>(first_result.get());
+ GGML_ASSERT(first_tts_res != nullptr);
+
+ res->status = 200;
+ res->content_type = content_type;
+ res->data = std::move(first_tts_res->audio);
+ bool is_done = first_tts_res->final;
+
+ res->set_next([res_this = res.get(), is_done](std::string & output) mutable -> bool {
+ // flush buffered audio before is_done - the first result can already be final
+ if (!res_this->data.empty()) {
+ output = std::move(res_this->data);
+ res_this->data.clear();
+ return true;
+ }
+ if (is_done) {
+ return false;
+ }
+ if (res_this->should_stop()) {
+ return false;
+ }
+
+ server_response_reader & rd = res_this->rd;
+ if (!rd.has_next()) {
+ return false;
+ }
+ auto result = rd.next([&res_this]() { return res_this->should_stop(); });
+ if (!result || result->is_error()) {
+ return false;
+ }
+ auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
+ GGML_ASSERT(tts_res != nullptr);
+ output = std::move(tts_res->audio);
+ is_done = tts_res->final;
+ return true;
+ });
+ }
+
+ return res;
+ };
+
this->post_systemone = [this](const server_http_req & req) {
auto res = create_response();
const auto & decision = ctx_server.decision;
diff --git a/tools/server/server-context.h b/tools/server/server-context.h
index da75e5327..a579b0eca 100644
--- a/tools/server/server-context.h
+++ b/tools/server/server-context.h
@@ -24,6 +24,8 @@ struct server_context_meta {
bool has_inp_image;
bool has_inp_audio;
bool has_inp_video;
+ bool has_cap_chat;
+ bool has_cap_tts;
bool has_audio_stream; // realtime transcription, see mtmd_support_audio_stream()
json json_ui_settings;
int slot_n_ctx;
@@ -155,6 +157,7 @@ struct server_routes {
server_http_context::handler_t post_embeddings;
server_http_context::handler_t post_embeddings_oai;
server_http_context::handler_t post_rerank;
+ server_http_context::handler_t post_tts;
server_http_context::handler_t post_systemone;
server_http_context::handler_t get_lora_adapters;
server_http_context::handler_t post_lora_adapters;
diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
index c8e235129..d38549db5 100644
--- a/tools/server/server-models.cpp
+++ b/tools/server/server-models.cpp
@@ -521,27 +521,31 @@ void server_model_meta::update_caps(const common_params & base) {
return;
}
- // read the output modalities and the trained context from the GGUF metadata
- std::vector<std::string> output_modalities = {"text"};
- if (!params.model.path.empty()) {
- const common_gguf_info info = common_get_gguf_info(params.model.path);
- output_modalities = server_model_output_modalities(info.decision_type);
- n_ctx_train = info.n_ctx_train;
- }
-
bool inp_image = false;
bool inp_audio = false;
+ bool gen_audio = false;
try {
if (!params.no_mmproj && !params.mmproj.path.empty()) {
mtmd_caps caps = mtmd_get_cap_from_file(params.mmproj.path.c_str());
inp_image = caps.inp_vision;
inp_audio = caps.inp_audio;
+ gen_audio = caps.gen_audio;
}
} catch (const std::exception & e) {
LOG_WRN("failed to read the multimodal capabilities of '%s': %s\n", name.c_str(), e.what());
// keep the output modalities from the GGUF metadata
}
+ // read the output modalities and the trained context from the GGUF metadata
+ std::vector<std::string> output_modalities = {"text"};
+ if (!params.model.path.empty()) {
+ const common_gguf_info info = common_get_gguf_info(params.model.path);
+ // same rule as mtmd_helper_model_can_chat()
+ const bool can_chat = !gen_audio || info.has_chat_tmpl;
+ output_modalities = server_model_output_modalities(info.decision_type, can_chat, gen_audio);
+ n_ctx_train = info.n_ctx_train;
+ }
+
// offline discovery cannot see video; a loaded model reports it
architecture = server_model_architecture_json(inp_image, inp_audio, false, output_modalities);
}
diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp
index 54e126aa0..1e2dce888 100644
--- a/tools/server/server-schema.cpp
+++ b/tools/server/server-schema.cpp
@@ -511,6 +511,58 @@ task_params eval_llama_cmpl_schema(
return params;
}
+//
+// TTS schema
+//
+
+std::vector<std::unique_ptr<field>> make_tts_schema(server_task & task) {
+ std::vector<std::unique_ptr<field>> fields;
+ auto add = [&](field * f) {
+ fields.emplace_back(f);
+ };
+
+ add((new field_num("top_k", task.tts_inp.data.top_k))
+ ->set_limits(0, INT32_MAX)
+ ->set_desc("Top-k for the acoustic code predictor, 0 to use the model default"));
+
+ add((new field_num("top_p", task.tts_inp.data.top_p))
+ ->set_limits(0.0f, 1.0f)
+ ->set_desc("Top-p for the acoustic code predictor, 0.0 to use the model default"));
+
+ add((new field_num("seed", task.tts_inp.data.seed))
+ ->set_desc("RNG seed for the backbone sampler and the codec/vocoder (-1 = random)"));
+
+ add((new field_num("n_predict", task.params.n_predict))
+ ->set_hard_limits(-1, INT32_MAX)
+ ->add_alias("max_tokens")
+ ->set_desc("Max number of audio frames to generate"));
+
+ add((new field_num("repeat_penalty", task.params.sampling.penalty_repeat))
+ ->set_desc("Repetition penalty applied to the backbone sampler over the whole generation"));
+
+ add((new field_bool("stream", task.params.stream))
+ ->set_desc("Stream the audio as it becomes available instead of waiting for the full generation"));
+
+ return fields;
+}
+
+void eval_tts_schema(const common_params & params_base, server_task & task, const json & data) {
+ // baseline defaults, individual requests can override them
+ // TODO @ngxson : change the default values based on model
+ task.params.sampling = params_base.sampling;
+ task.params.sampling.penalty_repeat = 1.05f;
+ task.params.n_predict = -1;
+ task.tts_inp.data.top_k = 0;
+ task.tts_inp.data.top_p = 0.0f;
+ task.tts_inp.data.seed = params_base.sampling.seed;
+
+ field_eval_context ctx(task.params);
+
+ for (const auto & f : make_tts_schema(task)) {
+ f->eval(ctx, data);
+ }
+}
+
//
// eval() implementations
//
diff --git a/tools/server/server-schema.h b/tools/server/server-schema.h
index d0a81431b..542136a97 100644
--- a/tools/server/server-schema.h
+++ b/tools/server/server-schema.h
@@ -101,4 +101,12 @@ task_params eval_llama_cmpl_schema(
const std::vector<llama_logit_bias> & logit_bias_eog,
const json & data);
+std::vector<std::unique_ptr<field>> make_tts_schema(server_task & task);
+
+// evaluates the /tts request params into task.params and task.tts_inp.data
+void eval_tts_schema(
+ const common_params & params_base,
+ server_task & task,
+ const json & data);
+
} // namespace server_schema
diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
index 2e513a33e..d7ac2a1dc 100644
--- a/tools/server/server-task.cpp
+++ b/tools/server/server-task.cpp
@@ -1508,6 +1508,17 @@ json server_task_result_rerank::to_json() {
};
}
+//
+// server_task_result_tts
+//
+json server_task_result_tts::to_json() {
+ return json {
+ {"sample_rate", sample_rate},
+ {"n_bytes", audio.size()},
+ {"final", final},
+ };
+}
+
//
// server_task_result_decision
//
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index de39ce3b8..810cefbfe 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -10,6 +10,7 @@
// TODO: prevent including the whole server-common.h as we only use server_tokens
#include "server-common.h"
+#include "mtmd-helper.h"
enum server_task_type {
@@ -29,6 +30,7 @@ enum server_task_type {
SERVER_TASK_TYPE_SLOT_ERASE,
SERVER_TASK_TYPE_GET_LORA,
SERVER_TASK_TYPE_SET_LORA,
+ SERVER_TASK_TYPE_TTS,
};
// TODO: change this to more generic "response_format" to replace the "format_response_*" in server-common
@@ -219,6 +221,9 @@ struct server_task {
// used by SERVER_TASK_TYPE_SET_LORA
std::map<int, float> set_lora; // mapping adapter ID -> scale
+ // used by SERVER_TASK_TYPE_TTS
+ mtmd_helper::gen_audio::inp tts_inp;
+
server_task() = default;
server_task(server_task_type type) : type(type) {}
@@ -256,6 +261,7 @@ struct server_task {
switch (type) {
case SERVER_TASK_TYPE_COMPLETION:
case SERVER_TASK_TYPE_INFILL:
+ case SERVER_TASK_TYPE_TTS:
case SERVER_TASK_TYPE_TRANSCRIBE_RT:
return true;
default:
@@ -526,6 +532,16 @@ struct server_task_result_embd : server_task_result {
json to_json_oaicompat();
};
+struct server_task_result_tts : server_task_result {
+ std::string audio; // raw bytes for this chunk (WAV or PCM, per request's out_type)
+ int32_t sample_rate = 0;
+ bool final = false; // true for the last chunk of a request
+
+ virtual bool is_stop() override { return final; }
+
+ virtual json to_json() override;
+};
+
struct server_task_result_rerank : server_task_result {
float score = -1e6;
diff --git a/tools/server/server.cpp b/tools/server/server.cpp
index 41f29ded5..ce44405b8 100644
--- a/tools/server/server.cpp
+++ b/tools/server/server.cpp
@@ -247,6 +247,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
routes.post_control = models_routes->proxy_post;
routes.post_responses_oai = models_routes->proxy_post;
routes.post_transcriptions_oai = models_routes->proxy_post;
+ routes.post_tts = models_routes->proxy_post;
routes.ws_realtime = models_routes->proxy_ws;
routes.post_anthropic_messages = models_routes->proxy_post;
routes.post_anthropic_count_tokens = models_routes->proxy_post;
@@ -293,6 +294,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
ctx_http.post("/responses", ex_wrapper(routes.post_responses_oai));
ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));
ctx_http.post("/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));
+ ctx_http.post("/tts", ex_wrapper(routes.post_tts));
ctx_http.ws ("/v1/realtime", routes.ws_realtime);
ctx_http.post("/v1/messages", ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API
ctx_http.post("/infill", ex_wrapper(routes.post_infill));
diff --git a/tools/tts/README.md b/tools/tts/README.md
index 1b08d5ef3..55b99932c 100644
--- a/tools/tts/README.md
+++ b/tools/tts/README.md
@@ -12,6 +12,8 @@ Simple usage:
llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF -p "Hello world" --output out.wav
```
+Note: it's highly recommended to always provide a speaker reference voice (via `--tts-speaker-file`); otherwise, the model's performance may be degraded.
+
Common params:
- Sampling params such as `--top-k`, `--top-p`, `--temp`, etc.
- `-n <number_of_frames>` limits the output length, e.g. `-n 500`. Note that how many milliseconds each frame represents varies by model
diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp
index 6fd193632..aea794e65 100644
--- a/tools/tts/tts.cpp
+++ b/tools/tts/tts.cpp
@@ -111,8 +111,9 @@ int main(int argc, char ** argv) {
speaker_bitmap.reset(wrapper.bitmap);
}
- mtmd_helper::gen_audio gen(lctx, mctx.get());
- mtmd_helper_gen_audio_inp inp{};
+ mtmd_helper::gen_audio gen;
+ gen.init(lctx, mctx.get());
+ mtmd_helper_gen_audio_inp inp = mtmd_helper_gen_audio_inp_default();
inp.seq_id = 0;
inp.prompt = params.prompt.c_str();
inp.prompt_len = params.prompt.size();
@@ -121,7 +122,6 @@ int main(int argc, char ** argv) {
inp.top_k = params.sampling.top_k;
inp.top_p = params.sampling.top_p;
inp.seed = params.sampling.seed;
- inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
//
// stage 1: process prompt via backbone model, generate semantic representation