Commit b1c902f02 for llama.cpp
commit b1c902f026d32535b5e8eae55e5e47b379e57672
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Thu Nov 5 20:09:33 2026 +0100
llama, mtmd: (chatterbox) add use_classifier_free_guidance as gguf metadata (#31698)
* llama, mtmd: add use_cfg as gguf metadata
* use_classifier_free_guidance
diff --git a/common/common.cpp b/common/common.cpp
index ba4a3e892..a2801b82f 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1313,6 +1313,12 @@ common_init_result::common_init_result(common_params & params, bool model_only)
LOG_INF("%s", "decision model reads the embeddings output, enabling embedding mode\n");
}
+ if (llama_model_classifier_free_guidance(model)) {
+ params.kv_unified = true;
+ cparams.kv_unified = true;
+ cparams.n_seq_max *= 2;
+ }
+
// embeddings need the whole batch in one ubatch, so n_batch must not be larger than n_ubatch
// (server.cpp does this check for --embedding, but before the model is loaded)
if (cparams.embeddings && cparams.n_batch > cparams.n_ubatch) {
diff --git a/conversion/chatterbox.py b/conversion/chatterbox.py
index 4ab93860c..bca0a2633 100644
--- a/conversion/chatterbox.py
+++ b/conversion/chatterbox.py
@@ -212,6 +212,8 @@ class ChatterboxTalkerModel(TextModel):
self.gguf_writer.add_sampling_temp(sampling["temp"])
self.gguf_writer.add_sampling_penalty_repeat(sampling["penalty_repeat"])
self.gguf_writer.add_sampling_penalty_last_n(PENALTY_LAST_N)
+ if not self.is_turbo:
+ self.gguf_writer.add_classifier_free_guidance(True)
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
if self.is_turbo:
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 170df4e6f..69af1b909 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -178,6 +178,7 @@ class Keys:
NORM_BEFORE_RESIDUAL = "{arch}.norm_before_residual"
NORM_BEFORE_FC = "{arch}.norm_before_fc"
REALTIME_DELAY_TOKENS = "{arch}.realtime_delay_tokens"
+ CLASSIFIER_FREE_GUIDANCE = "{arch}.classifier_free_guidance"
class Adapters:
COUNT = "{arch}.adapters.count"
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index f6b98a57d..21b354ca4 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1202,6 +1202,9 @@ class GGUFWriter:
def add_realtime_delay_tokens(self, value: int) -> None:
self.add_uint32(Keys.LLM.REALTIME_DELAY_TOKENS.format(arch=self.arch), value)
+ def add_classifier_free_guidance(self, value: bool) -> None:
+ self.add_bool(Keys.LLM.CLASSIFIER_FREE_GUIDANCE.format(arch=self.arch), value)
+
def add_pooling_type(self, value: PoolingType) -> None:
self.add_uint32(Keys.LLM.POOLING_TYPE.format(arch=self.arch), value.value)
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 265c2ddeb..0702fc72d 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -315,6 +315,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_ATTENTION_TEMPERATURE_LENGTH, "%s.attention.temperature_length" },
{ LLM_KV_ATTENTION_TEMPERATURE_SCALE, "%s.attention.temperature_scale" },
{ LLM_KV_REALTIME_DELAY_TOKENS, "%s.realtime_delay_tokens" },
+ { LLM_KV_CLASSIFIER_FREE_GUIDANCE, "%s.classifier_free_guidance" },
{ LLM_KV_ATTENTION_KEY_LENGTH_MLA, "%s.attention.key_length_mla" },
{ LLM_KV_ATTENTION_VALUE_LENGTH_MLA, "%s.attention.value_length_mla" },
{ LLM_KV_ATTENTION_KEY_LENGTH_SWA, "%s.attention.key_length_swa" },
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 7bf0d3ae5..225803cb3 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -285,6 +285,7 @@ enum llm_kv {
LLM_KV_METIS_ROUTE_LENGTH,
LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT,
LLM_KV_REALTIME_DELAY_TOKENS,
+ LLM_KV_CLASSIFIER_FREE_GUIDANCE,
LLM_KV_MUDD_FEED_FORWARD_LENGTH,
LLM_KV_MUDD_TAP_COUNT,
LLM_KV_MUDD_TAP_INDICES,
diff --git a/src/llama-ext.h b/src/llama-ext.h
index 23a781404..b55c0c357 100644
--- a/src/llama-ext.h
+++ b/src/llama-ext.h
@@ -202,3 +202,6 @@ LLAMA_API uint32_t llama_model_get_tok_embd_rows(const struct llama_model * mode
// realtime models: number of tokens the output lags behind the audio input, used by mtmd; 0 if not applicable
LLAMA_API uint32_t llama_model_n_tokens_delay(const struct llama_model * model);
+
+// each sequence is paired with an unconditional one at n_seq_max / 2 + seq_id, in a unified KV cache
+LLAMA_API bool llama_model_classifier_free_guidance(const struct llama_model * model);
diff --git a/src/llama-hparams.h b/src/llama-hparams.h
index 2f7b97823..7b738407d 100644
--- a/src/llama-hparams.h
+++ b/src/llama-hparams.h
@@ -304,6 +304,9 @@ struct llama_hparams {
// realtime models: number of tokens the output lags behind the audio input
uint32_t n_tokens_delay = 0;
+ // each sequence is paired with an unconditional one
+ bool classifier_free_guidance = false;
+
// gemma3n altup
uint32_t n_altup = 4; // altup_num_inputs
uint32_t i_altup_act = 0; // altup_active_idx
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 27809ccad..8c3ef97f7 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -1476,6 +1476,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false);
ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
ml.get_key(LLM_KV_CLASSIFIER_POOLING_TYPE, hparams.pooling_type_cls, false);
+ ml.get_key(LLM_KV_CLASSIFIER_FREE_GUIDANCE, hparams.classifier_free_guidance, false);
std::string act_cls;
if (ml.get_key(LLM_KV_CLASSIFIER_ACTIVATION, act_cls, false)) {
const auto it = LLM_CLS_ACT_TYPES_FROM_STRING.find(act_cls);
@@ -4123,6 +4124,10 @@ uint32_t llama_model_n_tokens_delay(const struct llama_model * model) {
return model->hparams.n_tokens_delay;
}
+bool llama_model_classifier_free_guidance(const struct llama_model * model) {
+ return model->hparams.classifier_free_guidance;
+}
+
uint32_t llama_model_get_tok_embd_rows(const struct llama_model * model, const llama_token * tokens, uint32_t n_tokens, float * out) {
if (model->vocab.n_tokens() == 0 || model->tok_embd == nullptr || tokens == nullptr || out == nullptr) {
return 0;
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index ab5cbd424..d776ed1e8 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -1730,8 +1730,8 @@ public:
// the uncond sequence lives half a seq space away from the cond one
const int32_t n_seq_max = (int32_t) llama_n_seq_max(lctx);
seq_uncond = (llama_seq_id) (n_seq_max / 2 + seq_id);
- if (n_seq_max < 2 || seq_id >= n_seq_max / 2) {
- LOG_ERR("mtmd_helper_gen_audio: multilingual chatterbox needs a paired cfg sequence "
+ if (!llama_model_classifier_free_guidance(llama_get_model(lctx)) || seq_id >= n_seq_max / 2) {
+ LOG_ERR("mtmd_helper_gen_audio: multilingual chatterbox needs a backbone with classifier_free_guidance and a paired cfg sequence "
"(n_seq_max = %d, seq_id = %d)\n", n_seq_max, seq_id);
return 1;
}
diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp
index 6354be6d6..8578f8869 100644
--- a/tools/tts/tts.cpp
+++ b/tools/tts/tts.cpp
@@ -69,11 +69,6 @@ int main(int argc, char ** argv) {
// always enable embd, so that we can pass hidden states to the audio generation helper
params.embedding = true;
- // for chatterbox multilingual: its cfg pairs slot i with sequence n_parallel + i in a unified kv cache
- // TODO: this applies to every model, find a better way for a pipeline to request extra sequences
- params.n_parallel *= 2;
- params.kv_unified = true;
-
llama_backend_init();
llama_numa_init(params.numa);