Commit efa28e950 for llama.cpp
commit efa28e950ea3a41648aaf354b3a743dd4708f954
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Sat Sep 19 11:27:46 2026 +0300
test-llama-archs : generate dummy test vocab (#29084)
Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp
diff --git a/include/llama.h b/include/llama.h
index ac2215dc7..31bbf8b0d 100644
--- a/include/llama.h
+++ b/include/llama.h
@@ -77,6 +77,7 @@ extern "C" {
LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram
LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization
LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming
+ LLAMA_VOCAB_TYPE_TEST = 7, // Dummy tokenizer for testing: rolling hash of fixed-size chunks -> tokens, tokens -> hex
};
enum llama_rope_type {
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index 0f5155b2e..39160a417 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -387,13 +387,13 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_TOKENIZER_SCORES, scores);
add_kv(LLM_KV_TOKENIZER_MERGES, vocab.get_bpe_merges());
// FIXME llama_token is type i32 but when reading in a GGUF file u32 is expected, not an issue for writing though
- add_kv(LLM_KV_TOKENIZER_BOS_ID, uint32_t(vocab.token_bos()));
- add_kv(LLM_KV_TOKENIZER_EOS_ID, uint32_t(vocab.token_eos()));
- add_kv(LLM_KV_TOKENIZER_EOT_ID, uint32_t(vocab.token_eot()));
- add_kv(LLM_KV_TOKENIZER_EOM_ID, uint32_t(vocab.token_eom()));
- add_kv(LLM_KV_TOKENIZER_UNK_ID, uint32_t(vocab.token_unk()));
- add_kv(LLM_KV_TOKENIZER_SEP_ID, uint32_t(vocab.token_sep()));
- add_kv(LLM_KV_TOKENIZER_PAD_ID, uint32_t(vocab.token_pad()));
+ if (vocab.token_bos() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_BOS_ID, uint32_t(vocab.token_bos())); }
+ if (vocab.token_eos() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOS_ID, uint32_t(vocab.token_eos())); }
+ if (vocab.token_eot() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOT_ID, uint32_t(vocab.token_eot())); }
+ if (vocab.token_eom() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_EOM_ID, uint32_t(vocab.token_eom())); }
+ if (vocab.token_unk() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_UNK_ID, uint32_t(vocab.token_unk())); }
+ if (vocab.token_sep() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_SEP_ID, uint32_t(vocab.token_sep())); }
+ if (vocab.token_pad() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_PAD_ID, uint32_t(vocab.token_pad())); }
// add_kv(LLM_KV_TOKENIZER_CLS_ID, uint32_t(vocab.token_bos())); // deprecated
// add_kv(LLM_KV_TOKENIZER_MASK_ID, ???);
add_kv(LLM_KV_TOKENIZER_ADD_BOS, vocab.get_add_bos());
@@ -404,12 +404,12 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, vocab.get_precompiled_charsmap());
// add_kv(LLM_KV_TOKENIZER_HF_JSON, ???);
// add_kv(LLM_KV_TOKENIZER_RWKV, ???);
- add_kv(LLM_KV_TOKENIZER_FIM_PRE_ID, uint32_t(vocab.token_fim_pre()));
- add_kv(LLM_KV_TOKENIZER_FIM_SUF_ID, uint32_t(vocab.token_fim_suf()));
- add_kv(LLM_KV_TOKENIZER_FIM_MID_ID, uint32_t(vocab.token_fim_mid()));
- add_kv(LLM_KV_TOKENIZER_FIM_PAD_ID, uint32_t(vocab.token_fim_pad()));
- add_kv(LLM_KV_TOKENIZER_FIM_REP_ID, uint32_t(vocab.token_fim_rep()));
- add_kv(LLM_KV_TOKENIZER_FIM_SEP_ID, uint32_t(vocab.token_fim_sep()));
+ if (vocab.token_fim_pre() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_PRE_ID, uint32_t(vocab.token_fim_pre())); }
+ if (vocab.token_fim_suf() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_SUF_ID, uint32_t(vocab.token_fim_suf())); }
+ if (vocab.token_fim_mid() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_MID_ID, uint32_t(vocab.token_fim_mid())); }
+ if (vocab.token_fim_pad() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_PAD_ID, uint32_t(vocab.token_fim_pad())); }
+ if (vocab.token_fim_rep() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_REP_ID, uint32_t(vocab.token_fim_rep())); }
+ if (vocab.token_fim_sep() != LLAMA_TOKEN_NULL) { add_kv(LLM_KV_TOKENIZER_FIM_SEP_ID, uint32_t(vocab.token_fim_sep())); }
// TODO: implement LoRA support
// add_kv(LLM_KV_ADAPTER_TYPE, ???);
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index 737e07275..e038637ce 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -2087,6 +2087,16 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
special_unk_id = LLAMA_TOKEN_NULL;
special_sep_id = LLAMA_TOKEN_NULL;
special_pad_id = LLAMA_TOKEN_NULL;
+ } else if (tokenizer_model == "test") {
+ type = LLAMA_VOCAB_TYPE_TEST;
+
+ // default special tokens
+ special_bos_id = LLAMA_TOKEN_NULL;
+ special_eos_id = LLAMA_TOKEN_NULL;
+ special_unk_id = LLAMA_TOKEN_NULL;
+ special_sep_id = LLAMA_TOKEN_NULL;
+ special_pad_id = LLAMA_TOKEN_NULL;
+ special_mask_id = LLAMA_TOKEN_NULL;
} else if (tokenizer_model == "plamo2") {
type = LLAMA_VOCAB_TYPE_PLAMO2;
@@ -3134,6 +3144,7 @@ std::string llama_vocab::impl::type_name() const{
case LLAMA_VOCAB_TYPE_UGM: return "UGM";
case LLAMA_VOCAB_TYPE_RWKV: return "RWKV";
case LLAMA_VOCAB_TYPE_PLAMO2: return "PLaMo2";
+ case LLAMA_VOCAB_TYPE_TEST: return "TEST";
default: return "unknown";
}
}
@@ -3222,6 +3233,9 @@ void llama_vocab::impl::init_tokenizer(enum llama_vocab_type type) {
case LLAMA_VOCAB_TYPE_PLAMO2:
tokenizer = std::make_unique<llm_tokenizer_plamo2>(vocab);
break;
+ case LLAMA_VOCAB_TYPE_TEST:
+ tokenizer = std::make_unique<llm_tokenizer>();
+ break;
default:
GGML_ABORT("unsupported vocab type");
}
@@ -3595,6 +3609,42 @@ std::vector<llama_token> llama_vocab::impl::tokenize(
}
}
} break;
+ case LLAMA_VOCAB_TYPE_TEST:
+ {
+ const uint32_t n_vocab = vocab.n_tokens();
+ constexpr size_t chunk_size = 5;
+
+ // reserve output to avoid repeated reallocations
+ size_t n_tokens = 0;
+ for (const auto & fragment : fragment_buffer) {
+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
+ n_tokens += (fragment.length + chunk_size - 1) / chunk_size;
+ } else {
+ ++n_tokens;
+ }
+ }
+ output.reserve(output.size() + n_tokens);
+
+ for (const auto & fragment : fragment_buffer) {
+ if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
+ const auto & text = fragment.raw_text;
+ const size_t begin = fragment.offset;
+ const size_t end = begin + fragment.length;
+ size_t pos = begin;
+ while (pos < end) {
+ const size_t n = std::min(chunk_size, end - pos);
+ uint64_t hash = 0;
+ for (size_t i = 0; i < n; ++i) {
+ hash = hash*31 + (uint8_t) text[pos + i];
+ }
+ output.push_back((llama_token)(hash % n_vocab));
+ pos += n;
+ }
+ } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
+ output.push_back(fragment.token);
+ }
+ }
+ } break;
case LLAMA_VOCAB_TYPE_NONE:
GGML_ABORT("fatal error");
}
@@ -3693,6 +3743,11 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t
memcpy(buf, result.data(), result.size());
return (int)result.size();
}
+ case LLAMA_VOCAB_TYPE_TEST: {
+ // tokens -> text: simply stringify the token id in hex
+ std::string result = format("%x", token);
+ return _try_copy(result.data(), result.size());
+ }
case LLAMA_VOCAB_TYPE_PLAMO2: {
// PLaMo-2 uses similar token handling as BPE/SPM
if (vocab.is_byte(token)) {
@@ -3963,6 +4018,9 @@ llama_token llama_vocab::byte_to_token(uint8_t ch) const {
snprintf(hex_str, sizeof(hex_str), "<0x%02X>", ch);
return pimpl->token_to_id.at(hex_str);
}
+ case LLAMA_VOCAB_TYPE_TEST:
+ // TEST tokens have no byte-level mapping
+ return LLAMA_TOKEN_NULL;
default:
GGML_ABORT("fatal error");
}
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 568f7234c..f848fc139 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -338,7 +338,19 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f);
}
- ms.add_kv(LLM_KV_TOKENIZER_MODEL, "no_vocab");
+ // dummy tokenizer: token ids are derived from fixed-size chunks and detokenized as hex ids
+ {
+ std::vector<std::string> tokenizer_list(n_vocab);
+ std::vector<float> tokenizer_scores(n_vocab, 0.0f);
+
+ ms.add_kv(LLM_KV_TOKENIZER_MODEL, "test");
+ for (uint32_t i = 0; i < n_vocab; i++) {
+ tokenizer_list[i] = "tok_" + std::to_string(i);
+ }
+ ms.add_kv(LLM_KV_TOKENIZER_LIST, tokenizer_list);
+ ms.add_kv(LLM_KV_TOKENIZER_SCORES, tokenizer_scores);
+ }
+
// ms.add_kv(LLM_KV_DENSE_2_FEAT_OUT, n_embd);
// ms.add_kv(LLM_KV_DENSE_3_FEAT_IN, n_embd);