Commit 4c3c5810f for llama.cpp
commit 4c3c5810f87efe177949f1b85ae509d38a2a311f
Author: Kashif Rasul <kashif.rasul@gmail.com>
Date: Mon Oct 19 23:01:40 2026 +0200
hybrid dna tokenizer fixes (#30825)
* vocab : support Carbon-14B HybridDNA tokenizer
Carbon-14B auto-tags untagged DNA runs and uses N-padded tail tokens
(tail=pad_n). Also strip whitespace inside <dna> tags and handle a stray
</dna> like the HF tokenizer.
* convert : fix special tokens and merges for older Carbon models
Carbon-500M/3B/8B don't register the Qwen added tokens and ship no
tokenizer.json, so take the token types, merges and special ids from the
base tokenizer.
* Update conversion/base.py
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
* Update src/llama-vocab.cpp
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
* Update src/llama-vocab.cpp
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
* Update src/llama-vocab.cpp
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
* Update conversion/base.py
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
* Update conversion/base.py
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
* Update conversion/base.py
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
* address review comments
* Update src/llama-vocab.cpp
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
---------
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
diff --git a/conversion/base.py b/conversion/base.py
index 14a52f01d..dec1d4322 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -10,6 +10,7 @@ import json
import os
import re
import sys
+import tempfile
from enum import IntEnum
from pathlib import Path
from hashlib import sha256
@@ -2169,20 +2170,34 @@ class TextModel(ModelBase):
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
special_vocab.add_to_gguf(self.gguf_writer)
+ def _is_hybriddna(self) -> bool:
+ tokenizer_config_file = self.dir_model / "tokenizer_config.json"
+ if not tokenizer_config_file.is_file():
+ return False
+ with open(tokenizer_config_file, "r", encoding="utf-8") as f:
+ return json.load(f).get("tokenizer_class") == "HybridDNATokenizer"
+
def _set_vocab_hybriddna(self):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
- vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab)) # ty: ignore[unresolved-attribute]
- assert max(tokenizer.vocab.values()) < vocab_size # ty: ignore[unresolved-attribute]
+ vocab = tokenizer.get_vocab()
+ vocab_size = self.hparams.get("vocab_size", len(vocab))
+ assert max(vocab.values()) < vocab_size
- reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} # ty: ignore[unresolved-attribute]
- # k-mers can share text with a base-vocab BPE token (e.g. CCCCCC) and get
- # dropped by get_vocab(); a reserved marker suffix (U+E000) keeps each
- # k-mer's own id (llama.cpp strips it on detokenization)
- for kmer in tokenizer.kmers: # ty: ignore[unresolved-attribute]
+ reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab.items()}
+ # k-mers can share text with a base-vocab BPE token (e.g. CCCCCC) and get dropped by get_vocab()
+ # a reserved marker suffix (U+E000) keeps each k-mer's own id (llama.cpp strips it on detokenization)
+ # tail="pad_n" (Carbon-14B) adds N-padded tail tokens ("CT" -> "CTNNNN") that get the same marker
+ # llama.cpp detects them to pad with N and strips the padding on detokenization, tail="char" is unsupported
+ tail = getattr(tokenizer, "tail", "pad_a")
+ if tokenizer.k != 6 or tail not in ("pad_a", "pad_n"):
+ raise NotImplementedError(f"HybridDNATokenizer with k={tokenizer.k}, tail={tail!r}")
+ for kmer in [*tokenizer.kmers, *getattr(tokenizer, "npad_tokens", [])]:
reverse_vocab[tokenizer.dna_token_to_id[kmer]] = kmer + "\ue000" # ty: ignore[unresolved-attribute]
- added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute]
- added_tokens_decoder = tokenizer.added_tokens_decoder # ty: ignore[unresolved-attribute]
+ # older HybridDNATokenizer versions (Carbon-500M/3B/8B) do not register the base tokenizer's added tokens (<|im_start|>, <think>, ...) as their own
+ # but they are still encoded as special tokens: take their types from the base tokenizer
+ base_tokenizer = getattr(tokenizer, "_base_tokenizer", tokenizer)
+ added_tokens_decoder = {**base_tokenizer.added_tokens_decoder, **tokenizer.added_tokens_decoder} # ty: ignore[unresolved-attribute]
tokens: list[str] = []
toktypes: list[int] = []
@@ -2192,7 +2207,7 @@ class TextModel(ModelBase):
toktypes.append(gguf.TokenType.UNUSED)
else:
token: str = reverse_vocab[i]
- if token in added_vocab:
+ if i in added_tokens_decoder:
if added_tokens_decoder[i].special or self.does_token_look_special(token):
toktypes.append(gguf.TokenType.CONTROL)
else:
@@ -2204,10 +2219,19 @@ class TextModel(ModelBase):
tokpre = self.get_vocab_base_pre(tokenizer)
self.gguf_writer.add_tokenizer_model("hybriddna")
self.gguf_writer.add_tokenizer_pre(tokpre)
+ self.gguf_writer.add_dna_auto_tags(getattr(tokenizer, "auto_dna_tags", False))
self.gguf_writer.add_token_list(tokens)
self.gguf_writer.add_token_types(toktypes)
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
+ if not special_vocab.merges:
+ # Carbon-500M/3B/8B ship only tokenizer.py, no tokenizer.json
+ # take the merges and special token ids from the base BPE tokenizer, keeping the model's own chat template (if any)
+ with tempfile.TemporaryDirectory() as tmp:
+ base_tokenizer.save_pretrained(tmp)
+ base_special_vocab = gguf.SpecialVocab(Path(tmp), load_merges=True)
+ base_special_vocab.chat_template = special_vocab.chat_template
+ special_vocab = base_special_vocab
special_vocab.add_to_gguf(self.gguf_writer)
def _set_vocab_qwen(self):
diff --git a/conversion/llama.py b/conversion/llama.py
index 41d8c2309..b04f54d41 100644
--- a/conversion/llama.py
+++ b/conversion/llama.py
@@ -129,8 +129,9 @@ class LlamaModel(TextModel):
tokenizer_config_json = json.load(f)
if (add_prefix_space := tokenizer_config_json.get("add_prefix_space")) is not None:
self.gguf_writer.add_add_space_prefix(add_prefix_space)
- if tokenizer_config_json.get("tokenizer_class") == "HybridDNATokenizer":
- return self._set_vocab_hybriddna()
+
+ if self._is_hybriddna():
+ return self._set_vocab_hybriddna()
try:
self._set_vocab_sentencepiece()
diff --git a/conversion/qwen.py b/conversion/qwen.py
index 0c0d0c1ed..e1013dd53 100644
--- a/conversion/qwen.py
+++ b/conversion/qwen.py
@@ -59,6 +59,10 @@ class Qwen2Model(TextModel):
model_arch = gguf.MODEL_ARCH.QWEN2
def set_vocab(self):
+ # HuggingFaceBio/Carbon-14B: Qwen3 with the HybridDNATokenizer (Qwen3 BPE plus DNA 6-mers)
+ if self._is_hybriddna():
+ self._set_vocab_hybriddna()
+ return
try:
self._set_vocab_sentencepiece()
except FileNotFoundError:
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index e098efae4..8c19e22da 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -366,6 +366,7 @@ class Keys:
ADD_BOS = "tokenizer.ggml.add_bos_token"
ADD_EOS = "tokenizer.ggml.add_eos_token"
ADD_SEP = "tokenizer.ggml.add_sep_token"
+ DNA_AUTO_TAGS = "tokenizer.ggml.dna_auto_tags"
ADD_PREFIX = "tokenizer.ggml.add_space_prefix"
REMOVE_EXTRA_WS = "tokenizer.ggml.remove_extra_whitespaces"
PRECOMPILED_CHARSMAP = "tokenizer.ggml.precompiled_charsmap"
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index 93ec95eb6..7becd74af 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1314,6 +1314,9 @@ class GGUFWriter:
def add_add_sep_token(self, value: bool) -> None:
self.add_bool(Keys.Tokenizer.ADD_SEP, value)
+ def add_dna_auto_tags(self, value: bool) -> None:
+ self.add_bool(Keys.Tokenizer.DNA_AUTO_TAGS, value)
+
def add_add_space_prefix(self, value: bool) -> None:
self.add_bool(Keys.Tokenizer.ADD_PREFIX, value)
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index ffa35bdea..961011a2c 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -430,6 +430,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" },
{ LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" },
{ LLM_KV_TOKENIZER_ADD_SEP, "tokenizer.ggml.add_sep_token" },
+ { LLM_KV_TOKENIZER_DNA_AUTO_TAGS, "tokenizer.ggml.dna_auto_tags" },
{ LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" },
{ LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" },
{ LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" },
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 14ce29ab3..3ea86c601 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -394,6 +394,7 @@ enum llm_kv {
LLM_KV_TOKENIZER_ADD_BOS,
LLM_KV_TOKENIZER_ADD_EOS,
LLM_KV_TOKENIZER_ADD_SEP,
+ LLM_KV_TOKENIZER_DNA_AUTO_TAGS,
LLM_KV_TOKENIZER_ADD_PREFIX,
LLM_KV_TOKENIZER_REMOVE_EXTRA_WS,
LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP,
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index 05d2527dc..a237a4b04 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -1776,16 +1776,15 @@ private:
// text at load
static const std::string dna_kmer_marker = "\xee\x80\x80";
+// HybridDNATokenizer (HuggingFaceBio/Carbon-*): text inside <dna>...</dna> is cut into fixed 6-mers, everything else goes through the base BPE
+// Mirrors HybridDNATokenizer.encode, with optional tagging of untagged DNA (auto_dna_tags, Carbon-14B) and tail="pad_n" vocab, whose dedicated tokens pad a trailing partial k-mer with N instead of A ("CT" -> "CTNNNN")
struct llm_tokenizer_hybriddna_session : llm_tokenizer_bpe_session {
llm_tokenizer_hybriddna_session(const llama_vocab & vocab, const llm_tokenizer_bpe & tokenizer) : llm_tokenizer_bpe_session{vocab, tokenizer}, vocab{vocab} {}
void tokenize(const std::string & text, std::vector<llama_token> & output) override {
- static const std::string open_tag = "<dna>";
- static const std::string close_tag = "</dna>";
-
- const auto dna_begin_id = vocab.text_to_token(open_tag);
- const auto dna_end_id = vocab.text_to_token(close_tag);
- const auto dna_oov_id = vocab.text_to_token("<oov>");
+ dna_begin_id = vocab.text_to_token(open_tag);
+ dna_end_id = vocab.text_to_token(close_tag);
+ dna_oov_id = vocab.text_to_token("<oov>");
// Fall back to plain BPE if the DNA pieces aren't in the vocab.
if (dna_begin_id == LLAMA_TOKEN_NULL || dna_end_id == LLAMA_TOKEN_NULL || dna_oov_id == LLAMA_TOKEN_NULL) {
@@ -1793,63 +1792,187 @@ struct llm_tokenizer_hybriddna_session : llm_tokenizer_bpe_session {
return;
}
- const size_t k = 6;
- size_t pos = 0;
+ pad_char = vocab.text_to_token("A" + std::string(k - 1, 'N') + dna_kmer_marker) != LLAMA_TOKEN_NULL ? 'N' : 'A';
+ // split on explicit tags as HybridDNATokenizer._split_by_dna_tags does; only the text outside them is auto-tagged
+ size_t pos = 0;
while (pos < text.size()) {
const size_t start = text.find(open_tag, pos);
- if (start == std::string::npos) {
- if (pos < text.size()) {
- llm_tokenizer_bpe_session::tokenize(text.substr(pos), output);
- }
+ const size_t end = text.find(close_tag, pos);
+
+ if (start == std::string::npos && end == std::string::npos) {
+ tokenize_text(text.substr(pos), output);
break;
}
- if (start > pos) {
- llm_tokenizer_bpe_session::tokenize(text.substr(pos, start - pos), output);
+ if (start == std::string::npos || end < start) {
+ // as in _split_by_dna_tags, text before a </dna> without a matching <dna> is DNA (no begin token)
+ tokenize_dna(text.substr(pos, end - pos), false, true, output);
+ pos = end + close_tag.size();
+ continue;
}
- output.push_back(dna_begin_id);
- const size_t content_start = start + open_tag.size();
- const size_t end = text.find(close_tag, content_start);
- const size_t content_end = (end == std::string::npos) ? text.size() : end;
-
- emit_dna_kmers(text.substr(content_start, content_end - content_start), k, dna_oov_id, output);
+ tokenize_text(text.substr(pos, start - pos), output);
+ const size_t content_start = start + open_tag.size();
if (end == std::string::npos) {
+ // an unterminated <dna> runs to the end of the text (no end token)
+ tokenize_dna(text.substr(content_start), true, false, output);
break;
}
- output.push_back(dna_end_id);
+ tokenize_dna(text.substr(content_start, end - content_start), true, true, output);
pos = end + close_tag.size();
}
}
private:
- void emit_dna_kmers(const std::string & raw, size_t k, llama_token oov_id, std::vector<llama_token> & output) {
- std::string seq = raw;
- for (char & c : seq) {
- if (c >= 'a' && c <= 'z') {
- c = char(c - 32);
+ static bool is_letter(char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); }
+ static bool is_acgt (char c) { return c == 'A' || c == 'C' || c == 'G' || c == 'T'; }
+ static char to_upper (char c) { return c >= 'a' && c <= 'z' ? char(c - 32) : c; }
+
+ // IUPAC nucleotide codes, either case (HybridDNATokenizer.IUPAC_RUN_RE)
+ static bool is_iupac(char c) {
+ c = to_upper(c);
+ return std::string("ACGTRYSWKMBDHVN").find(c) != std::string::npos;
+ }
+
+ // HybridDNATokenizer._normalize_spaced_dna: join displays like "ACGTA CGTAC ..." when they hold at least twelve bases
+ // (at least six chunks of 1-5 uppercase ACGT separated by spaces or tabs, not touching a letter)
+ static std::string normalize_spaced_dna(const std::string & text) {
+ const auto is_blank = [](char c) { return c == ' ' || c == '\t'; };
+ std::string out;
+ size_t p = 0;
+ while (p < text.size()) {
+ if (is_acgt(text[p]) && (p == 0 || !is_letter(text[p - 1]))) {
+ size_t match_end = std::string::npos;
+ size_t chunk = p;
+ for (int n_chunks = 1; ; n_chunks++) {
+ size_t chunk_end = chunk;
+ while (chunk_end < text.size() && chunk_end - chunk < 5 && is_acgt(text[chunk_end])) {
+ chunk_end++;
+ }
+ if (chunk_end == chunk) {
+ break;
+ }
+ if (n_chunks >= 6 && (chunk_end == text.size() || !is_letter(text[chunk_end]))) {
+ match_end = chunk_end; // the regex is greedy, so the last valid end wins
+ }
+ size_t next = chunk_end;
+ while (next < text.size() && is_blank(text[next])) {
+ next++;
+ }
+ if (next == chunk_end) {
+ break;
+ }
+ chunk = next;
+ }
+ if (match_end != std::string::npos) {
+ std::string seq;
+ for (size_t i = p; i < match_end; i++) {
+ if (!is_blank(text[i])) {
+ seq += text[i];
+ }
+ }
+ out += seq.size() >= 12 ? seq : text.substr(p, match_end - p);
+ p = match_end;
+ continue;
+ }
}
+ out += text[p++];
}
+ return out;
+ }
- // k-mers carry the reserved marker suffix; a non-ACGT k-mer simply
- // isn't in the vocab and falls back to <oov>
- auto kmer_token = [&](const std::string & kmer) {
- const auto tok = vocab.text_to_token(kmer + dna_kmer_marker);
- return tok != LLAMA_TOKEN_NULL ? tok : oov_id;
+ // text outside explicit tags: with auto_dna_tags, tag DNA runs as HybridDNATokenizer._auto_tag_text_segment does
+ // (pure ACGT runs of at least 6 bases, or runs of at least 12 with at most 20% IUPAC ambiguity codes)
+ void tokenize_text(const std::string & raw, std::vector<llama_token> & output) {
+ auto bpe = [&](const std::string & piece) {
+ if (!piece.empty()) {
+ llm_tokenizer_bpe_session::tokenize(piece, output);
+ }
};
+ if (!vocab.get_dna_auto_tags()) {
+ bpe(raw);
+ return;
+ }
+ const std::string text = normalize_spaced_dna(raw);
+ size_t prev = 0;
size_t i = 0;
- for (; i + k <= seq.size(); i += k) {
- output.push_back(kmer_token(seq.substr(i, k)));
+ while (i < text.size()) {
+ if (!is_iupac(text[i])) {
+ i++;
+ continue;
+ }
+ size_t j = i;
+ size_t n_ambiguous = 0;
+ for (; j < text.size() && is_iupac(text[j]); j++) {
+ n_ambiguous += !is_acgt(to_upper(text[j]));
+ }
+ const size_t len = j - i;
+ if ((n_ambiguous == 0 && len >= 6) || (n_ambiguous > 0 && len >= 12 && 5 * n_ambiguous <= len)) {
+ bpe(text.substr(prev, i - prev));
+ tokenize_dna(text.substr(i, len), true, true, output);
+ prev = j;
+ }
+ i = j;
+ }
+ bpe(text.substr(prev));
+ }
+
+ // HybridDNATokenizer._process_dna_sequence: strip, uppercase, then one token per k bases
+ // a k-mer with anything but ACGT is <oov>, and a trailing partial k-mer is right-padded with pad_char
+ void tokenize_dna(const std::string & content, bool has_start, bool has_end, std::vector<llama_token> & output) {
+ if (has_start) {
+ output.push_back(dna_begin_id);
+ }
+
+ auto cpts = unicode_cpts_from_utf8(content);
+ size_t first = 0;
+ size_t last = cpts.size();
+ while (first < last && unicode_cpt_flags_from_cpt(cpts[first]).is_whitespace) {
+ first++;
+ }
+ while (last > first && unicode_cpt_flags_from_cpt(cpts[last - 1]).is_whitespace) {
+ last--;
+ }
+
+ // k-mers carry the reserved marker suffix, so they never resolve to a base-vocab token
+ auto kmer_token = [&](size_t from, size_t to) {
+ std::string kmer;
+ for (size_t i = from; i < to; i++) {
+ const uint32_t c = cpts[i] < 128 ? (uint32_t) to_upper((char) cpts[i]) : cpts[i];
+ if (c > 127 || !is_acgt((char) c)) {
+ return dna_oov_id;
+ }
+ kmer += (char) c;
+ }
+ kmer.append(k - kmer.size(), pad_char);
+ const auto tok = vocab.text_to_token(kmer + dna_kmer_marker);
+ return tok != LLAMA_TOKEN_NULL ? tok : dna_oov_id;
+ };
+
+ size_t i = first;
+ for (; i + k <= last; i += k) {
+ output.push_back(kmer_token(i, i + k));
+ }
+ if (i < last) {
+ output.push_back(kmer_token(i, last));
}
- if (i < seq.size()) {
- std::string kmer = seq.substr(i);
- kmer.append(k - kmer.size(), 'A');
- output.push_back(kmer_token(kmer));
+
+ if (has_end) {
+ output.push_back(dna_end_id);
}
}
+ const std::string open_tag = "<dna>";
+ const std::string close_tag = "</dna>";
+ const size_t k = 6;
+
+ llama_token dna_begin_id = LLAMA_TOKEN_NULL;
+ llama_token dna_end_id = LLAMA_TOKEN_NULL;
+ llama_token dna_oov_id = LLAMA_TOKEN_NULL;
+ char pad_char = 'A';
+
const llama_vocab & vocab;
};
@@ -1957,6 +2080,7 @@ struct llama_vocab::impl {
bool add_bos = false;
bool add_eos = false;
bool add_sep = false;
+ bool dna_auto_tags = false; // hybriddna: tag untagged DNA runs before tokenizing
bool ignore_merges = false;
bool clean_spaces = false; // clean_up_tokenization_spaces
bool remove_extra_whitespaces = false;
@@ -2689,6 +2813,12 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
if (text.size() > dna_kmer_marker.size()
&& text.compare(text.size() - dna_kmer_marker.size(), dna_kmer_marker.size(), dna_kmer_marker) == 0) {
text.erase(text.size() - dna_kmer_marker.size());
+ // tail="pad_n": "CTNNNN" detokenizes to its real bases "CT"
+ // full k-mers are ACGT only, so only N-padded tail tokens end in N
+ const size_t last_base = text.find_last_not_of('N');
+ if (last_base != std::string::npos) {
+ text.erase(last_base + 1);
+ }
}
}
}
@@ -2781,6 +2911,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
if (ml.get_key(LLM_KV_TOKENIZER_ADD_SEP, temp, false)) {
add_sep = temp;
}
+ if (ml.get_key(LLM_KV_TOKENIZER_DNA_AUTO_TAGS, temp, false)) {
+ dna_auto_tags = temp;
+ }
// workaround for Gemma 4
// ref: https://github.com/ggml-org/llama.cpp/pull/21500
@@ -4312,6 +4445,10 @@ bool llama_vocab::get_add_sep() const {
return pimpl->add_sep;
}
+bool llama_vocab::get_dna_auto_tags() const {
+ return pimpl->dna_auto_tags;
+}
+
bool llama_vocab::get_ignore_merges() const {
return pimpl->ignore_merges;
}
diff --git a/src/llama-vocab.h b/src/llama-vocab.h
index 898a23ae6..415acd22b 100644
--- a/src/llama-vocab.h
+++ b/src/llama-vocab.h
@@ -149,6 +149,7 @@ struct llama_vocab {
bool get_add_bos () const;
bool get_add_eos () const;
bool get_add_sep () const;
+ bool get_dna_auto_tags () const;
bool get_ignore_merges () const;
bool get_clean_spaces () const;
bool get_remove_extra_whitespaces () const;