Commit 7481354a1 for llama.cpp

commit 7481354a174714a3447ea0f8723c1dd1b3cbf8e2
Author: Toki Nasin <141258697+tokinasin@users.noreply.github.com>
Date:   Wed Oct 7 18:34:16 2026 +0900

    convert : fix token configuration for PLaMo-3 (#29843)

    * convert : Fix token configuration for PLaMo-3

    The reasoning and tool calling tags in PLaMo-3 consist of three tokens
    each. For example, for reasoning:

    * reasoning start: `<|plamo:begin_`, `think`, `:plamo|>`
    * reasoning end: `<|plamo:end_`, `think`, `:plamo|>`

    Registering `<|plamo:begin_`, `<|plamo:end_`, and `:plamo|>` as
    `USER_DEFINED` so that they are parsed correctly.

    Also PLaMo-3 models use <|plamo:tag|> as EOT, while PLaMo-2 models
    use <|plamo:op|>.

    Take EOT token as a parameter and look it up so that PLaMo-2 and
    PLaMo-3 can use their appropriate ones.

    * use NORMAL instead of USER_DEFINED

diff --git a/conversion/base.py b/conversion/base.py
index 3b6fe04c2..306a9f0f2 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -2499,7 +2499,11 @@ class TextModel(ModelBase):
         if template is not None:
             self.gguf_writer.add_chat_template(template)

-    def _set_vocab_plamo(self):
+    def _set_vocab_plamo(
+        self,
+        eot_token: str,
+        normal_tokens: Iterable[str] = (),
+    ):
         # PLaMo models use a custom tokenizer with a .jsonl file
         tokenizer_jsonl_path = self.dir_model / "tokenizer.jsonl"
         tokenizer_config_path = self.dir_model / "tokenizer_config.json"
@@ -2523,27 +2527,30 @@ class TextModel(ModelBase):
         tokens = []
         scores = []
         toktypes = []
+        normal_tokens = set(normal_tokens)

         with open(tokenizer_jsonl_path, "r", encoding="utf-8") as f:
             for line_num, line in enumerate(f):
                 if line.strip():
                     token_data = json.loads(line)
                     # Format: [token, score, type, ?, ?, ?, ?]
-                    token = token_data[0].encode("utf-8")
+                    token_str = token_data[0]
+                    token = token_str.encode("utf-8")
                     score = float(token_data[1])
                     token_type_str = token_data[2] if len(token_data) > 2 else "NORMAL"

                     tokens.append(token)
                     scores.append(score)

-                    if token_type_str == "UNKNOWN":
+                    if token_str in normal_tokens:
+                        toktypes.append(gguf.TokenType.NORMAL)
+                    elif token_type_str == "UNKNOWN":
                         toktypes.append(gguf.TokenType.UNKNOWN)
                     elif token_type_str == "CONTROL":
                         toktypes.append(gguf.TokenType.CONTROL)
                     elif token_type_str == "BYTE":
                         toktypes.append(gguf.TokenType.BYTE)
                     else:
-                        token_str = token_data[0]
                         if token_str.startswith("<|plamo:") and token_str.endswith("|>"):
                             toktypes.append(gguf.TokenType.CONTROL)
                         else:
@@ -2580,8 +2587,7 @@ class TextModel(ModelBase):
             token_id = tokens.index(tokenizer_config["unk_token"].encode("utf-8"))
             self.gguf_writer.add_unk_token_id(token_id)

-        # Add <|plamo:op|> as EOT to ensure appropriate end of generation
-        self.gguf_writer.add_eot_token_id(4)
+        self.gguf_writer.add_eot_token_id(tokens.index(eot_token.encode("utf-8")))

         self.gguf_writer.add_add_space_prefix(False)

diff --git a/conversion/plamo.py b/conversion/plamo.py
index 52055e4a1..a843e059a 100644
--- a/conversion/plamo.py
+++ b/conversion/plamo.py
@@ -64,7 +64,7 @@ class Plamo2Model(TextModel):
     model_arch = gguf.MODEL_ARCH.PLAMO2

     def set_vocab(self):
-        self._set_vocab_plamo()
+        self._set_vocab_plamo(eot_token="<|plamo:op|>")

     def set_gguf_parameters(self):
         hparams = self.hparams
@@ -170,7 +170,10 @@ class Plamo3Model(TextModel):
             })

     def set_vocab(self):
-        self._set_vocab_plamo()
+        self._set_vocab_plamo(
+            eot_token="<|plamo:tag|>",
+            normal_tokens=("<|plamo:begin_", "<|plamo:end_", ":plamo|>"),
+        )

         tokenizer_config_path = self.dir_model / "tokenizer_config.json"
         tokenizer_config = {}