Commit 23f7aabc1 for llama.cpp
commit 23f7aabc16c72f6f707dba227251c35d7bd1ed96
Author: Gabe Goodhart <ghart@us.ibm.com>
Date: Fri Oct 16 12:00:41 2026 -0600
model : Granite Switch Decision (#30676)
* feat(convert): Add decision.chat to support decision models that can also chat
Branch: GraniteSwitchDecision
AI-usage: full (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
* feat(convert): Add conversion for Granite Switch as decision model
The template follows from the usage sample:
https://huggingface.co/ibm-granite/granite-switch-4.1-8b#guardian-and-decision-side-by-side
Branch: GraniteSwitchDecision
AI-usage: draft (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
* feat(common): Add support for chat+decision models in common
Branch: GraniteSwitchDecision
AI-usage: full (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
* feat(common): Add granite switch decision model enum
Branch: GraniteSwitchDecision
AI-usage: full (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
* feat(server): Plumb through decision_chat bool to support decision+chat models
Branch: GraniteSwitchDecision
AI-usage: full (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
* feat(server): Add Granite Switch decision support
Branch: GraniteSwitchDecision
AI-usage: draft (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
* docs: Update server README for granite switch decisions / chat+decisions
Branch: GraniteSwitchDecision
AI-usage: draft (Claude + Sonnet 5.5)
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
* Revert "feat(convert): Add decision.chat to support decision models that can also chat"
This reverts commit 952ef567647e53ac9beda7a1f79d6927240d068d.
* Revert "feat(common): Add support for chat+decision models in common"
This reverts commit 93ed782e3a8b2fd5968732c6525c7ad29ad4bea2.
* Revert "feat(server): Plumb through decision_chat bool to support decision+chat models"
This reverts commit 97e9af60ddaccbd82f1eb0a488adff43b16ee095.
* fix: If Granite Switch supports decisions, it always supports text too
Branch: GraniteSwitchDecision
AI-usage: none
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
* docs: Remove reference to decision.chat in server README
Branch: GraniteSwitchDecision
AI-usage: none
Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
diff --git a/common/common.cpp b/common/common.cpp
index c5db22832..b59e66e3b 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1174,6 +1174,7 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
{ COMMON_DECISION_TYPE_VLLM_D3, "vllm-d3" },
{ COMMON_DECISION_TYPE_LFM2_D1, "lfm2-d1" },
{ COMMON_DECISION_TYPE_LFM2_D1_OMNI, "lfm2-d1-omni" },
+ { COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"},
// for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
};
diff --git a/common/common.h b/common/common.h
index 2cd6e2ec9..0c2d5e46f 100644
--- a/common/common.h
+++ b/common/common.h
@@ -970,6 +970,7 @@ enum common_decision_type {
COMMON_DECISION_TYPE_VLLM_D3, // same as pplx-decider, non-causal attention
COMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question type
COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout
+ COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch
COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
};
diff --git a/conversion/granite.py b/conversion/granite.py
index 4cbeb7617..d2ef7c836 100644
--- a/conversion/granite.py
+++ b/conversion/granite.py
@@ -8,7 +8,7 @@ import torch
if TYPE_CHECKING:
from torch import Tensor
-from .base import MmprojModel, ModelBase, gguf, logger
+from .base import MmprojModel, ModelBase, gguf, jinja_str_or_json, logger
from .llama import LlamaModel
from .mamba import Mamba2Model
@@ -284,10 +284,39 @@ class GraniteSwitchModel(GraniteMoeModel):
self.gguf_writer.add_adapter_lora_rank(self._max_lora_rank)
self.gguf_writer.add_adapter_token_ids_activate(self.hparams["adapter_token_ids"])
self.gguf_writer.add_adapter_token_ids_substitute(self.hparams["adapter_substitute_token_ids"])
+ if self._has_decision_adapter():
+ self.gguf_writer.add_decision_type(gguf.DecisionType.GRANITE_SWITCH)
router_gain = float(self.hparams.get("control_token_gain", 15.0))
self.gguf_writer.add_adapter_router_gain(router_gain)
logger.info("gguf: (graniteswitch) num_adapters=%s max_lora_rank=%s n_slots=%s router_gain=%s", self._n_adapters, self._max_lora_rank, self._n_slots, router_gain)
+ def _has_decision_adapter(self) -> bool:
+ return "decision" in self.hparams.get("adapter_names", [])
+
+ def set_vocab(self):
+ super().set_vocab()
+ if self._has_decision_adapter():
+ self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])
+
+ def _systemone_template(self) -> str:
+ # prompt of the decision aLoRA: a state turn, then a question turn that opens with the control token
+ # (it replaces the first token of "### DECISION"), the answer is the label token that comes next
+ description = jinja_str_or_json("o.description")
+ option = (
+ "{% if type != 'noul' %}{{ letters[loop.index0] }}: {% if o.description %}" + description + "{% endif %}"
+ "{% elif o.key == 'true' %}A: {% if o.description %}" + description + "{% else %}The statement is true.{% endif %}"
+ "{% else %}B: {% if o.description %}" + description + "{% else %}The statement is false.{% endif %}{% endif %}"
+ )
+ return (
+ "{% set letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' %}"
+ "<|start_of_role|>user<|end_of_role|>State:\n" + jinja_str_or_json("state") + "<|end_of_text|>\n"
+ "<|start_of_role|>user<|end_of_role|><|decision|> DECISION\nQuestion:\n" + jinja_str_or_json("instructions")
+ + "{% if type == 'score' %} Rate along the ordered levels below (lowest first).{% endif %}"
+ "{{ '\n\nOptions:\n' }}"
+ "{% for o in options %}" + option + "{{ '\n' }}{% endfor %}"
+ "{{ '\nReply with exactly one option label.<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>' }}"
+ )
+
def _lora_a(self, data: Tensor) -> Tensor:
# on-disk A: [n_adapters, 1, max_rank, in] -> [n_adapters+1, max_rank, in]
a = data.squeeze(1)
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 00b51165c..4261e76e3 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -6283,6 +6283,7 @@ class DecisionType:
VLLM_D3 = "vllm-d3" # same as pplx-decider, non-causal attention
LFM2_D1 = "lfm2-d1" # same as openjev, the labels depend on the question type
LFM2_D1_OMNI = "lfm2-d1-omni" # same head as laya on a bidirectional LFM2 trunk, other prompt layout
+ GRANITE_SWITCH = "granite-switch" # same as openjev, run by the decision aLoRA of granite switch
class VisionProjectorType:
diff --git a/tools/server/README.md b/tools/server/README.md
index 7d697633e..e4a0c418b 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -1723,9 +1723,9 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo
The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly.
-The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
+The number of options of a `choice` question is limited by the model, for example: 52 for openjev and granite-switch, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
-For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available.
+For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.
*Image input:*
diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp
index c32c0d1a5..5577c339d 100644
--- a/tools/server/server-common.cpp
+++ b/tools/server/server-common.cpp
@@ -161,6 +161,8 @@ std::vector<std::string> server_model_output_modalities(common_decision_type dec
case COMMON_DECISION_TYPE_LFM2_D1:
case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
return {"decisions"};
+ case COMMON_DECISION_TYPE_GRANITE_SWITCH:
+ return {"text", "decisions"};
default:
// fallback when there is no decision type or the metadata is bad
return {"text"};
diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp
index b3c19e1f0..3905c81b3 100644
--- a/tools/server/server-decision.cpp
+++ b/tools/server/server-decision.cpp
@@ -87,7 +87,7 @@ void server_decision_context::init(const llama_model * model) {
bias[level] = std::strtof(val, nullptr);
}
- if (model_type == COMMON_DECISION_TYPE_OPENJEV) {
+ if (model_type == COMMON_DECISION_TYPE_OPENJEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) {
// one letter per option, each must be a single token
const std::string letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
for (const char c : letters) {
@@ -753,7 +753,7 @@ void server_decision_context::fill_task(
const std::string prompt = render(state, questions, question, variant, files.size());
- if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) {
+ if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_GRANITE_SWITCH || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) {
// lev reads the ratings of a noul question at its first labels, not at the digits
task.decision.labels.assign(labels.begin(), labels.begin() + n_outputs(question));
}
diff --git a/tools/server/server-decision.h b/tools/server/server-decision.h
index bfe9fd0ee..ed9724429 100644
--- a/tools/server/server-decision.h
+++ b/tools/server/server-decision.h
@@ -39,6 +39,7 @@ struct server_decision_context {
bool can_share_prompt() const {
switch (type) {
case COMMON_DECISION_TYPE_OPENJEV:
+ case COMMON_DECISION_TYPE_GRANITE_SWITCH:
case COMMON_DECISION_TYPE_LEV:
case COMMON_DECISION_TYPE_KEV:
case COMMON_DECISION_TYPE_NIMBLE: