Commit 7e3eeee04 for llama.cpp
commit 7e3eeee0480902c90cbdff172f407dc063cf0770
Author: PSEUDOTENSOR / Jonathan McKinney <pseudotensor@gmail.com>
Date: Tue Oct 20 09:54:27 2026 -0700
model : add H2O-Lightning-4B decision model (#30361)
* model : add H2O-Lightning-4B decision model
Adds the h2o-lightning decision type for h2oai/h2o-lightning-4b
(Qwen3.5-4B with a merged decision LoRA).
None of the existing types match this model. It answers with " A"
rather than "A", uses up to 255 labels, and was trained on Python
json.dumps text, so floats like 12.0 must render as 12.0.
The converter writes the decision metadata and the systemone template
from the model's serve_config.json and refuses a config that doesn't
match. Tests skip unless the tiny test model is available locally.
Assisted-by: Claude Opus 5.5
* rm redundant code
* put labels to gguf
---------
Co-authored-by: Jonathan C. McKinney <jon.mckinney@h2o.ai>
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
diff --git a/common/common.cpp b/common/common.cpp
index b59e66e3b..b0ddfcb91 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1175,6 +1175,7 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
{ COMMON_DECISION_TYPE_LFM2_D1, "lfm2-d1" },
{ COMMON_DECISION_TYPE_LFM2_D1_OMNI, "lfm2-d1-omni" },
{ COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"},
+ { COMMON_DECISION_TYPE_H2O_LIGHTNING, "h2o-lightning" },
// for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
};
diff --git a/common/common.h b/common/common.h
index b172b4efe..b089abd7e 100644
--- a/common/common.h
+++ b/common/common.h
@@ -975,6 +975,7 @@ enum common_decision_type {
COMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question type
COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout
COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch
+ COMMON_DECISION_TYPE_H2O_LIGHTNING, // same as openjev, the labels are read after an answer prefill
COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
};
diff --git a/conversion/__init__.py b/conversion/__init__.py
index e3c10d5fc..efa537b0f 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -226,6 +226,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"MuseSparkForConditionalGeneration": "muse_spark",
"OpenELMForCausalLM": "openelm",
"OpenJevModel": "qwen",
+ "H2OLightningModel": "h2o_lightning",
"OrionForCausalLM": "orion",
"PLMForCausalLM": "plm",
"PLaMo2ForCausalLM": "plamo",
@@ -375,6 +376,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"Qwen3VLForConditionalGeneration": "qwen3vl",
"Qwen3VLMoeForConditionalGeneration": "qwen3vl",
"OpenJevModel": "qwen3vl",
+ "H2OLightningModel": "h2o_lightning",
"Qwen3_5ForConditionalGeneration": "qwen3vl",
"Qwen3_5MoeForConditionalGeneration": "qwen3vl",
"Qwen4ExpForConditionalGeneration": "qwen4exp",
diff --git a/conversion/h2o_lightning.py b/conversion/h2o_lightning.py
new file mode 100644
index 000000000..764da4b97
--- /dev/null
+++ b/conversion/h2o_lightning.py
@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import json
+
+from pathlib import Path
+from typing import Any
+
+from .base import ModelBase, gguf, logger
+from .qwen import Qwen3_5TextModel
+from .qwen3vl import Qwen3VLVisionModel
+
+
+def _is_h2o_lightning_checkpoint(dir_model: Path) -> bool:
+ return (dir_model / "h2o_lightning_shim.py").is_file() and (dir_model / "serve_config.json").is_file()
+
+
+@ModelBase.register_hparams_loader(_is_h2o_lightning_checkpoint)
+def _load_h2o_lightning_hparams(dir_model: Path) -> dict[str, Any]:
+ logger.info("gguf: detected H2O-Lightning checkpoint")
+ hparams = ModelBase.load_hparams(dir_model, False, guess=False)
+ hparams["architectures"] = ["H2OLightningModel"]
+ return hparams
+
+
+@ModelBase.register("H2OLightningModel")
+@ModelBase.example("h2oai/h2o-lightning-4b")
+class H2OLightningModel(Qwen3_5TextModel):
+ model_arch = gguf.MODEL_ARCH.QWEN35
+
+ # prompt and calibration follow serve_config.json and h2o_lightning_shim.py of the model repo
+ # the state budget is hard-coded in server-decision.cpp
+ _MAX_STATE_TOKENS = 32000
+ _DEFAULT_INSTRUCTIONS = "Answer the question below."
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ with open(self.dir_model / "serve_config.json", "r", encoding="utf-8") as f:
+ self._serve_config = json.load(f)
+ prompt = self._serve_config["prompt"]
+ expected = {
+ "state_compact": (prompt.get("state_compact", True), True),
+ "max_state_tokens": (prompt.get("max_state_tokens"), self._MAX_STATE_TOKENS),
+ "layout": (prompt.get("layout", "plain"), "plain"),
+ "chat_template": (prompt.get("chat_template", True), True),
+ "enable_thinking": (prompt.get("enable_thinking", False), False),
+ }
+ for name, (got, want) in expected.items():
+ if got != want:
+ raise ValueError(f"serve_config.json: {name} is {got!r}, the h2o-lightning decision type expects {want!r}")
+
+ def get_vocab_base_pre(self, tokenizer) -> str:
+ # transformers loads this checkpoint as Qwen2Tokenizer: the model was trained and is served with the Qwen2 split
+ pre = super().get_vocab_base_pre(tokenizer)
+ if pre != "qwen35":
+ raise ValueError(f"expected the Qwen3.5 tokenizer, got pre-tokenizer {pre!r}")
+ return "qwen2"
+
+ def set_vocab(self):
+ super().set_vocab()
+ self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])
+
+ def _systemone_template(self) -> str:
+ # the server gives the template texts rendered as the shim renders them, it only lays them out
+ prompt = self._serve_config["prompt"]
+ nl = "{{ '\\n' }}"
+ description = (
+ "{% if type == 'noul' %}"
+ "{% if o.description %}{{ o.description }}{% elif o.key == 'true' %}the statement holds{% else %}it does not{% endif %}"
+ "{% elif o.description is none %}{{ o.key }}{% else %}{{ o.description }}{% endif %}"
+ )
+ user = (
+ "{% set user %}record: {{ state }}" + nl
+ + "question: {{ instructions if instructions else '" + self._DEFAULT_INSTRUCTIONS + "' }}" + nl
+ + "options:{% for o in options %}" + nl + "{{ o.label | trim }}) {{ o.key }}: " + description + "{% endfor %}{% endset %}"
+ )
+ return (
+ "<|im_start|>system" + nl + prompt["system"].strip() + "<|im_end|>" + nl
+ + "<|im_start|>user" + nl + "{% for image in images %}{{ image }}{% endfor %}" + user + "{{ user | trim }}<|im_end|>" + nl
+ + "<|im_start|>assistant" + nl + "<think>{{ '\\n\\n' }}</think>{{ '\\n\\n' }}" + prompt["prefill"]
+ )
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_decision_type(gguf.DecisionType.H2O_LIGHTNING)
+ # the answer follows "Answer:", so the token of a label has a leading space
+ cfg = self._serve_config
+ self.gguf_writer.add_decision_labels([" " + label for label in cfg["labels"][:cfg.get("max_options", len(cfg["labels"]))]])
+ for name in ("choice", "noul", "score"):
+ self.gguf_writer.add_decision_temperature(name, self._serve_config["temperature"][name])
+ # a request with images has this temperature for all question types
+ self.gguf_writer.add_decision_temperature("image", self._serve_config["image"]["temperature"])
+
+
+@ModelBase.register("H2OLightningModel")
+class H2OLightningVisionModel(Qwen3VLVisionModel):
+ pass
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index b2d8d59e0..2840348cb 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -344,6 +344,7 @@ class Keys:
ROUTING_BLOCK_COUNT = "{arch}.decision.routing_block_count"
HEAD_COUNT = "{arch}.decision.head_count"
MAX_HEAD_TOKENS = "{arch}.decision.max_head_tokens"
+ LABELS = "{arch}.decision.labels" # the text of the token read for each option, in order
TEMPERATURE = "{arch}.decision.temperature.{name}" # name: "<type>" or "<type>.<n_opt bucket>"
SCORE_BIAS = "{arch}.decision.score_bias.{n}.{i}" # added to the logit of level i of a score question with n levels
@@ -6339,6 +6340,7 @@ class DecisionType:
LFM2_D1 = "lfm2-d1" # same as openjev, the labels depend on the question type
LFM2_D1_OMNI = "lfm2-d1-omni" # same head as laya on a bidirectional LFM2 trunk, other prompt layout
GRANITE_SWITCH = "granite-switch" # same as openjev, run by the decision aLoRA of granite switch
+ H2O_LIGHTNING = "h2o-lightning" # same as openjev, labels read after an answer prefill
class VisionProjectorType:
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index 2943e0186..e8a96e571 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1401,6 +1401,9 @@ class GGUFWriter:
def add_decision_max_head_tokens(self, value: int) -> None:
self.add_uint32(Keys.Decision.MAX_HEAD_TOKENS.format(arch=self.arch), value)
+ def add_decision_labels(self, labels: Sequence[str]) -> None:
+ self.add_array(Keys.Decision.LABELS.format(arch=self.arch), labels)
+
def add_decision_temperature(self, name: str, value: float) -> None:
self.add_float32(Keys.Decision.TEMPERATURE.format(arch=self.arch, name=name), value)
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 961011a2c..1b954f176 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -383,6 +383,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_DECISION_BLOCK_COUNT, "%s.decision.block_count" },
{ LLM_KV_DECISION_ROUTING_BLOCK_COUNT, "%s.decision.routing_block_count" },
{ LLM_KV_DECISION_HEAD_COUNT, "%s.decision.head_count" },
+ { LLM_KV_DECISION_LABELS, "%s.decision.labels" },
{ LLM_KV_TARGET_LAYERS, "%s.target_layers" },
{ LLM_KV_TARGET_HIDDEN_SIZE, "%s.target_hidden_size" },
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 3ea86c601..5a8ca4250 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -431,6 +431,7 @@ enum llm_kv {
LLM_KV_DECISION_BLOCK_COUNT,
LLM_KV_DECISION_ROUTING_BLOCK_COUNT,
LLM_KV_DECISION_HEAD_COUNT,
+ LLM_KV_DECISION_LABELS,
LLM_KV_TARGET_LAYERS,
LLM_KV_TARGET_HIDDEN_SIZE,
diff --git a/src/llama-ext.h b/src/llama-ext.h
index 8b3932bd7..933cb8013 100644
--- a/src/llama-ext.h
+++ b/src/llama-ext.h
@@ -122,6 +122,9 @@ enum llama_decision_order {
// The embeddings output has one value per entry: row i is the score of option i
LLAMA_API bool llama_batch_ext_set_decision_order(struct llama_batch_ext * batch, int32_t idx, enum llama_decision_order order);
+// Label of option i of a decision model ("<arch>.decision.labels"), nullptr past the last one
+LLAMA_API const char * llama_model_decision_label(const struct llama_model * model, uint32_t i);
+
// mirrors:
// LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx);
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 803a8329e..6712f0f22 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -1549,6 +1549,9 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
hparams.n_cls_out = classifier_labels.size();
}
+ // for decision models
+ ml.get_arr(LLM_KV_DECISION_LABELS, decision_labels, false);
+
// per-arch hparams
load_arch_hparams(ml);
@@ -3438,6 +3441,14 @@ const char * llama_model_cls_label(const struct llama_model * model, uint32_t i)
return nullptr;
}
+const char * llama_model_decision_label(const struct llama_model * model, uint32_t i) {
+ if (i < model->decision_labels.size()) {
+ return model->decision_labels[i].c_str();
+ }
+
+ return nullptr;
+}
+
// deprecated
int32_t llama_n_ctx_train(const llama_model * model) {
return llama_model_n_ctx_train(model);
diff --git a/src/llama-model.h b/src/llama-model.h
index a187c2749..9fe9d77a2 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -724,6 +724,7 @@ struct llama_model {
// for classifier models
std::vector<std::string> classifier_labels;
+ std::vector<std::string> decision_labels;
struct ggml_tensor * tok_embd = nullptr;
struct ggml_tensor * type_embd = nullptr;
diff --git a/tools/server/README.md b/tools/server/README.md
index e1c5804da..8d9f53569 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -1725,13 +1725,13 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo
The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly.
-The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for granite-switch, laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
+The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for granite-switch, laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1, lfm2-d1-omni and h2o-lightning. For laya, long questions and options are truncated to the token budget the model was trained with.
-For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.
+For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. An h2o-lightning state longer than 32000 tokens keeps its first and last 16000 tokens. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.
*Image input:*
-Image input needs a model that supports it (for example: openjev, clef, pplx-decider, vllm-d3, lfm2-d1, lfm2-d1-omni) and its multimodal projector, see `--mmproj`.
+Image input needs a model that supports it (for example: openjev, clef, pplx-decider, vllm-d3, lfm2-d1, lfm2-d1-omni, h2o-lightning) and its multimodal projector, see `--mmproj`.
Images can be given in two ways, and both can be used in the same request:
@@ -1760,6 +1760,8 @@ All the images are placed before the state in the prompt, the ones from `files`
The probabilities are scaled with the temperatures stored in the model file. They are not guaranteed to be calibrated for your data.
+For h2o-lightning, `score` questions can have 1 to 255 levels, and the `instructions` and `type` of a question can be left out (`choice` by default; `"instructions": null` is written as `null`).
+
*Examples:*
```shell
diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp
index 3ea20c3d2..075687602 100644
--- a/tools/server/server-common.cpp
+++ b/tools/server/server-common.cpp
@@ -215,6 +215,7 @@ std::vector<std::string> server_model_output_modalities(common_decision_type dec
case COMMON_DECISION_TYPE_VLLM_D3:
case COMMON_DECISION_TYPE_LFM2_D1:
case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
+ case COMMON_DECISION_TYPE_H2O_LIGHTNING:
return {"decisions"};
case COMMON_DECISION_TYPE_GRANITE_SWITCH:
return {"text", "decisions"};
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 74812189c..84cd67768 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -5890,7 +5890,7 @@ void server_routes::init_routes() {
const auto questions = decision.parse_questions(body);
std::vector<raw_buffer> files;
- const json state = decision.parse_state(body, files);
+ const json state = decision.prepare_state(decision.parse_state(body, files));
if (!files.empty() && (!decision.can_use_images() || !meta->has_inp_image)) {
res->error(format_error_response("This server does not support image input for decisions. For a model that supports it, start it with `--mmproj`", ERROR_TYPE_NOT_SUPPORTED));
return res;
diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp
index be9bc8ef4..ce4cb4d09 100644
--- a/tools/server/server-decision.cpp
+++ b/tools/server/server-decision.cpp
@@ -19,6 +19,10 @@ static const char * decision_question_type_name(server_decision_question_type ty
// lev reads noul from a rating scale: 0 = certainly no, 8 = certainly yes
static const size_t DECISION_LEV_N_RATINGS = 9;
+// h2o-lightning follows serve_config.json and h2o_lightning_shim.py of the model repo
+// a longer state keeps its head and its tail
+static const size_t DECISION_H2O_MAX_STATE_TOKENS = 32000;
+
static std::string decision_meta_str(const llama_model * model, const std::string & key) {
char buf[256];
const int32_t n = llama_model_meta_val_str(model, key.c_str(), buf, sizeof(buf));
@@ -164,6 +168,39 @@ void server_decision_context::init(const llama_model * model) {
throw std::runtime_error("decision model has no mask token");
}
n_options_max = 255;
+ } else if (model_type == COMMON_DECISION_TYPE_H2O_LIGHTNING) {
+ // the answer is read at the token of a label, its text is given to the template
+ for (uint32_t i = 0; llama_model_decision_label(model, i) != nullptr; i++) {
+ const std::string text = llama_model_decision_label(model, i);
+ const auto toks = common_tokenize(vocab, text, false, false);
+ if (toks.size() != 1 || std::find(labels.begin(), labels.end(), toks[0]) != labels.end()) {
+ throw std::runtime_error(string_format("decision label \"%s\" is not a single distinct token", text.c_str()));
+ }
+ labels.push_back(toks[0]);
+ label_texts.push_back(text);
+ }
+ if (labels.empty()) {
+ throw std::runtime_error("decision model has no labels");
+ }
+ // the texts of the control and user-defined tokens are quoted in the input, so that they stay text
+ for (llama_token id = 0; id < llama_vocab_n_tokens(vocab); id++) {
+ if (llama_vocab_get_attr(vocab, id) & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) {
+ const std::string text = llama_vocab_get_text(vocab, id);
+ if (!text.empty()) {
+ quoted_spellings.push_back(text);
+ }
+ }
+ }
+ std::stable_sort(quoted_spellings.begin(), quoted_spellings.end(), [](const std::string & a, const std::string & b) {
+ return a.size() > b.size();
+ });
+ for (const auto & spelling : quoted_spellings) {
+ if (quoted_first.find(spelling[0]) == std::string::npos) {
+ quoted_first += spelling[0];
+ }
+ }
+ n_options_max = labels.size();
+ noul_true_first = true;
} else {
throw std::runtime_error("unsupported decision model type: " + type_name);
}
@@ -201,8 +238,10 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
throw err("must be an object");
}
// the vllm-d3 prompt has a default question
- const json instructions = q.contains("instructions") ? q.at("instructions") : json();
- if (instructions.is_null() && type != COMMON_DECISION_TYPE_VLLM_D3) {
+ // h2o-lightning: the instructions can be left out, the template has a default, and the type defaults to choice
+ const bool is_h2o = type == COMMON_DECISION_TYPE_H2O_LIGHTNING;
+ const json instructions = q.contains("instructions") ? q.at("instructions") : (is_h2o ? json(std::string()) : json());
+ if (instructions.is_null() && type != COMMON_DECISION_TYPE_VLLM_D3 && !is_h2o) {
throw err("\"instructions\" must be provided");
}
if (strict_payload && (!decision_is_payload(instructions) || instructions == "")) {
@@ -213,7 +252,7 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
question.id = id;
question.instructions = instructions;
- const std::string type_name = json_value(q, "type", std::string());
+ const std::string type_name = json_value(q, "type", std::string(is_h2o ? "choice" : ""));
const json criteria = q.contains("criteria") ? q.at("criteria") : json();
if (type_name == "choice") {
@@ -237,7 +276,11 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
}
} else if (type_name == "score") {
question.type = SERVER_DECISION_QUESTION_SCORE;
- if (!criteria.is_array() || criteria.size() < 2 || criteria.size() > 10) {
+ // h2o-lightning: from 1 level, the maximum is n_options_max
+ if (is_h2o && (!criteria.is_array() || criteria.empty())) {
+ throw err("\"criteria\" must be a non-empty array");
+ }
+ if (!is_h2o && (!criteria.is_array() || criteria.size() < 2 || criteria.size() > 10)) {
throw err("\"criteria\" must be an array of 2 to 10 levels");
}
for (size_t i = 0; i < criteria.size(); i++) {
@@ -262,8 +305,8 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
json description;
if (criteria.is_object() && criteria.contains(key)) {
description = criteria.at(key);
- } else if (criteria.is_object() && type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {
- // lfm2-d1-omni also reads the descriptions under "no" and "yes"
+ } else if (criteria.is_object() && (type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || is_h2o)) {
+ // lfm2-d1-omni and h2o-lightning also read the descriptions under "no" and "yes"
const char * alias = std::string(key) == "true" ? "yes" : "no";
description = criteria.contains(alias) ? criteria.at(alias) : json();
}
@@ -506,6 +549,280 @@ static json decision_d1omni_escape(const json & val) {
return val;
}
+//
+// h2o-lightning: the texts are rendered the way the reference shim renders them with Python (json.dumps, str.lstrip)
+//
+
+// one code point of UTF-8 text at s[i], i moves past it; an invalid byte is read as is
+static uint32_t decision_utf8_next(const std::string & s, size_t & i) {
+ const unsigned char c = s[i];
+ size_t n = c < 0x80 ? 1 : (c >> 5) == 0x6 ? 2 : (c >> 4) == 0xe ? 3 : (c >> 3) == 0x1e ? 4 : 1;
+ if (i + n > s.size()) {
+ n = 1;
+ }
+ uint32_t cp = n == 1 ? c : n == 2 ? (c & 0x1f) : n == 3 ? (c & 0x0f) : (c & 0x07);
+ for (size_t k = 1; k < n; k++) {
+ cp = (cp << 6) | (s[i + k] & 0x3f);
+ }
+ i += n;
+ return cp;
+}
+
+// str.isspace() of Python
+static bool decision_py_isspace(uint32_t cp) {
+ return (cp >= 0x09 && cp <= 0x0d) || (cp >= 0x1c && cp <= 0x20) || cp == 0x85 || cp == 0xa0 || cp == 0x1680 ||
+ (cp >= 0x2000 && cp <= 0x200a) || cp == 0x2028 || cp == 0x2029 || cp == 0x202f || cp == 0x205f || cp == 0x3000;
+}
+
+static std::string decision_py_rstrip(const std::string & s) {
+ size_t end = 0;
+ for (size_t i = 0; i < s.size();) {
+ if (!decision_py_isspace(decision_utf8_next(s, i))) {
+ end = i;
+ }
+ }
+ return s.substr(0, end);
+}
+
+static std::string decision_py_lstrip(const std::string & s) {
+ size_t i = 0;
+ while (i < s.size()) {
+ size_t next = i;
+ if (!decision_py_isspace(decision_utf8_next(s, next))) {
+ break;
+ }
+ i = next;
+ }
+ return s.substr(i);
+}
+
+// a code point as JSON escapes, a surrogate pair above the BMP
+static std::string decision_py_escape_cp(uint32_t cp) {
+ if (cp < 0x10000) {
+ return string_format("\\u%04x", cp);
+ }
+ cp -= 0x10000;
+ return string_format("\\u%04x\\u%04x", 0xd800 + (cp >> 10), 0xdc00 + (cp & 0x3ff));
+}
+
+// repr() of a Python float: the shortest digits that read back the same, in exponent form below 1e-4 and from 1e16
+static std::string decision_py_float(double v) {
+ char buf[64];
+ // the fewest digits that read back the same
+ for (int prec = 0; prec < 17; prec++) {
+ snprintf(buf, sizeof(buf), "%.*e", prec, v);
+ if (std::strtod(buf, nullptr) == v) {
+ break;
+ }
+ }
+ const std::string sci(buf); // for example "-1.25e+03"
+ const size_t pos_e = sci.find('e');
+ const int exp = std::atoi(sci.c_str() + pos_e + 1);
+ std::string sign;
+ std::string digits;
+ for (size_t i = 0; i < pos_e; i++) {
+ if (sci[i] == '-') {
+ sign = "-";
+ } else if (sci[i] != '.') {
+ digits += sci[i];
+ }
+ }
+ const int n = (int) digits.size();
+ if (exp < -4 || exp >= 16) {
+ return sign + digits.substr(0, 1) + (n > 1 ? "." + digits.substr(1) : "") + string_format("e%c%02d", exp < 0 ? '-' : '+', std::abs(exp));
+ }
+ if (exp >= n - 1) {
+ return sign + digits + std::string(exp - (n - 1), '0') + ".0";
+ }
+ if (exp >= 0) {
+ return sign + digits.substr(0, exp + 1) + "." + digits.substr(exp + 1);
+ }
+ return sign + "0." + std::string(-exp - 1, '0') + digits;
+}
+
+static void decision_py_str(std::string & out, const std::string & s, bool ensure_ascii) {
+ out += '"';
+ for (size_t i = 0; i < s.size();) {
+ const unsigned char c = s[i];
+ switch (c) {
+ case '"': out += "\\\""; i++; continue;
+ case '\\': out += "\\\\"; i++; continue;
+ case '\b': out += "\\b"; i++; continue;
+ case '\f': out += "\\f"; i++; continue;
+ case '\n': out += "\\n"; i++; continue;
+ case '\r': out += "\\r"; i++; continue;
+ case '\t': out += "\\t"; i++; continue;
+ default: break;
+ }
+ if (c < 0x20 || (ensure_ascii && c == 0x7f)) {
+ out += string_format("\\u%04x", c);
+ i++;
+ } else if (c < 0x80 || !ensure_ascii) {
+ out += (char) c;
+ i++;
+ } else {
+ out += decision_py_escape_cp(decision_utf8_next(s, i));
+ }
+ }
+ out += '"';
+}
+
+// json.dumps() of Python: indent < 0 is no indent, keys stay in their order
+static void decision_py_dump(std::string & out, const json & val, int indent, int level, const std::string & item_sep, const std::string & key_sep, bool ensure_ascii) {
+ const auto newline = [&](int lvl) {
+ return indent < 0 ? std::string() : "\n" + std::string(indent * lvl, ' ');
+ };
+ if (val.is_null()) {
+ out += "null";
+ } else if (val.is_boolean()) {
+ out += val.get<bool>() ? "true" : "false";
+ } else if (val.is_number_float()) {
+ out += decision_py_float(val.get<double>());
+ } else if (val.is_number()) {
+ out += val.dump();
+ } else if (val.is_string()) {
+ decision_py_str(out, val.get<std::string>(), ensure_ascii);
+ } else if (val.is_array()) {
+ if (val.empty()) {
+ out += "[]";
+ return;
+ }
+ out += "[" + newline(level + 1);
+ for (size_t i = 0; i < val.size(); i++) {
+ if (i > 0) {
+ out += item_sep + newline(level + 1);
+ }
+ decision_py_dump(out, val.at(i), indent, level + 1, item_sep, key_sep, ensure_ascii);
+ }
+ out += newline(level) + "]";
+ } else if (val.is_object()) {
+ if (val.empty()) {
+ out += "{}";
+ return;
+ }
+ out += "{" + newline(level + 1);
+ bool first = true;
+ for (const auto & [key, item] : val.items()) {
+ if (!first) {
+ out += item_sep + newline(level + 1);
+ }
+ first = false;
+ decision_py_str(out, key, ensure_ascii);
+ out += key_sep;
+ decision_py_dump(out, item, indent, level + 1, item_sep, key_sep, ensure_ascii);
+ }
+ out += newline(level) + "}";
+ }
+}
+
+static std::string decision_py_dumps(const json & val, int indent, const std::string & item_sep, const std::string & key_sep, bool ensure_ascii) {
+ std::string out;
+ decision_py_dump(out, val, indent, 0, item_sep, key_sep, ensure_ascii);
+ return out;
+}
+
+// a state longer than max_state_tokens keeps its head and its tail around a "..." line
+std::string server_decision_context::h2o_cut_state(const std::string & text) const {
+ // a byte-level BPE never has more tokens than bytes
+ const size_t max_state_tokens = DECISION_H2O_MAX_STATE_TOKENS;
+ if (text.size() <= max_state_tokens) {
+ return text;
+ }
+ const llama_tokens tokens = common_tokenize(vocab, text, false, true);
+ if (tokens.size() <= max_state_tokens) {
+ return text;
+ }
+ // each half is the text of its token pieces, a cut inside a character gives U+FFFD as in a JSON response of the server
+ const size_t half = max_state_tokens / 2;
+ auto text_of = [&](llama_tokens::const_iterator begin, llama_tokens::const_iterator end) {
+ std::string out;
+ for (auto it = begin; it != end; ++it) {
+ out += common_token_to_piece(vocab, *it, true);
+ }
+ return json::parse(json::make(out).dump_safe()).get<std::string>();
+ };
+ return text_of(tokens.begin(), tokens.begin() + half) + "\n...\n" + text_of(tokens.end() - half, tokens.end());
+}
+
+// the special token texts written in the input are quoted with JSON escapes, so they stay text
+std::string server_decision_context::h2o_quote(const std::string & text) const {
+ std::string out;
+ for (size_t i = 0; i < text.size();) {
+ bool is_quoted = false;
+ if (quoted_first.find(text[i]) == std::string::npos) {
+ out += text[i++];
+ continue;
+ }
+ for (const auto & spelling : quoted_spellings) {
+ if (text.compare(i, spelling.size(), spelling) == 0) {
+ std::string quoted = spelling;
+ string_replace_all(quoted, "<", "\\u003c");
+ string_replace_all(quoted, ">", "\\u003e");
+ out += quoted;
+ i += spelling.size();
+ is_quoted = true;
+ break;
+ }
+ }
+ if (!is_quoted) {
+ out += text[i++];
+ }
+ }
+ return out;
+}
+
+json server_decision_context::prepare_state(const json & state) const {
+ if (type != COMMON_DECISION_TYPE_H2O_LIGHTNING) {
+ return state;
+ }
+ // done once per request: compact JSON, the cut, the quoting
+ return h2o_quote(h2o_cut_state(state.is_string() ? state.get<std::string>() : decision_py_dumps(state, -1, ",", ":", false)));
+}
+
+json server_decision_context::render_h2o(const json & state, const server_decision_question & question) const {
+ // the instructions are JSON with an indent of 1, the descriptions are default JSON, the state is from prepare_state()
+ const std::string instructions = decision_py_lstrip(question.instructions.is_string() ?
+ question.instructions.get<std::string>() : decision_py_dumps(question.instructions, 1, ",", ": ", true));
+
+ json options = json::array();
+ for (size_t i = 0; i < question.options.size(); i++) {
+ const auto & opt = question.options[i];
+ json description; // null if not provided, the template has a default, except a score level that is shown as JSON null
+ if (question.type == SERVER_DECISION_QUESTION_SCORE && opt.description.is_null()) {
+ description = "null";
+ } else if (!opt.description.is_null()) {
+ description = h2o_quote(opt.description.is_string() ? opt.description.get<std::string>() : decision_py_dumps(opt.description, -1, ", ", ": ", false));
+ }
+ const std::string key = h2o_quote(opt.key);
+
+ // the user turn is stripped like Python's str.strip(), the template only trims ASCII whitespace
+ // so the text that ends the turn is stripped here, a space keeps a description that is only whitespace
+ if (i + 1 == question.options.size()) {
+ const bool is_key = question.type == SERVER_DECISION_QUESTION_CHOICE && description.is_null();
+ if (is_key || description.is_string()) {
+ const std::string text = is_key ? key : description.get<std::string>();
+ const std::string stripped = decision_py_rstrip(text);
+ if (stripped != text) {
+ description = stripped.empty() ? " " : stripped;
+ }
+ }
+ }
+
+ json option = json::object();
+ option["label"] = label_texts[i];
+ option["key"] = key;
+ option["description"] = description;
+ options.push_back(option);
+ }
+
+ json inp = json::object();
+ inp["type"] = decision_question_type_name(question.type);
+ inp["state"] = state;
+ inp["instructions"] = h2o_quote(instructions);
+ inp["options"] = options;
+ return inp;
+}
+
// given to the lfm2-d1-omni template: text between the pieces of the prompt, and at the start of the pieces that are cut to a token budget
static const std::string D1OMNI_MARKER = "<<d1omni:";
static const std::string D1OMNI_SEP = "<<d1omni:sep>>";
@@ -672,7 +989,8 @@ std::string server_decision_context::render(
size_t n_images,
bool is_audio) const {
// the template is given raw JSON values, it serializes the ones that are not strings
- json inp = json{
+ // h2o-lightning: the template is given texts
+ json inp = type == COMMON_DECISION_TYPE_H2O_LIGHTNING ? render_h2o(state, question) : json{
{"id", question.id},
{"type", decision_question_type_name(question.type)},
{"instructions", question.instructions},
@@ -756,7 +1074,8 @@ void server_decision_context::fill_task(
const std::string prompt = render(state, questions, question, variant, files.size());
- if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_GRANITE_SWITCH || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) {
+ if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_GRANITE_SWITCH || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3 ||
+ type == COMMON_DECISION_TYPE_H2O_LIGHTNING) {
// lev reads the ratings of a noul question at its first labels, not at the digits
task.decision.labels.assign(labels.begin(), labels.begin() + n_outputs(question));
}
@@ -769,7 +1088,8 @@ void server_decision_context::fill_task(
task.decision.label_groups.push_back(group.size());
}
}
- if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3 || type == COMMON_DECISION_TYPE_LFM2_D1) {
+ if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3 || type == COMMON_DECISION_TYPE_LFM2_D1 ||
+ type == COMMON_DECISION_TYPE_H2O_LIGHTNING) {
if (!files.empty()) {
task.tokens = process_mtmd_prompt(mctx, prompt, files, init_opt);
return;
@@ -1181,7 +1501,12 @@ json server_decision_context::format_answer(const server_decision_question & que
// softmax over the outputs of each variant, then the average of the variants
// lfm2-d1-omni: image and audio answers are not calibrated
- const float temperature = has_media && type == COMMON_DECISION_TYPE_LFM2_D1_OMNI ? 1.0f : get_temperature(question);
+ float temperature = has_media && type == COMMON_DECISION_TYPE_LFM2_D1_OMNI ? 1.0f : get_temperature(question);
+ // a model can have its own temperature for requests with media
+ const auto it_image = temperatures.find("image");
+ if (has_media && it_image != temperatures.end()) {
+ temperature = it_image->second;
+ }
std::vector<double> probs(n, 0.0);
for (size_t v = 0; v < scores.size(); v++) {
const auto & s = scores[v];
diff --git a/tools/server/server-decision.h b/tools/server/server-decision.h
index ed9724429..b631099a8 100644
--- a/tools/server/server-decision.h
+++ b/tools/server/server-decision.h
@@ -46,6 +46,7 @@ struct server_decision_context {
case COMMON_DECISION_TYPE_PPLX_DECIDER:
case COMMON_DECISION_TYPE_VLLM_DECISION2:
case COMMON_DECISION_TYPE_LFM2_D1:
+ case COMMON_DECISION_TYPE_H2O_LIGHTNING:
return true;
default:
return false;
@@ -66,6 +67,7 @@ struct server_decision_context {
case COMMON_DECISION_TYPE_VLLM_D3:
case COMMON_DECISION_TYPE_LFM2_D1:
case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
+ case COMMON_DECISION_TYPE_H2O_LIGHTNING:
return true;
default:
return false;
@@ -75,6 +77,9 @@ struct server_decision_context {
// throw std::invalid_argument on bad input
std::vector<server_decision_question> parse_questions(const json & body) const;
+ // the state as the model reads it, done once per request; returns the state for most models
+ json prepare_state(const json & state) const;
+
// returns the state without its images, they are appended to files in order
// images come from "files" (alias "images") and from the image_url and input_audio parts of a state made of chat messages
// an image can be an audio clip if the model supports it
@@ -121,7 +126,7 @@ private:
bool strict_payload = false; // state, instructions and descriptions must be text, an object or an array
bool canonical_legend = false; // non-string score descriptions are returned as canonical JSON text
- // OPENJEV, LEV, NIMBLE, PPLX_DECIDER, VLLM_D3
+ // OPENJEV, LEV, NIMBLE, PPLX_DECIDER, VLLM_D3, H2O_LIGHTNING
std::vector<llama_token> labels;
std::vector<std::string> label_texts; // only if the label of an option is given to the template
@@ -132,6 +137,10 @@ private:
size_t max_head_tokens = 0; // question + options
size_t max_option_tokens = 48;
+ // H2O_LIGHTNING
+ std::vector<std::string> quoted_spellings; // texts of the special tokens, quoted in the input, longest first
+ std::string quoted_first; // the first bytes of these texts
+
std::string render(
const json & state,
const std::vector<server_decision_question> & questions,
@@ -143,6 +152,10 @@ private:
size_t n_outputs(const server_decision_question & question) const;
// LFM2_D1: label text and tokens of each option
void d1_labels(const server_decision_question & question, std::vector<std::string> & texts, std::vector<llama_tokens> & groups) const;
+ // H2O_LIGHTNING: the input of the template, its texts are rendered as the reference shim does
+ json render_h2o(const json & state, const server_decision_question & question) const;
+ std::string h2o_cut_state(const std::string & text) const;
+ std::string h2o_quote(const std::string & text) const;
void fill_task_laya(llama_tokens & tokens, const server_decision_question & question, server_task & task) const;
void fill_task_d1omni(
const json & state,