Commit ab8d5ea for stable-diffusion.cpp
commit ab8d5ea8c3a0e321656348eb2a018695d9d5e541
Author: leejet <leejet714@gmail.com>
Date: Sun Oct 18 14:50:36 2026 +0800
feat: support Diffusers-format MiniMax-H3 LoRAs (#2159)
diff --git a/src/model/adapter/lora.hpp b/src/model/adapter/lora.hpp
index e9d20a4..1e5377c 100644
--- a/src/model/adapter/lora.hpp
+++ b/src/model/adapter/lora.hpp
@@ -1,6 +1,7 @@
#ifndef __SD_MODEL_ADAPTER_LORA_HPP__
#define __SD_MODEL_ADAPTER_LORA_HPP__
+#include <cmath>
#include <mutex>
#include "core/ggml_extend.h"
#include "core/ggml_extend_backend.h"
@@ -11,15 +12,18 @@
#include "model/adapter/lora_ops.h"
#include "model_loader.h"
#include "model_manager.h"
+#include "name_conversion.h"
#define LORA_GRAPH_BASE_SIZE 10240
struct LoraModel : public GGMLRunner {
std::string lora_id;
- float multiplier = 1.0f;
+ float multiplier = 1.0f;
+ float metadata_alpha = -1.f;
std::unordered_map<std::string, ggml_tensor*> lora_tensors;
std::map<ggml_tensor*, ggml_tensor*> original_tensor_to_final_tensor;
std::set<std::string> applied_lora_tensors;
+ std::set<std::string> swapped_lora_up_tensors;
std::set<std::string> skipped_incompatible_lora_tensors;
std::set<std::string> warned_incompatible_model_tensors;
std::string file_path;
@@ -62,6 +66,31 @@ struct LoraModel : public GGMLRunner {
if (load_failed || !registered_params.empty())
return false;
model_manager->set_n_threads(n_threads);
+ metadata_alpha = -1.f;
+ const auto metadata = model_manager->loader().file_metadata(source_file);
+ const auto alpha = metadata.find("alpha");
+ if (alpha != metadata.end()) {
+ try {
+ size_t parsed = 0;
+ metadata_alpha = std::stof(alpha->second, &parsed);
+ if (parsed != alpha->second.size() || !std::isfinite(metadata_alpha) || metadata_alpha < 0.f) {
+ throw std::invalid_argument("invalid alpha");
+ }
+ } catch (const std::exception&) {
+ LOG_ERROR("invalid LoRA alpha metadata '%s' in '%s'", alpha->second.c_str(), file_path.c_str());
+ return false;
+ }
+ LOG_VERBOSE("LoRA '%s' uses metadata alpha %.6g when tensor alpha/scale is absent", file_path.c_str(), metadata_alpha);
+ }
+ if (sd_version_is_minimax_h3(source_version)) {
+ for (const auto& entry : model_manager->loader().file_tensors(source_file)) {
+ const auto name = convert_tensor_name(entry.first, VERSION_COUNT);
+ if (ends_with(name, ".ff.net.0.proj.weight.lora_up")) {
+ // Diffusers SwiGLU stores [value, gate]; MiniMax-H3 uses [gate, value].
+ swapped_lora_up_tensors.insert(convert_tensor_name(entry.first, source_version));
+ }
+ }
+ }
const auto sources = model_manager->loader().file_tensors(source_file, source_version);
std::map<std::string, ggml_tensor*> tensors;
std::map<std::string, ggml_tensor*> scalars;
@@ -116,6 +145,8 @@ struct LoraModel : public GGMLRunner {
scalar_values.clear();
original_tensor_to_final_tensor.clear();
applied_lora_tensors.clear();
+ swapped_lora_up_tensors.clear();
+ metadata_alpha = -1.f;
skipped_incompatible_lora_tensors.clear();
warned_incompatible_model_tensors.clear();
applied = false;
@@ -210,6 +241,10 @@ struct LoraModel : public GGMLRunner {
auto iter = lora_tensors.find(lora_up_name);
if (iter != lora_tensors.end()) {
lora_up = ggml_ext_cast_f32(ctx, backend, iter->second);
+ if (swapped_lora_up_tensors.count(lora_up_name)) {
+ auto halves = ggml_ext_chunk(ctx, lora_up, 2, 1);
+ lora_up = ggml_concat(ctx, halves[1], halves[0], 1);
+ }
}
iter = lora_tensors.find(lora_mid_name);
@@ -236,6 +271,7 @@ struct LoraModel : public GGMLRunner {
float scale_value = 1.0f;
int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1];
+ scale_value = metadata_alpha >= 0.f ? metadata_alpha / rank : 1.f;
iter = lora_tensors.find(scale_name);
if (iter != lora_tensors.end()) {
scale_value = scalar_value(iter->second);
@@ -723,6 +759,10 @@ struct LoraModel : public GGMLRunner {
iter = lora_tensors.find(lora_up_name);
if (iter != lora_tensors.end()) {
lora_up = iter->second;
+ if (swapped_lora_up_tensors.count(lora_up_name)) {
+ auto halves = ggml_ext_chunk(ctx, lora_up, 2, 1);
+ lora_up = ggml_concat(ctx, halves[1], halves[0], 1);
+ }
if (is_conv2d && lora_up->type != GGML_TYPE_F16) {
lora_up = ggml_cast(ctx, lora_up, GGML_TYPE_F16);
}
@@ -791,6 +831,7 @@ struct LoraModel : public GGMLRunner {
std::string scale_tensor_name;
int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1];
+ scale_value = metadata_alpha >= 0.f ? metadata_alpha / rank : 1.f;
iter = lora_tensors.find(scale_name);
if (iter != lora_tensors.end()) {
scale_value = scalar_value(iter->second);
diff --git a/src/model_loader.h b/src/model_loader.h
index 63ecfb6..aae3590 100644
--- a/src/model_loader.h
+++ b/src/model_loader.h
@@ -105,6 +105,8 @@ public:
bool del_file(FileId id);
uint64_t file_revision(FileId id) const;
std::string file_path(FileId id) const;
+ std::map<std::string, std::string> file_metadata(FileId id) const;
+ String2TensorStorage file_tensors(FileId id) const;
String2TensorStorage file_tensors(FileId id, SDVersion version) const;
bool load_file_tensors(FileId id, SDVersion version, on_new_tensor_cb_t callback, const std::set<std::string>& names, bool use_mmap = false) const;
bool refresh_files(bool include_isolated = true);
diff --git a/src/model_loader_files.cpp b/src/model_loader_files.cpp
index 8e838b1..9ac5456 100644
--- a/src/model_loader_files.cpp
+++ b/src/model_loader_files.cpp
@@ -324,6 +324,24 @@ ModelLoader ModelLoader::file_reader(FileId id, SDVersion version) const {
return reader;
}
+std::map<std::string, std::string> ModelLoader::file_metadata(FileId id) const {
+ for (const auto& file : files_) {
+ if (file.id == id) {
+ return file.metadata;
+ }
+ }
+ return {};
+}
+
+String2TensorStorage ModelLoader::file_tensors(FileId id) const {
+ for (const auto& file : files_) {
+ if (file.id == id) {
+ return file.tensors;
+ }
+ }
+ return {};
+}
+
String2TensorStorage ModelLoader::file_tensors(FileId id, SDVersion version) const {
return file_reader(id, version).tensor_storage_map;
}
diff --git a/src/name_conversion.cpp b/src/name_conversion.cpp
index a342eb8..8f9f1fc 100644
--- a/src/name_conversion.cpp
+++ b/src/name_conversion.cpp
@@ -1024,6 +1024,24 @@ static std::string convert_ming_image_dit_name(std::string name) {
return name;
}
+std::string convert_diffusers_dit_to_original_minimax_h3(std::string name) {
+ static const std::vector<std::pair<std::string, std::string>> prefix_map = {
+ {"transformer_blocks.", "blocks."},
+ {"token_refiner.refiner_blocks.", "token_refiner.blocks."},
+ };
+ static const std::vector<std::pair<std::string, std::string>> name_map = {
+ {".attn.to_q.weight", ".attn.qkv_proj.weight"},
+ {".attn.to_k.weight", ".attn.qkv_proj.weight.1"},
+ {".attn.to_v.weight", ".attn.qkv_proj.weight.2"},
+ {".attn.to_out.0.", ".attn.out_proj."},
+ {".ff.net.0.proj.", ".mlp.fc1."},
+ {".ff.net.2.", ".mlp.fc2."},
+ };
+ replace_with_prefix_map(name, prefix_map);
+ replace_with_name_map(name, name_map);
+ return name;
+}
+
std::string convert_diffusion_model_name(std::string name, std::string prefix, SDVersion version) {
if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) {
name = convert_diffusers_unet_to_original_sd1(name);
@@ -1047,6 +1065,8 @@ std::string convert_diffusion_model_name(std::string name, std::string prefix, S
name = convert_diffusers_dit_to_original_krea2(name);
} else if (sd_version_is_pixart(version)) {
name = convert_diffusers_dit_to_original_pixart(name);
+ } else if (sd_version_is_minimax_h3(version)) {
+ name = convert_diffusers_dit_to_original_minimax_h3(name);
}
return name;
}