Commit f4b5326 for stable-diffusion.cpp
commit f4b5326da7ca4168dd1e0726de21400a10a9a1cc
Author: Wagner Bruna <wbruna@users.noreply.github.com>
Date: Tue Oct 6 06:50:16 2026 -0300
feat: support for merging LoRA weights on model conversion (#2079)
Co-authored-by: leejet <leejet714@gmail.com>
diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp
index dc79b8c..21c1ce7 100644
--- a/examples/cli/main.cpp
+++ b/examples/cli/main.cpp
@@ -716,6 +716,15 @@ int main(int argc, const char* argv[]) {
}
if (cli_params.mode == CONVERT) {
+ std::vector<sd_lora_t> loras;
+ loras.reserve(gen_params.lora_map.size() + gen_params.high_noise_lora_map.size());
+ for (const auto& [path, multiplier] : gen_params.lora_map) {
+ loras.push_back({false, multiplier, path.c_str()});
+ }
+ for (const auto& [path, multiplier] : gen_params.high_noise_lora_map) {
+ loras.push_back({true, multiplier, path.c_str()});
+ }
+
bool success = convert_with_components(ctx_params.model_path.c_str(),
ctx_params.clip_l_path.c_str(),
ctx_params.clip_g_path.c_str(),
@@ -726,7 +735,9 @@ int main(int argc, const char* argv[]) {
ctx_params.wtype,
ctx_params.tensor_type_rules.c_str(),
cli_params.convert_name,
- ctx_params.n_threads);
+ ctx_params.n_threads,
+ loras.empty() ? nullptr : loras.data(),
+ (int)loras.size());
if (!success) {
LOG_ERROR("convert '%s'/'%s' to '%s' failed",
ctx_params.model_path.c_str(),
diff --git a/examples/common/common.cpp b/examples/common/common.cpp
index dc15b4e..89b226f 100644
--- a/examples/common/common.cpp
+++ b/examples/common/common.cpp
@@ -2271,14 +2271,15 @@ bool SDGenerationParams::from_json_str(
return true;
}
-void SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_dir) {
+bool SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_dir) {
if (lora_model_dir.empty()) {
- return;
+ return true;
}
static const std::regex re(R"(<lora:([^:>]+):([^>]+)>)");
static const std::vector<std::string> valid_ext = {".gguf", ".safetensors", ".pt", ".ckpt"};
std::smatch m;
+ bool missing = false;
std::string tmp = prompt;
while (std::regex_search(tmp, m, re)) {
@@ -2320,8 +2321,9 @@ void SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_d
}
if (!found) {
LOG_WARN("can not found lora %s", final_path.lexically_normal().string().c_str());
- tmp = m.suffix().str();
- prompt = std::regex_replace(prompt, re, "", std::regex_constants::format_first_only);
+ tmp = m.suffix().str();
+ prompt = std::regex_replace(prompt, re, "", std::regex_constants::format_first_only);
+ missing = true;
continue;
}
}
@@ -2337,6 +2339,8 @@ void SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_d
tmp = m.suffix().str();
}
+
+ return !missing;
}
bool SDGenerationParams::width_and_height_are_set() const {
@@ -2458,7 +2462,7 @@ bool SDGenerationParams::initialize_cache_params() {
return true;
}
-bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict) {
+bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict, bool validate_missing_loras) {
vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str();
if (high_noise_sample_params.sample_steps <= 0) {
@@ -2502,7 +2506,9 @@ bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::s
prompt_with_lora = prompt;
if (!lora_model_dir.empty()) {
- extract_and_remove_lora(lora_model_dir);
+ if (!extract_and_remove_lora(lora_model_dir) && validate_missing_loras) {
+ return false;
+ }
}
return true;
}
@@ -2630,7 +2636,7 @@ bool SDGenerationParams::resolve_and_validate(SDMode mode,
const std::string& lora_model_dir,
const std::string& hires_upscalers_dir,
bool strict) {
- if (!resolve(lora_model_dir, hires_upscalers_dir, strict)) {
+ if (!resolve(lora_model_dir, hires_upscalers_dir, strict, mode == CONVERT)) {
return false;
}
if (!validate(mode)) {
diff --git a/examples/common/common.h b/examples/common/common.h
index d0d39b7..8b60c76 100644
--- a/examples/common/common.h
+++ b/examples/common/common.h
@@ -313,12 +313,12 @@ struct SDGenerationParams {
const std::function<std::string(const std::string&)>& lora_path_resolver = {});
bool parse_image_preprocess_json(const std::string& json_str);
bool initialize_cache_params();
- void extract_and_remove_lora(const std::string& lora_model_dir);
+ bool extract_and_remove_lora(const std::string& lora_model_dir);
bool width_and_height_are_set() const;
void set_width_and_height_if_unset(int w, int h);
int get_resolved_width() const;
int get_resolved_height() const;
- bool resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict = false);
+ bool resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict = false, bool validate_missing_loras = false);
bool validate(SDMode mode);
bool resolve_and_validate(SDMode mode,
const std::string& lora_model_dir,
diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h
index 8e5577d..cfedbac 100644
--- a/include/stable-diffusion.h
+++ b/include/stable-diffusion.h
@@ -601,7 +601,9 @@ SD_API bool convert_with_components(const char* model_path,
enum sd_type_t output_type,
const char* tensor_type_rules,
bool convert_name,
- int n_threads);
+ int n_threads,
+ const sd_lora_t* loras,
+ int lora_count);
SD_API bool preprocess_canny(sd_image_t image,
float high_threshold,
diff --git a/src/convert.cpp b/src/convert.cpp
index c82d255..7fb0825 100644
--- a/src/convert.cpp
+++ b/src/convert.cpp
@@ -4,6 +4,7 @@
#include <cstring>
#include <exception>
#include <fstream>
+#include <map>
#include <memory>
#include <mutex>
#include <regex>
@@ -11,11 +12,15 @@
#include <thread>
#include <vector>
+#include "core/ggml_extend_backend.h"
#include "core/util.h"
#include "model_io/gguf_io.h"
#include "model_io/safetensors_io.h"
#include "model_io/streaming_writer.h"
#include "model_loader.h"
+#include "model_manager.h"
+#include "name_conversion.h"
+#include "runtime/imatrix.h"
struct TensorExportInfo {
TensorStorage storage;
@@ -75,9 +80,9 @@ static size_t export_tensor_nbytes(const TensorExportInfo& info) {
return static_cast<size_t>(output_storage.nbytes());
}
-static TensorWritePlan tensor_write_plan_from_export_info(const TensorExportInfo& info) {
+static TensorWritePlan tensor_write_plan_from_export_info(const TensorExportInfo& info, const std::string& name) {
TensorWritePlan plan;
- plan.name = info.storage.name;
+ plan.name = name;
plan.type = info.type;
plan.n_dims = info.storage.n_dims;
for (int i = 0; i < SD_MAX_DIMS; i++) {
@@ -86,11 +91,14 @@ static TensorWritePlan tensor_write_plan_from_export_info(const TensorExportInfo
return plan;
}
-static std::vector<TensorWritePlan> tensor_write_plans_from_export_infos(const std::vector<TensorExportInfo>& tensors) {
+static std::vector<TensorWritePlan> tensor_write_plans_from_export_infos(const std::vector<TensorExportInfo>& tensors,
+ bool convert_name,
+ SDVersion version) {
std::vector<TensorWritePlan> plans;
plans.reserve(tensors.size());
for (const TensorExportInfo& info : tensors) {
- plans.push_back(tensor_write_plan_from_export_info(info));
+ const std::string name = convert_name ? convert_tensor_name(info.storage.name, version) : info.storage.name;
+ plans.push_back(tensor_write_plan_from_export_info(info, name));
}
return plans;
}
@@ -158,16 +166,42 @@ static bool load_tensor_for_export(ModelLoader& model_loader, TensorExportJob& j
return true;
}
+static bool export_tensor_from_memory(const TensorExportInfo& info,
+ const std::map<std::string, ggml_tensor*>& mem,
+ SDVersion version,
+ std::vector<uint8_t>& out) {
+ auto found = mem.find(info.storage.name);
+ if (found == mem.end() || found->second == nullptr || found->second->data == nullptr) {
+ return false;
+ }
+ const TensorStorage& storage = info.storage;
+ out.resize(export_tensor_nbytes(info));
+ if (out.size() > 0) {
+ std::vector<float> imatrix = get_imatrix_collector().get_values(convert_tensor_name(info.storage.name, version));
+ convert_tensor(found->second->data,
+ found->second->type,
+ out.data(),
+ info.type,
+ (int)(storage.nelements() / storage.ne[0]),
+ (int)storage.ne[0],
+ std::move(imatrix));
+ }
+ return true;
+}
+
static bool stream_tensor_data(ModelLoader& model_loader,
const std::string& output_path,
const std::vector<TensorExportInfo>& tensors,
const StreamingModelWriter& writer,
int n_threads,
+ const std::map<std::string, ggml_tensor*>* mem,
std::string* error) {
n_threads = n_threads > 0 ? n_threads : sd_get_num_physical_cores();
n_threads = std::max(1, n_threads);
LOG_INFO("streaming convert with %d threads", n_threads);
+ const SDVersion version = model_loader.get_sd_version();
+
int64_t start_time = ggml_time_ms();
uint64_t bytes_written = 0;
size_t tensors_written = 0;
@@ -241,7 +275,10 @@ static bool stream_tensor_data(ModelLoader& model_loader,
TensorExportJob job;
job.info = tensors[tensor_index];
try {
- job.success = load_tensor_for_export(model_loader, job);
+ job.success = (mem != nullptr) && export_tensor_from_memory(job.info, *mem, version, job.data);
+ if (!job.success) {
+ job.success = load_tensor_for_export(model_loader, job);
+ }
} catch (const std::exception& e) {
job.error = e.what();
job.success = false;
@@ -298,8 +335,10 @@ static bool write_model_file_streaming(ModelLoader& model_loader,
const std::vector<TensorExportInfo>& tensors,
StreamingModelWriter& writer,
int n_threads,
+ bool convert_name,
+ const std::map<std::string, ggml_tensor*>* mem,
std::string* error) {
- std::vector<TensorWritePlan> plans = tensor_write_plans_from_export_infos(tensors);
+ std::vector<TensorWritePlan> plans = tensor_write_plans_from_export_infos(tensors, convert_name, model_loader.get_sd_version());
if (!writer.write_metadata(output_path, plans, error)) {
return false;
}
@@ -307,7 +346,7 @@ static bool write_model_file_streaming(ModelLoader& model_loader,
return false;
}
model_loader.process_model_files(false, false);
- return stream_tensor_data(model_loader, output_path, tensors, writer, n_threads, error);
+ return stream_tensor_data(model_loader, output_path, tensors, writer, n_threads, mem, error);
}
static bool init_convert_path(ModelLoader& model_loader, const char* path, const char* prefix, bool& loaded_any) {
@@ -326,7 +365,9 @@ static bool export_loaded_model(ModelLoader& model_loader,
const char* output_path,
sd_type_t output_type,
const char* tensor_type_rules,
- int n_threads) {
+ int n_threads,
+ bool convert_name,
+ const std::map<std::string, ggml_tensor*>* mem = nullptr) {
ggml_type type = sd_type_to_ggml_type(output_type);
bool output_is_safetensors = ends_with(output_path, ".safetensors");
TensorTypeRules type_rules = parse_tensor_type_rules(tensor_type_rules);
@@ -341,7 +382,7 @@ static bool export_loaded_model(ModelLoader& model_loader,
} else {
writer = std::make_unique<GGUFStreamingWriter>();
}
- success = write_model_file_streaming(model_loader, output_path, tensors, *writer, n_threads, &error);
+ success = write_model_file_streaming(model_loader, output_path, tensors, *writer, n_threads, convert_name, mem, &error);
}
if (!success && !error.empty()) {
@@ -351,6 +392,174 @@ static bool export_loaded_model(ModelLoader& model_loader,
return success;
}
+static bool has_active_loras(const sd_lora_t* loras, int lora_count) {
+ if (loras == nullptr) {
+ return false;
+ }
+ for (int i = 0; i < lora_count; i++) {
+ if (loras[i].multiplier != 0.0f) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static std::vector<ModelManager::LoraSpec> build_lora_specs(const sd_lora_t* loras, int lora_count) {
+ std::vector<ModelManager::LoraSpec> specs;
+ specs.reserve(lora_count);
+ for (int i = 0; i < lora_count; i++) {
+ ModelManager::LoraSpec spec;
+ spec.path = loras[i].path != nullptr ? loras[i].path : "";
+ spec.multiplier = loras[i].multiplier;
+ spec.is_high_noise = loras[i].is_high_noise;
+ spec.required = true;
+ specs.push_back(std::move(spec));
+ }
+ return specs;
+}
+
+static ModelComponent component_for_tensor_name(const std::string& name) {
+ if (starts_with(name, "text_encoders.") || starts_with(name, "cond_stage_model.")) {
+ return ModelComponent::Conditioner;
+ }
+ if (starts_with(name, "model.high_noise_diffusion_model.")) {
+ return ModelComponent::HighNoiseDiffusion;
+ }
+ if (starts_with(name, "model.diffusion_model.")) {
+ return ModelComponent::Diffusion;
+ }
+ if (starts_with(name, "vae.") || starts_with(name, "first_stage_model.")) {
+ return ModelComponent::VAE;
+ }
+ return ModelComponent::Diffusion;
+}
+
+static bool load_model_into_memory(ModelLoader& model_loader,
+ const sd_lora_t* loras,
+ int lora_count,
+ int n_threads,
+ ModelManager& manager,
+ ggml_backend_t cpu,
+ ggml_context* ctx,
+ std::map<std::string, ggml_tensor*>& mem,
+ std::vector<ggml_tensor*>& all_tensors,
+ ggml_type output_type,
+ const TensorTypeRules& type_rules) {
+ const String2TensorStorage& storage_map = model_loader.get_tensor_storage_map();
+ if (storage_map.empty()) {
+ LOG_ERROR("no tensors to load into memory for convert");
+ return false;
+ }
+
+ const SDVersion version = model_loader.get_sd_version();
+ ModelLoader merge_loader = model_loader;
+ merge_loader.convert_tensors_name();
+
+ std::map<ModelComponent, std::map<std::string, ggml_tensor*>> groups;
+ size_t total_bytes = 0;
+ for (const auto& [name, storage] : storage_map) {
+ ggml_type load_type = storage.type;
+ const ggml_type export_type = get_export_tensor_type(model_loader, storage, output_type, type_rules);
+ const bool quant_src = ggml_is_quantized(storage.type);
+ const bool quant_dst = ggml_is_quantized(export_type);
+ if (quant_src && quant_dst) {
+ load_type = GGML_TYPE_F16;
+ } else if (quant_src || quant_dst) {
+ load_type = quant_src ? export_type : storage.type;
+ } else if (storage.type == GGML_TYPE_F32 || export_type == GGML_TYPE_F32) {
+ load_type = GGML_TYPE_F32;
+ }
+ ggml_tensor* tensor = ggml_new_tensor(ctx, load_type, storage.n_dims, storage.ne);
+ if (tensor == nullptr) {
+ LOG_ERROR("failed to create tensor '%s' for in-memory convert", name.c_str());
+ return false;
+ }
+ const std::string merge_name = convert_tensor_name(name, version);
+ auto& group = groups[component_for_tensor_name(merge_name)];
+ if (!group.emplace(merge_name, tensor).second) {
+ LOG_ERROR("duplicate tensor name '%s' after name conversion", merge_name.c_str());
+ return false;
+ }
+ mem[name] = tensor;
+ all_tensors.push_back(tensor);
+ total_bytes += ggml_nbytes(tensor);
+ }
+ LOG_INFO("loading %zu tensors (%.2f GB) into memory for convert",
+ all_tensors.size(), total_bytes / (1024.0 * 1024.0 * 1024.0));
+
+ manager.set_n_threads(n_threads);
+ manager.set_enable_mmap(false);
+ if (!manager.set_loader(std::move(merge_loader))) {
+ LOG_ERROR("failed to set model loader for in-memory convert");
+ return false;
+ }
+
+ for (const auto& [component, tensors] : groups) {
+ if (!manager.register_param_tensors(component, tensors, ModelManager::ResidencyMode::ParamBackend, cpu, cpu)) {
+ LOG_ERROR("failed to register %s tensors for in-memory convert", model_component_name(component));
+ return false;
+ }
+ }
+
+ std::vector<ModelManager::LoraSpec> lora_specs = build_lora_specs(loras, lora_count);
+ if (!manager.prepare_lora_sources(lora_specs)) {
+ LOG_ERROR("failed to prepare LoRA sources for convert");
+ return false;
+ }
+ if (!manager.set_loras(lora_specs, version)) {
+ LOG_ERROR("failed to set LoRAs for convert");
+ return false;
+ }
+ if (!manager.prepare_params(all_tensors)) {
+ LOG_ERROR("failed to load model into memory for convert");
+ return false;
+ }
+
+ return true;
+}
+
+static bool convert_model_in_memory(ModelLoader& model_loader,
+ const char* output_path,
+ sd_type_t output_type,
+ const char* tensor_type_rules,
+ bool convert_name,
+ int n_threads,
+ const sd_lora_t* loras,
+ int lora_count) {
+ auto cpu = std::unique_ptr<ggml_backend, decltype(&ggml_backend_free)>(sd_backend_cpu_init(), ggml_backend_free);
+ if (cpu == nullptr) {
+ LOG_ERROR("failed to init CPU backend for in-memory convert");
+ return false;
+ }
+
+ ggml_init_params ctx_params;
+ ctx_params.mem_size = model_loader.get_tensor_storage_map().size() * ggml_tensor_overhead();
+ ctx_params.mem_buffer = nullptr;
+ ctx_params.no_alloc = true;
+ ggml_context* ctx = ggml_init(ctx_params);
+ if (ctx == nullptr) {
+ LOG_ERROR("ggml_init failed for in-memory convert");
+ return false;
+ }
+
+ ggml_type type = sd_type_to_ggml_type(output_type);
+ TensorTypeRules type_rules = parse_tensor_type_rules(tensor_type_rules);
+
+ std::vector<ggml_tensor*> all_tensors;
+ std::map<std::string, ggml_tensor*> mem;
+ bool success = false;
+ {
+ // The manager owns the buffers backing `mem` and must outlive the export; it is
+ // destroyed (freeing those buffers) before the tensor context is freed below.
+ ModelManager manager;
+ if (load_model_into_memory(model_loader, loras, lora_count, n_threads, manager, cpu.get(), ctx, mem, all_tensors, type, type_rules)) {
+ success = export_loaded_model(model_loader, output_path, output_type, tensor_type_rules, n_threads, convert_name, &mem);
+ }
+ }
+ ggml_free(ctx);
+ return success;
+}
+
bool convert_with_components(const char* model_path,
const char* clip_l_path,
const char* clip_g_path,
@@ -361,10 +570,22 @@ bool convert_with_components(const char* model_path,
sd_type_t output_type,
const char* tensor_type_rules,
bool convert_name,
- int n_threads) {
+ int n_threads,
+ const sd_lora_t* loras,
+ int lora_count) {
if (!validate_tensor_types(output_type, tensor_type_rules)) {
return false;
}
+
+ if (loras != nullptr) {
+ for (int i = 0; i < lora_count; i++) {
+ LOG_INFO("lora %d: '%s' (multiplier %.2f%s)",
+ i + 1, loras[i].path != nullptr ? loras[i].path : "",
+ loras[i].multiplier,
+ loras[i].is_high_noise ? ", high noise" : "");
+ }
+ }
+
ModelLoader model_loader;
bool loaded_any = false;
@@ -382,11 +603,17 @@ bool convert_with_components(const char* model_path,
return false;
}
+ if (has_active_loras(loras, lora_count)) {
+ LOG_INFO("loading the full model into memory to apply LoRAs");
+ return convert_model_in_memory(model_loader, output_path, output_type, tensor_type_rules, convert_name, n_threads,
+ loras, lora_count);
+ }
+
if (convert_name) {
model_loader.convert_tensors_name();
}
- return export_loaded_model(model_loader, output_path, output_type, tensor_type_rules, n_threads);
+ return export_loaded_model(model_loader, output_path, output_type, tensor_type_rules, n_threads, false, nullptr);
}
bool convert(const char* input_path,
@@ -405,5 +632,7 @@ bool convert(const char* input_path,
output_type,
tensor_type_rules,
convert_name,
+ 0,
+ nullptr,
0);
}
diff --git a/src/model_loader.cpp b/src/model_loader.cpp
index 7428e2d..4c16065 100644
--- a/src/model_loader.cpp
+++ b/src/model_loader.cpp
@@ -115,7 +115,7 @@ void convert_tensor(void* src,
ggml_type dst_type,
int nrows,
int n_per_row,
- std::vector<float> imatrix = {}) {
+ std::vector<float> imatrix) {
int n = nrows * n_per_row;
if (src_type == dst_type) {
size_t nbytes = n * ggml_type_size(src_type) / ggml_blck_size(src_type);
diff --git a/src/model_loader.h b/src/model_loader.h
index e51b260..63ecfb6 100644
--- a/src/model_loader.h
+++ b/src/model_loader.h
@@ -13,6 +13,14 @@
TensorTypeRules parse_tensor_type_rules(const std::string& tensor_type_rules);
+void convert_tensor(void* src,
+ ggml_type src_type,
+ void* dst,
+ ggml_type dst_type,
+ int nrows,
+ int n_per_row,
+ std::vector<float> imatrix = {});
+
class MmapWrapper;
struct ModelFileData {