Commit 9b7da27 for stable-diffusion.cpp
commit 9b7da27374d30345f1c576de7531aa36117c0498
Author: losewayy <1134285612@qq.com>
Date: Wed Oct 21 00:06:47 2026 +0800
fix(server): route OpenAI image edits by reference conditioning support (#2167)
diff --git a/examples/server/api.md b/examples/server/api.md
index cff4e84..f2f6d32 100644
--- a/examples/server/api.md
+++ b/examples/server/api.md
@@ -155,6 +155,10 @@ image establishes the generation dimensions if `size` is omitted. Input
geometry follows `image_preprocess`: references preserve their dimensions by
default, while init and mask use the generation canvas preset.
+When the request starts without preconfigured reference images, the first uploaded image becomes the img2img init image if the loaded model does not support reference-image conditioning, the final reference list is empty, a mask is present, or an explicit numeric `strength` override is provided. Otherwise, only the final reference list is used for image conditioning, without initializing the img2img latent. Requests with preconfigured references retain their existing init-image selection.
+
+Routing uses the reference list and mask after applying `sd_cpp_extra_args`. Both `ref_images: []` and `ref_images: null` clear the reference list; `mask_image: null` clears an uploaded mask. An explicit `init_image` in `sd_cpp_extra_args`, including `null`, takes precedence over the implicit init-image selection.
+
Reference encoding then follows model presets and `ref_image_args`. To skip
input geometry for references and disable resizing before VAE encoding, include
this in `prompt`:
diff --git a/examples/server/openai_edit_policy.h b/examples/server/openai_edit_policy.h
new file mode 100644
index 0000000..c963a88
--- /dev/null
+++ b/examples/server/openai_edit_policy.h
@@ -0,0 +1,44 @@
+#ifndef __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__
+#define __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__
+
+#include "common/resource_owners.hpp"
+
+enum class OpenAIEditInitImageAction {
+ KeepExisting,
+ UseUploaded,
+ Clear,
+};
+
+inline bool openai_edit_uses_uploaded_image_as_init(bool supports_ref_image_conditioning,
+ bool has_ref_images,
+ bool has_mask,
+ bool has_explicit_strength_override) {
+ return !supports_ref_image_conditioning || !has_ref_images || has_mask || has_explicit_strength_override;
+}
+
+inline OpenAIEditInitImageAction apply_openai_edit_init_image_policy(
+ SDImageOwner& init_image,
+ SDImageOwner& uploaded_image,
+ bool uploaded_image_was_first_reference,
+ bool supports_ref_image_conditioning,
+ bool has_ref_images,
+ bool has_mask,
+ bool has_extra_init_image_override,
+ bool has_explicit_strength_override) {
+ if (!uploaded_image_was_first_reference || has_extra_init_image_override) {
+ return OpenAIEditInitImageAction::KeepExisting;
+ }
+
+ if (openai_edit_uses_uploaded_image_as_init(supports_ref_image_conditioning,
+ has_ref_images,
+ has_mask,
+ has_explicit_strength_override)) {
+ init_image = std::move(uploaded_image);
+ return OpenAIEditInitImageAction::UseUploaded;
+ }
+
+ init_image.reset();
+ return OpenAIEditInitImageAction::Clear;
+}
+
+#endif // __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__
diff --git a/examples/server/routes_openai.cpp b/examples/server/routes_openai.cpp
index af0cdfa..097b042 100644
--- a/examples/server/routes_openai.cpp
+++ b/examples/server/routes_openai.cpp
@@ -7,6 +7,7 @@
#include "common/common.h"
#include "common/media_io.h"
#include "common/resource_owners.hpp"
+#include "openai_edit_policy.h"
static std::string extract_and_remove_sd_cpp_extra_args(std::string& text) {
std::regex re("<sd_cpp_extra_args>(.*?)</sd_cpp_extra_args>");
@@ -157,7 +158,24 @@ static bool build_openai_edit_request(const httplib::Request& req,
request.gen_params.height = height;
request.gen_params.batch_count = n;
- std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
+ std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
+ const json extra_args = sd_cpp_extra_args_str.empty()
+ ? json::object()
+ : json::parse(sd_cpp_extra_args_str, nullptr, false);
+ const bool has_extra_init_image_override = extra_args.is_object() && extra_args.contains("init_image");
+ const bool has_extra_ref_images_override = extra_args.is_object() && extra_args.contains("ref_images");
+ const bool has_extra_strength_override = extra_args.is_object() &&
+ extra_args.contains("strength") &&
+ extra_args["strength"].is_number();
+ const bool has_extra_mask_image = extra_args.is_object() &&
+ extra_args.contains("mask_image") &&
+ !extra_args["mask_image"].is_null();
+ const bool supports_ref_image_conditioning = sd_ctx_supports_ref_image_conditioning(runtime.sd_ctx);
+ const bool may_need_uploaded_init_image = !has_extra_init_image_override &&
+ (!supports_ref_image_conditioning || !mask_bytes.empty() ||
+ has_extra_mask_image || has_extra_strength_override || has_extra_ref_images_override);
+ SDImageOwner uploaded_image_for_init;
+ bool uploaded_image_was_first_reference = false;
for (auto& bytes : images_bytes) {
int img_w = 0;
int img_h = 0;
@@ -177,10 +195,9 @@ static bool build_openai_edit_request(const httplib::Request& req,
request.gen_params.set_width_and_height_if_unset(image_owner.get().width, image_owner.get().height);
if (is_first_ref_image) {
- request.gen_params.init_image = image_owner;
- if (request.gen_params.init_image.get().data == nullptr) {
- error_message = "could not allocate init image";
- return false;
+ uploaded_image_was_first_reference = true;
+ if (may_need_uploaded_init_image) {
+ uploaded_image_for_init = image_owner;
}
}
@@ -214,6 +231,21 @@ static bool build_openai_edit_request(const httplib::Request& req,
return false;
}
+ const auto init_image_action = apply_openai_edit_init_image_policy(
+ request.gen_params.init_image,
+ uploaded_image_for_init,
+ uploaded_image_was_first_reference,
+ supports_ref_image_conditioning,
+ !request.gen_params.ref_images.empty(),
+ request.gen_params.mask_image.get().data != nullptr,
+ has_extra_init_image_override,
+ has_extra_strength_override);
+ if (init_image_action == OpenAIEditInitImageAction::UseUploaded &&
+ request.gen_params.init_image.get().data == nullptr) {
+ error_message = "could not allocate init image";
+ return false;
+ }
+
// Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
error_message = "invalid params";
diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h
index 267e7a7..404d8cb 100644
--- a/include/stable-diffusion.h
+++ b/include/stable-diffusion.h
@@ -486,6 +486,8 @@ SD_API int32_t sd_get_num_physical_cores();
SD_API const char* sd_get_system_info();
SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx);
SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx);
+/** Returns whether the loaded model consumes ref_images as conditioning inputs. */
+SD_API bool sd_ctx_supports_ref_image_conditioning(const sd_ctx_t* sd_ctx);
// ControlNet hot-swap APIs are not safe to call while generation is in flight.
SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path);
diff --git a/src/model.h b/src/model.h
index c8d803a..0c1a37a 100644
--- a/src/model.h
+++ b/src/model.h
@@ -365,6 +365,29 @@ static inline bool sd_version_is_control(SDVersion version) {
return version == VERSION_FLUX_CONTROLS || version == VERSION_FLEX_2;
}
+// These model families consume uploaded reference images as generation
+// conditioning. Inpaint and control variants keep their init-image semantics.
+static inline bool sd_version_supports_ref_image_conditioning(SDVersion version) {
+ if (sd_version_is_inpaint(version) || sd_version_is_control(version) || version == VERSION_CHROMA_RADIANCE) {
+ return false;
+ }
+
+ return sd_version_is_unet_edit(version) ||
+ sd_version_is_flux(version) ||
+ sd_version_is_flux3(version) ||
+ sd_version_is_flux2(version) ||
+ sd_version_is_qwen_image(version) ||
+ sd_version_is_longcat(version) ||
+ sd_version_is_z_image(version) ||
+ sd_version_is_boogu_image(version) ||
+ sd_version_is_llada_image(version) ||
+ sd_version_is_krea2(version) ||
+ sd_version_is_mage_flow(version) ||
+ sd_version_is_anima(version) ||
+ sd_version_is_sefi_image(version) ||
+ sd_version_is_pid(version);
+}
+
static bool sd_version_is_inpaint_or_unet_edit(SDVersion version) {
return sd_version_is_unet_edit(version) || sd_version_is_inpaint(version) || sd_version_is_control(version);
}
diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp
index 1cc4c09..d476f92 100644
--- a/src/stable-diffusion.cpp
+++ b/src/stable-diffusion.cpp
@@ -704,6 +704,13 @@ SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx) {
return sd_version_supports_video_generation(sd_ctx->sd->version);
}
+SD_API bool sd_ctx_supports_ref_image_conditioning(const sd_ctx_t* sd_ctx) {
+ if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
+ return false;
+ }
+ return sd_version_supports_ref_image_conditioning(sd_ctx->sd->version);
+}
+
SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path) {
if (sd_ctx == nullptr || sd_ctx->sd == nullptr || path == nullptr) {
return false;