Commit 0f76bf5 for stable-diffusion.cpp
commit 0f76bf5c1ec8944b24997617f20a8eada33ad9de
Author: Crystal Sight <admin@xway.ca>
Date: Sun Oct 25 04:18:47 2026 -0500
feat(server): accept ref_videos, ref_audios, ref_video_audios in sdcpp vid_gen (#2194)
diff --git a/examples/common/common.cpp b/examples/common/common.cpp
index 00301dc..637bbb6 100644
--- a/examples/common/common.cpp
+++ b/examples/common/common.cpp
@@ -1959,6 +1959,7 @@ static bool parse_image_json_field(const json& parent,
return decode_base64_image(parent.at(key).get<std::string>(), channels, 0, 0, out_image);
}
+/** Parses a JSON array of base64 image strings into owned images. */
static bool parse_image_array_json_field(const json& parent,
const char* key,
int channels,
@@ -1988,6 +1989,84 @@ static bool parse_image_array_json_field(const json& parent,
return true;
}
+/** Parses a JSON array of base64 WAV strings into owned audio clips. */
+static bool parse_audio_array_json_field(const json& parent,
+ const char* key,
+ std::vector<SDAudioOwner>& out_audios) {
+ if (!parent.contains(key)) {
+ return true;
+ }
+ if (parent.at(key).is_null()) {
+ out_audios.clear();
+ return true;
+ }
+ if (!parent.at(key).is_array()) {
+ return false;
+ }
+
+ out_audios.clear();
+ for (const auto& item : parent.at(key)) {
+ if (!item.is_string()) {
+ return false;
+ }
+ std::string encoded = item.get<std::string>();
+ auto comma_pos = encoded.find(',');
+ if (comma_pos != std::string::npos) {
+ encoded = encoded.substr(comma_pos + 1);
+ }
+ std::vector<uint8_t> audio_bytes = decode_base64_bytes(encoded);
+ if (audio_bytes.empty()) {
+ return false;
+ }
+ std::vector<float> samples;
+ uint32_t sample_rate = 0;
+ uint32_t channels = 0;
+ if (!load_wav_from_memory(audio_bytes.data(), audio_bytes.size(), samples, sample_rate, channels)) {
+ return false;
+ }
+ SDAudioOwner audio;
+ audio.reset(std::move(samples), sample_rate, channels);
+ out_audios.push_back(std::move(audio));
+ }
+ return true;
+}
+
+/** Parses a JSON array of per-video base64 frame arrays into owned videos. */
+static bool parse_video_array_json_field(const json& parent,
+ const char* key,
+ std::vector<std::vector<SDImageOwner>>& out_videos) {
+ if (!parent.contains(key)) {
+ return true;
+ }
+ if (parent.at(key).is_null()) {
+ out_videos.clear();
+ return true;
+ }
+ if (!parent.at(key).is_array()) {
+ return false;
+ }
+
+ out_videos.clear();
+ for (const auto& video : parent.at(key)) {
+ if (!video.is_array() || video.empty()) {
+ return false;
+ }
+ std::vector<SDImageOwner> frames;
+ for (const auto& frame : video) {
+ if (!frame.is_string()) {
+ return false;
+ }
+ SDImageOwner image;
+ if (!decode_base64_image(frame.get<std::string>(), 0, 0, 0, image)) {
+ return false;
+ }
+ frames.push_back(std::move(image));
+ }
+ out_videos.push_back(std::move(frames));
+ }
+ return true;
+}
+
static bool parse_lora_json_field(const json& parent,
const std::function<std::string(const std::string&)>& lora_path_resolver,
std::map<std::string, float>& lora_map,
@@ -2102,6 +2181,7 @@ bool SDGenerationParams::parse_image_preprocess_json(const std::string& json_str
return true;
}
+/** Populates generation params from a JSON payload (server APIs); returns false on any malformed field. */
bool SDGenerationParams::from_json_str(
const std::string& json_str,
const std::function<std::string(const std::string&)>& lora_path_resolver) {
@@ -2330,8 +2410,11 @@ bool SDGenerationParams::from_json_str(
return false;
}
if (!parse_image_array_json_field(j, "ref_images", 0, ref_images) ||
- !parse_image_array_json_field(j, "control_frames", 3, control_frames)) {
- LOG_ERROR("invalid input image array");
+ !parse_image_array_json_field(j, "control_frames", 3, control_frames) ||
+ !parse_audio_array_json_field(j, "ref_audios", ref_audios) ||
+ !parse_audio_array_json_field(j, "ref_video_audios", ref_video_audios) ||
+ !parse_video_array_json_field(j, "ref_videos", ref_videos)) {
+ LOG_ERROR("invalid input image/audio/video array");
return false;
}
@@ -2583,6 +2666,7 @@ bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::s
return true;
}
+/** Validates params for a generation mode; enforces mode-specific input rules. */
bool SDGenerationParams::validate(SDMode mode) {
if (!image_preprocess.empty() && mode != IMG_GEN && mode != VID_GEN) {
LOG_ERROR("--image-preprocess requires img_gen or vid_gen mode");
@@ -2636,12 +2720,13 @@ bool SDGenerationParams::validate(SDMode mode) {
return false;
}
- if (ref_video_audio_paths.size() > ref_video_paths.size()) {
+ if (ref_video_audio_paths.size() + ref_video_audios.size() > ref_video_paths.size() + ref_videos.size()) {
LOG_ERROR("error: each --ref-video-audio needs a corresponding --ref-video");
return false;
}
- if (mode != VID_GEN && (!ref_video_paths.empty() || !ref_video_audio_paths.empty() || !ref_audio_paths.empty())) {
+ if (mode != VID_GEN && (!ref_video_paths.empty() || !ref_video_audio_paths.empty() || !ref_audio_paths.empty() ||
+ !ref_videos.empty() || !ref_video_audios.empty() || !ref_audios.empty())) {
LOG_ERROR("error: reference video and audio inputs require vid_gen mode");
return false;
}
diff --git a/examples/common/media_io.cpp b/examples/common/media_io.cpp
index f09c216..6c03a2a 100644
--- a/examples/common/media_io.cpp
+++ b/examples/common/media_io.cpp
@@ -1437,70 +1437,94 @@ static uint32_t read_le32(const uint8_t* data) {
(static_cast<uint32_t>(data[3]) << 24);
}
+/** Loads a WAV file from disk and delegates decoding to load_wav_from_memory(). */
bool load_wav_from_file(const std::string& path,
std::vector<float>& interleaved_samples,
uint32_t& sample_rate,
uint32_t& channels) {
+ std::ifstream file(fs::u8path(path), std::ios::binary);
+ if (!file) {
+ interleaved_samples.clear();
+ sample_rate = 0;
+ channels = 0;
+ return false;
+ }
+ file.seekg(0, std::ios::end);
+ const std::streamoff file_size = file.tellg();
+ file.seekg(0, std::ios::beg);
+ if (file_size < 0) {
+ interleaved_samples.clear();
+ sample_rate = 0;
+ channels = 0;
+ return false;
+ }
+ std::vector<uint8_t> buffer(static_cast<size_t>(file_size));
+ if (file_size > 0 && !file.read(reinterpret_cast<char*>(buffer.data()), file_size)) {
+ interleaved_samples.clear();
+ sample_rate = 0;
+ channels = 0;
+ return false;
+ }
+ return load_wav_from_memory(buffer.data(), buffer.size(), interleaved_samples, sample_rate, channels);
+}
+
+/** Parses a RIFF/WAVE buffer in place; compares chunk sizes against the remaining
+ * bytes so payload-controlled sizes cannot wrap size_t on 32-bit builds. */
+bool load_wav_from_memory(const uint8_t* data,
+ size_t size,
+ std::vector<float>& interleaved_samples,
+ uint32_t& sample_rate,
+ uint32_t& channels) {
interleaved_samples.clear();
sample_rate = 0;
channels = 0;
- std::ifstream file(fs::u8path(path), std::ios::binary);
- uint8_t riff_header[12];
- if (!file.read(reinterpret_cast<char*>(riff_header), sizeof(riff_header)) ||
- std::memcmp(riff_header, "RIFF", 4) != 0 ||
- std::memcmp(riff_header + 8, "WAVE", 4) != 0) {
+ if (data == nullptr ||
+ size < 12 ||
+ std::memcmp(data, "RIFF", 4) != 0 ||
+ std::memcmp(data + 8, "WAVE", 4) != 0) {
return false;
}
uint16_t audio_format = 0;
uint16_t bits_per_sample = 0;
uint16_t block_align = 0;
- std::streampos data_pos = std::streampos(-1);
+ size_t data_pos = SIZE_MAX;
uint32_t data_size = 0;
- while (file.good()) {
- uint8_t chunk_header[8];
- if (!file.read(reinterpret_cast<char*>(chunk_header), sizeof(chunk_header))) {
- break;
- }
- uint32_t chunk_size = read_le32(chunk_header + 4);
- std::streampos chunk_data_pos = file.tellg();
-
- if (std::memcmp(chunk_header, "fmt ", 4) == 0) {
- if (chunk_size < 16) {
- return false;
- }
- std::vector<uint8_t> fmt(chunk_size);
- if (!file.read(reinterpret_cast<char*>(fmt.data()), chunk_size)) {
+ size_t pos = 12;
+ while (pos + 8 <= size) {
+ uint32_t chunk_size = read_le32(data + pos + 4);
+ size_t chunk_data_pos = pos + 8;
+ const size_t remaining = size - chunk_data_pos; // safe: chunk_data_pos <= size
+ // Chunk sizes come from the payload, so compare against the remaining bytes
+ // instead of adding them to offsets (avoids size_t wrap on 32-bit builds).
+ const bool data_fits = static_cast<size_t>(chunk_size) <= remaining;
+
+ if (std::memcmp(data + pos, "fmt ", 4) == 0) {
+ if (chunk_size < 16 || !data_fits) {
return false;
}
- audio_format = read_le16(fmt.data());
- channels = read_le16(fmt.data() + 2);
- sample_rate = read_le32(fmt.data() + 4);
- block_align = read_le16(fmt.data() + 12);
- bits_per_sample = read_le16(fmt.data() + 14);
+ audio_format = read_le16(data + chunk_data_pos);
+ channels = read_le16(data + chunk_data_pos + 2);
+ sample_rate = read_le32(data + chunk_data_pos + 4);
+ block_align = read_le16(data + chunk_data_pos + 12);
+ bits_per_sample = read_le16(data + chunk_data_pos + 14);
if (audio_format == 0xfffe && chunk_size >= 40) {
- audio_format = read_le16(fmt.data() + 24);
+ audio_format = read_le16(data + chunk_data_pos + 24);
}
- } else if (std::memcmp(chunk_header, "data", 4) == 0) {
+ } else if (std::memcmp(data + pos, "data", 4) == 0) {
data_pos = chunk_data_pos;
data_size = chunk_size;
- file.seekg(chunk_size, std::ios::cur);
- } else {
- file.seekg(chunk_size, std::ios::cur);
- }
-
- if (!file.good()) {
- break;
- }
- if ((chunk_size & 1) != 0) {
- file.seekg(1, std::ios::cur);
}
+ // Only advance past a chunk whose size and pad byte fully fit (keeps the sum
+ // overflow-free); a chunk truncated at the buffer end terminates the walk.
+ const bool chunk_fits = data_fits && ((chunk_size & 1) == 0 || static_cast<size_t>(chunk_size) < remaining);
+ pos = chunk_fits ? chunk_data_pos + static_cast<size_t>(chunk_size) + (chunk_size & 1) : size;
}
const uint32_t bytes_per_sample = (bits_per_sample + 7) / 8;
- if (data_pos == std::streampos(-1) || data_size == 0 || channels == 0 || sample_rate == 0 ||
+ if (data_pos == SIZE_MAX || data_size == 0 || channels == 0 || sample_rate == 0 ||
block_align == 0 || bytes_per_sample == 0 || block_align < channels * bytes_per_sample ||
(audio_format != 1 && audio_format != 3)) {
return false;
@@ -1510,16 +1534,14 @@ bool load_wav_from_file(const std::string& path,
if (frame_count == 0 || frame_count > SIZE_MAX / channels) {
return false;
}
- std::vector<uint8_t> pcm(data_size);
- file.clear();
- file.seekg(data_pos);
- if (!file.read(reinterpret_cast<char*>(pcm.data()), data_size)) {
+ if (static_cast<size_t>(data_size) > size - data_pos) {
return false;
}
+ const uint8_t* pcm = data + data_pos;
interleaved_samples.resize(static_cast<size_t>(frame_count * channels));
for (uint64_t frame = 0; frame < frame_count; ++frame) {
- const uint8_t* frame_data = pcm.data() + frame * block_align;
+ const uint8_t* frame_data = pcm + frame * block_align;
for (uint32_t channel = 0; channel < channels; ++channel) {
const uint8_t* sample_data = frame_data + channel * bytes_per_sample;
float sample = 0.0f;
diff --git a/examples/common/media_io.h b/examples/common/media_io.h
index 18042a8..7feb0ad 100644
--- a/examples/common/media_io.h
+++ b/examples/common/media_io.h
@@ -126,4 +126,11 @@ bool load_wav_from_file(const std::string& path,
uint32_t& sample_rate,
uint32_t& channels);
+/** Decodes a WAV buffer (PCM 8/16/24/32 or IEEE float 32/64) into interleaved samples. */
+bool load_wav_from_memory(const uint8_t* data,
+ size_t size,
+ std::vector<float>& interleaved_samples,
+ uint32_t& sample_rate,
+ uint32_t& channels);
+
#endif // __MEDIA_IO_H__
diff --git a/examples/server/api.md b/examples/server/api.md
index 1258b1a..b5e0250 100644
--- a/examples/server/api.md
+++ b/examples/server/api.md
@@ -183,6 +183,9 @@ Default-enabled features are `img2img`, `mask_image`, `lora`, `vae_tiling`,
| `clip_skip` | CLIP Skip |
| `img2img`, `mask_image` | Image initialization/strength and masks |
| `ref_images` | Reference image inputs |
+| `ref_videos` | Reference videos as frame arrays |
+| `ref_audios` | Standalone reference audio; runtime requires an Audio VAE or Audio Encoder |
+| `ref_video_audios` | Soundtracks paired with reference videos; runtime requires an Audio VAE |
| `controlnet` | Control image and ControlNet strength |
| `control_image` | Control image without a separate ControlNet (FLUX Canny/Depth) |
| `ip_adapter` | IP-Adapter image and strength |
diff --git a/examples/server/model_catalog.cpp b/examples/server/model_catalog.cpp
index eef6856..1041dea 100644
--- a/examples/server/model_catalog.cpp
+++ b/examples/server/model_catalog.cpp
@@ -291,6 +291,7 @@ static void apply_recipe_context(const json& settings, SDContextParams& params)
}
}
+// Known recipe feature names with their defaults; recipes may flip any of these.
static const std::map<std::string, bool> k_recipe_features = {
{"img2img", true},
{"mask_image", true},
@@ -312,6 +313,9 @@ static const std::map<std::string, bool> k_recipe_features = {
{"init_image", false},
{"end_image", false},
{"control_frames", false},
+ {"ref_videos", false},
+ {"ref_audios", false},
+ {"ref_video_audios", false},
{"vace", false},
{"high_noise", false},
{"cache_unet", false},
@@ -369,6 +373,9 @@ json model_generation_features(const ServerRuntime& runtime) {
features["controlnet"] = features.value("controlnet", false) && !runtime.ctx_params->control_net_path.empty();
features["high_noise"] = features.value("high_noise", false) && !runtime.ctx_params->high_noise_diffusion_model_path.empty();
features["preview_tae"] = features.value("preview_tae", false) && !runtime.ctx_params->taesd_path.empty();
+ features["ref_audios"] = features.value("ref_audios", false) &&
+ (!runtime.ctx_params->audio_vae_path.empty() || !runtime.ctx_params->audio_encoder_path.empty());
+ features["ref_video_audios"] = features.value("ref_video_audios", false) && !runtime.ctx_params->audio_vae_path.empty();
if (!runtime.ctx_params->pulid_weights_path.empty())
features["slg"] = false;
return features;
diff --git a/examples/server/recipes/minimax-h3-ref2va.json b/examples/server/recipes/minimax-h3-ref2va.json
index 085f5d5..2fb0417 100644
--- a/examples/server/recipes/minimax-h3-ref2va.json
+++ b/examples/server/recipes/minimax-h3-ref2va.json
@@ -45,7 +45,10 @@
},
"features": [
"flow_shift",
- "ref_images"
+ "ref_images",
+ "ref_videos",
+ "ref_audios",
+ "ref_video_audios"
],
"disabled_features": [
"img2img",
diff --git a/examples/server/recipes/wan22-s2v.json b/examples/server/recipes/wan22-s2v.json
index ba1ee4f..55440ad 100644
--- a/examples/server/recipes/wan22-s2v.json
+++ b/examples/server/recipes/wan22-s2v.json
@@ -44,7 +44,8 @@
},
"features": [
"flow_shift",
- "init_image"
+ "init_image",
+ "ref_audios"
],
"disabled_features": [
"img2img",
diff --git a/examples/server/routes_sdcpp.cpp b/examples/server/routes_sdcpp.cpp
index b972768..0dda9a9 100644
--- a/examples/server/routes_sdcpp.cpp
+++ b/examples/server/routes_sdcpp.cpp
@@ -212,11 +212,15 @@ static json make_img_gen_features_json() {
};
}
+/** Builds the default feature set advertised for vid_gen requests. */
static json make_vid_gen_features_json() {
return {
{"init_image", true},
{"end_image", true},
{"control_frames", true},
+ {"ref_videos", true},
+ {"ref_audios", true},
+ {"ref_video_audios", true},
{"high_noise_sample_params", true},
{"lora", true},
{"vae_tiling", true},
@@ -235,6 +239,7 @@ static json make_preview_modes_json() {
return modes;
}
+/** Builds the /capabilities payload: model, defaults, limits and feature maps. */
static json make_capabilities_json(ServerRuntime& runtime) {
refresh_lora_cache(runtime);
refresh_upscaler_cache(runtime);
@@ -351,8 +356,10 @@ static json make_capabilities_json(ServerRuntime& runtime) {
const json generation_features = model_generation_features(runtime);
auto recipe_features = [&](json features, bool video) {
features["init_image"] = generation_features.at(video ? "init_image" : "img2img");
- for (const char* key : {"mask_image", "ref_images", "end_image", "control_frames", "lora", "vae_tiling", "cache", "preview"}) {
- if (features.contains(key) || std::string(key) == "ref_images")
+ for (const char* key : {"mask_image", "ref_images", "end_image", "control_frames", "ref_videos", "ref_audios", "ref_video_audios", "lora", "vae_tiling", "cache", "preview"}) {
+ if (features.contains(key) || std::string(key) == "ref_images" ||
+ (video && (std::string(key) == "ref_videos" || std::string(key) == "ref_audios" ||
+ std::string(key) == "ref_video_audios")))
features[key] = generation_features.at(key);
}
if (video)